1 /*
2 * Copyright (c) Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <assert.h> /* assert */
29 #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
30 #include "mem.h"
31 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
32 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
33 #include "zstd.h" /* ZSTD_compressBound */
34 #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
35 #include "zdict.h" /* ZDICT_trainFromBuffer */
36 #include "datagen.h" /* RDG_genBuffer */
37 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
38 #include "xxhash.h" /* XXH64_* */
39 #include "seqgen.h"
40 #include "util.h"
41 #include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
42
43
44 /*-************************************
45 * Constants
46 **************************************/
47 #define KB *(1U<<10)
48 #define MB *(1U<<20)
49 #define GB *(1U<<30)
50
51 static const int nbTestsDefault = 10000;
52 static const U32 g_cLevelMax_smallTests = 10;
53 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
54 #define FUZ_COMPRESSIBILITY_DEFAULT 50
55 static const U32 prime32 = 2654435761U;
56
57
58 /*-************************************
59 * Display Macros
60 **************************************/
61 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
62 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
63 DISPLAY(__VA_ARGS__); \
64 if (g_displayLevel>=4) fflush(stderr); }
65 static U32 g_displayLevel = 2;
66
67 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
68 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
69
70 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } }
74
75 static U64 g_clockTime = 0;
76
77
78 /*-*******************************************************
79 * Check macros
80 *********************************************************/
81 #undef MIN
82 #undef MAX
83 #define MIN(a,b) ((a)<(b)?(a):(b))
84 #define MAX(a,b) ((a)>(b)?(a):(b))
85 /*! FUZ_rand() :
86 @return : a 27 bits random value, from a 32-bits `seed`.
87 `seed` is also modified */
88 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * seedPtr)89 static U32 FUZ_rand(U32* seedPtr)
90 {
91 static const U32 prime2 = 2246822519U;
92 U32 rand32 = *seedPtr;
93 rand32 *= prime32;
94 rand32 += prime2;
95 rand32 = FUZ_rotl32(rand32, 13);
96 *seedPtr = rand32;
97 return rand32 >> 5;
98 }
99
100 #define CHECK(cond, ...) { \
101 if (cond) { \
102 DISPLAY("Error => "); \
103 DISPLAY(__VA_ARGS__); \
104 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
105 (unsigned)seed, testNb, __LINE__); \
106 goto _output_error; \
107 } }
108
109 #define CHECK_Z(f) { \
110 size_t const err = f; \
111 CHECK(ZSTD_isError(err), "%s : %s ", \
112 #f, ZSTD_getErrorName(err)); \
113 }
114
115 #define CHECK_RET(ret, cond, ...) { \
116 if (cond) { \
117 DISPLAY("Error %llu => ", (unsigned long long)ret); \
118 DISPLAY(__VA_ARGS__); \
119 DISPLAY(" (line %u)\n", __LINE__); \
120 return ret; \
121 } }
122
123 #define CHECK_RET_Z(f) { \
124 size_t const err = f; \
125 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
126 #f, ZSTD_getErrorName(err)); \
127 }
128
129
130 /*======================================================
131 * Basic Unit tests
132 *======================================================*/
133
134 typedef struct {
135 void* start;
136 size_t size;
137 size_t filled;
138 } buffer_t;
139
140 static const buffer_t kBuffNull = { NULL, 0 , 0 };
141
FUZ_freeDictionary(buffer_t dict)142 static void FUZ_freeDictionary(buffer_t dict)
143 {
144 free(dict.start);
145 }
146
FUZ_createDictionary(const void * src,size_t srcSize,size_t blockSize,size_t requestedDictSize)147 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
148 {
149 buffer_t dict = kBuffNull;
150 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
151 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
152 if (!blockSizes) return kBuffNull;
153 dict.start = malloc(requestedDictSize);
154 if (!dict.start) { free(blockSizes); return kBuffNull; }
155 { size_t nb;
156 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
157 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
158 }
159 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
160 free(blockSizes);
161 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
162 dict.size = requestedDictSize;
163 dict.filled = dictSize;
164 return dict;
165 }
166 }
167
168 /* Round trips data and updates xxh with the decompressed data produced */
SEQ_roundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,void * data,size_t size,ZSTD_EndDirective endOp)169 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
170 XXH64_state_t* xxh, void* data, size_t size,
171 ZSTD_EndDirective endOp)
172 {
173 static BYTE compressed[1024];
174 static BYTE uncompressed[1024];
175
176 ZSTD_inBuffer cin = {data, size, 0};
177 size_t cret;
178
179 do {
180 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
181 ZSTD_inBuffer din = { compressed, 0, 0 };
182 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
183
184 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
185 if (ZSTD_isError(cret))
186 return cret;
187
188 din.size = cout.pos;
189 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
190 size_t dret;
191
192 dout.pos = 0;
193 dout.size = sizeof(uncompressed);
194 dret = ZSTD_decompressStream(dctx, &dout, &din);
195 if (ZSTD_isError(dret))
196 return dret;
197 XXH64_update(xxh, dout.dst, dout.pos);
198 if (dret == 0)
199 break;
200 }
201 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
202 return 0;
203 }
204
205 /* Generates some data and round trips it */
SEQ_generateRoundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,SEQ_stream * seq,SEQ_gen_type type,unsigned value)206 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
207 XXH64_state_t* xxh, SEQ_stream* seq,
208 SEQ_gen_type type, unsigned value)
209 {
210 static BYTE data[1024];
211 size_t gen;
212
213 do {
214 SEQ_outBuffer sout = {data, sizeof(data), 0};
215 size_t ret;
216 gen = SEQ_gen(seq, type, value, &sout);
217
218 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
219 if (ZSTD_isError(ret))
220 return ret;
221 } while (gen != 0);
222
223 return 0;
224 }
225
getCCtxParams(ZSTD_CCtx * zc,ZSTD_parameters * savedParams)226 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
227 {
228 int value;
229 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
236 savedParams->cParams.strategy = value;
237
238 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
241 savedParams->fParams.noDictIDFlag = !value;
242 return 0;
243 }
244
badParameters(ZSTD_CCtx * zc,ZSTD_parameters const savedParams)245 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
246 {
247 ZSTD_parameters params;
248 if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10;
249 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
250 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
251 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
252 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
253 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
254 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
255
256 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
257 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
258 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
259 return 0;
260 }
261
basicUnitTests(U32 seed,double compressibility)262 static int basicUnitTests(U32 seed, double compressibility)
263 {
264 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
265 void* CNBuffer = malloc(CNBufferSize);
266 size_t const skippableFrameSize = 200 KB;
267 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
268 void* compressedBuffer = malloc(compressedBufferSize);
269 size_t const decodedBufferSize = CNBufferSize;
270 void* decodedBuffer = malloc(decodedBufferSize);
271 size_t cSize;
272 int testResult = 0;
273 int testNb = 1;
274 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
275 ZSTD_CStream* zc = ZSTD_createCStream();
276 ZSTD_DStream* zd = ZSTD_createDStream();
277 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
278
279 ZSTD_inBuffer inBuff, inBuff2;
280 ZSTD_outBuffer outBuff;
281 buffer_t dictionary = kBuffNull;
282 size_t const dictSize = 128 KB;
283 unsigned dictID = 0;
284
285 /* Create compressible test buffer */
286 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
287 DISPLAY("Not enough memory, aborting \n");
288 goto _output_error;
289 }
290 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
291
292 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
293
294 /* Create dictionary */
295 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
296 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
297 if (!dictionary.start) {
298 DISPLAY("Error creating dictionary, aborting \n");
299 goto _output_error;
300 }
301 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
302
303 /* Basic compression test */
304 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
305 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
306 outBuff.dst = (char*)(compressedBuffer);
307 outBuff.size = compressedBufferSize;
308 outBuff.pos = 0;
309 inBuff.src = CNBuffer;
310 inBuff.size = CNBufferSize;
311 inBuff.pos = 0;
312 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
313 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
314 { size_t const r = ZSTD_endStream(zc, &outBuff);
315 if (r != 0) goto _output_error; } /* error, or some data not flushed */
316 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
317
318 /* generate skippable frame */
319 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
320 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
321 cSize = skippableFrameSize + 8;
322
323 /* Basic compression test using dict */
324 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
325 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
326 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
327 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
328 outBuff.dst = (char*)(compressedBuffer)+cSize;
329 assert(compressedBufferSize > cSize);
330 outBuff.size = compressedBufferSize - cSize;
331 outBuff.pos = 0;
332 inBuff.src = CNBuffer;
333 inBuff.size = CNBufferSize;
334 inBuff.pos = 0;
335 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
336 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
337 { size_t const r = ZSTD_endStream(zc, &outBuff);
338 if (r != 0) goto _output_error; } /* error, or some data not flushed */
339 cSize += outBuff.pos;
340 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
341 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
342
343 /* context size functions */
344 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
345 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
346 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
347 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
348 if (ZSTD_isError(cstreamSize)) goto _output_error;
349 if (ZSTD_isError(cdictSize)) goto _output_error;
350 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
351 }
352
353 /* context size functions */
354 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
355 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
356 size_t cstreamSize, cctxSize;
357 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
358 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
359 CHECK_Z(cstreamSize);
360 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
361 CHECK_Z(cctxSize);
362 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
363 ZSTD_freeCCtxParams(params);
364 DISPLAYLEVEL(3, "OK \n");
365 }
366
367 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
368 { size_t const s = ZSTD_sizeof_CStream(zc);
369 if (ZSTD_isError(s)) goto _output_error;
370 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
371 }
372
373 /* Attempt bad compression parameters */
374 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
375 { size_t r;
376 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
377 params.cParams.minMatch = 2;
378 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
379 if (!ZSTD_isError(r)) goto _output_error;
380 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
381 }
382
383 /* skippable frame test */
384 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
385 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
386 inBuff.src = compressedBuffer;
387 inBuff.size = cSize;
388 inBuff.pos = 0;
389 outBuff.dst = decodedBuffer;
390 outBuff.size = CNBufferSize;
391 outBuff.pos = 0;
392 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
393 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
394 if (r != 0) goto _output_error;
395 }
396 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
397 DISPLAYLEVEL(3, "OK \n");
398
399 /* Basic decompression test */
400 inBuff2 = inBuff;
401 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
402 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
403 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
404 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
405 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
406 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
407 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
408 DISPLAYLEVEL(3, "OK \n");
409
410 /* Re-use without init */
411 DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
412 outBuff.pos = 0;
413 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
414 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
415 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
416 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
417 DISPLAYLEVEL(3, "OK \n");
418
419 /* check regenerated data is byte exact */
420 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
421 { size_t i;
422 for (i=0; i<CNBufferSize; i++) {
423 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
424 } }
425 DISPLAYLEVEL(3, "OK \n");
426
427 /* context size functions */
428 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
429 { ZSTD_frameHeader fhi;
430 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
431 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
432 if (gfhError!=0) goto _output_error;
433 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
434 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
435 /* uses ZSTD_initDStream_usingDict() */
436 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
437 if (ZSTD_isError(s)) goto _output_error;
438 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
439 } }
440
441 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
442 { size_t const s = ZSTD_sizeof_DStream(zd);
443 if (ZSTD_isError(s)) goto _output_error;
444 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
445 }
446
447 /* Decompression by small increment */
448 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
449 { /* skippable frame */
450 size_t r = 1;
451 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
452 inBuff.src = compressedBuffer;
453 outBuff.dst = decodedBuffer;
454 inBuff.pos = 0;
455 outBuff.pos = 0;
456 while (r) { /* skippable frame */
457 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
458 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
459 inBuff.size = inBuff.pos + inSize;
460 outBuff.size = outBuff.pos + outSize;
461 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
462 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
463 if (ZSTD_isError(r)) goto _output_error;
464 }
465 /* normal frame */
466 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
467 r=1;
468 while (r) {
469 size_t const inSize = FUZ_rand(&coreSeed) & 15;
470 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
471 inBuff.size = inBuff.pos + inSize;
472 outBuff.size = outBuff.pos + outSize;
473 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
474 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
475 if (ZSTD_isError(r)) goto _output_error;
476 }
477 }
478 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
479 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
480 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
481 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
482 DISPLAYLEVEL(3, "OK \n");
483
484 /* check regenerated data is byte exact */
485 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
486 { size_t i;
487 for (i=0; i<CNBufferSize; i++) {
488 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
489 } }
490 DISPLAYLEVEL(3, "OK \n");
491
492 /* Decompression forward progress */
493 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
494 { /* skippable frame */
495 size_t r = 0;
496 int decNb = 0;
497 int const maxDec = 100;
498 inBuff.src = compressedBuffer;
499 inBuff.size = cSize;
500 inBuff.pos = 0;
501
502 outBuff.dst = decodedBuffer;
503 outBuff.pos = 0;
504 outBuff.size = CNBufferSize-1; /* 1 byte missing */
505
506 for (decNb=0; decNb<maxDec; decNb++) {
507 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
508 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
509 if (ZSTD_isError(r)) break;
510 }
511 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
512 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
513 }
514 DISPLAYLEVEL(3, "OK \n");
515
516 DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
517 inBuff.src = NULL;
518 inBuff.size = 0;
519 inBuff.pos = 0;
520 outBuff.dst = NULL;
521 outBuff.size = 0;
522 outBuff.pos = 0;
523 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
524 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
525 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
526 outBuff.dst = (char*)(compressedBuffer);
527 outBuff.size = compressedBufferSize;
528 outBuff.pos = 0;
529 { size_t const r = ZSTD_endStream(zc, &outBuff);
530 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
531 }
532 inBuff.src = outBuff.dst;
533 inBuff.size = outBuff.pos;
534 inBuff.pos = 0;
535 outBuff.dst = NULL;
536 outBuff.size = 0;
537 outBuff.pos = 0;
538 CHECK_Z( ZSTD_initDStream(zd) );
539 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
540 if (ret != 0) goto _output_error;
541 }
542 DISPLAYLEVEL(3, "OK\n");
543 /* _srcSize compression test */
544 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
545 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
546 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
547 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
548 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
549 outBuff.dst = (char*)(compressedBuffer);
550 outBuff.size = compressedBufferSize;
551 outBuff.pos = 0;
552 inBuff.src = CNBuffer;
553 inBuff.size = CNBufferSize;
554 inBuff.pos = 0;
555 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
556 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
557 { size_t const r = ZSTD_endStream(zc, &outBuff);
558 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
559 }
560 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
561 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
562 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
563 }
564 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
565
566 /* wrong _srcSize compression test */
567 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
568 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
569 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
570 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
571 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
572 outBuff.dst = (char*)(compressedBuffer);
573 outBuff.size = compressedBufferSize;
574 outBuff.pos = 0;
575 inBuff.src = CNBuffer;
576 inBuff.size = CNBufferSize;
577 inBuff.pos = 0;
578 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
579 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
580 { size_t const r = ZSTD_endStream(zc, &outBuff);
581 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
582 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
583
584 /* wrong _srcSize compression test */
585 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
586 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
587 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
588 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
589 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
590 outBuff.dst = (char*)(compressedBuffer);
591 outBuff.size = compressedBufferSize;
592 outBuff.pos = 0;
593 inBuff.src = CNBuffer;
594 inBuff.size = CNBufferSize;
595 inBuff.pos = 0;
596 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
597 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
598 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
599 }
600
601 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
602 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
603 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
604 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
605 outBuff.dst = (char*)compressedBuffer;
606 outBuff.size = compressedBufferSize;
607 outBuff.pos = 0;
608 inBuff.src = CNBuffer;
609 inBuff.size = CNBufferSize;
610 inBuff.pos = 0;
611 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
612 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
613 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
614 } }
615
616 /* Complex context re-use scenario */
617 DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
618 ZSTD_freeCStream(zc);
619 zc = ZSTD_createCStream();
620 if (zc==NULL) goto _output_error; /* memory allocation issue */
621 /* use 1 */
622 { size_t const inSize = 513;
623 DISPLAYLEVEL(5, "use1 ");
624 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
625 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
626 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
627 inBuff.src = CNBuffer;
628 inBuff.size = inSize;
629 inBuff.pos = 0;
630 outBuff.dst = (char*)(compressedBuffer)+cSize;
631 outBuff.size = ZSTD_compressBound(inSize);
632 outBuff.pos = 0;
633 DISPLAYLEVEL(5, "compress1 ");
634 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
635 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
636 DISPLAYLEVEL(5, "end1 ");
637 { size_t const r = ZSTD_endStream(zc, &outBuff);
638 if (r != 0) goto _output_error; } /* error, or some data not flushed */
639 }
640 /* use 2 */
641 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
642 DISPLAYLEVEL(5, "use2 ");
643 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
644 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
645 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
646 inBuff.src = CNBuffer;
647 inBuff.size = inSize;
648 inBuff.pos = 0;
649 outBuff.dst = (char*)(compressedBuffer)+cSize;
650 outBuff.size = ZSTD_compressBound(inSize);
651 outBuff.pos = 0;
652 DISPLAYLEVEL(5, "compress2 ");
653 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
654 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
655 DISPLAYLEVEL(5, "end2 ");
656 { size_t const r = ZSTD_endStream(zc, &outBuff);
657 if (r != 0) goto _output_error; } /* error, or some data not flushed */
658 }
659 DISPLAYLEVEL(3, "OK \n");
660
661 /* Decompression single pass with empty frame */
662 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
663 CHECK_Z(cSize);
664 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
665 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
666 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
667 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
668
669 outBuff.dst = decodedBuffer;
670 outBuff.pos = 0;
671 outBuff.size = CNBufferSize;
672
673 inBuff.src = compressedBuffer;
674 inBuff.size = cSize;
675 inBuff.pos = 0;
676 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
677 CHECK_Z(r);
678 CHECK(r != 0, "Entire frame must be decompressed");
679 CHECK(outBuff.pos != 0, "Wrong size!");
680 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
681 }
682 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
683 ZSTD_freeDCtx(dctx);
684 }
685 DISPLAYLEVEL(3, "OK \n");
686
687 /* Decompression with ZSTD_d_stableOutBuffer */
688 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
689 CHECK_Z(cSize);
690 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
691 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
692 size_t dctxSize1;
693 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
694
695 outBuff.dst = decodedBuffer;
696 outBuff.pos = 0;
697 outBuff.size = CNBufferSize;
698
699 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
700 inBuff.src = compressedBuffer;
701 inBuff.size = cSize;
702 inBuff.pos = 0;
703 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
704 CHECK_Z(r);
705 CHECK(r != 0, "Entire frame must be decompressed");
706 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
707 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
708 }
709 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
710 DISPLAYLEVEL(3, "OK \n");
711
712 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
713 outBuff.pos = 0;
714 inBuff.pos = 0;
715 inBuff.size = 0;
716 while (inBuff.pos < cSize) {
717 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
718 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
719 }
720 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
721 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
722 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
723 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
724 DISPLAYLEVEL(3, "OK \n");
725
726 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
727 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
728 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
729 inBuff.src = compressedBuffer;
730 inBuff.size = cSize;
731 inBuff.pos = 0;
732 outBuff.pos = 0;
733 outBuff.size = CNBufferSize - 1;
734 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
735 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
736 }
737 DISPLAYLEVEL(3, "OK \n");
738
739 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
740 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
741 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
742 inBuff.src = compressedBuffer;
743 inBuff.size = cSize - 1;
744 inBuff.pos = 0;
745 outBuff.pos = 0;
746 outBuff.size = CNBufferSize;
747 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
748 ++inBuff.size;
749 outBuff.pos = 0;
750 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
751 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
752 }
753 DISPLAYLEVEL(3, "OK \n");
754
755 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
756 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
757 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
758 outBuff.pos = 0;
759 inBuff.pos = 0;
760 inBuff.size = 0;
761 while (inBuff.pos < cSize) {
762 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
763 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
764 }
765 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
766 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
767 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
768 DISPLAYLEVEL(3, "OK \n");
769
770 ZSTD_freeDCtx(dctx);
771 }
772
773 /* Compression with ZSTD_c_stable{In,Out}Buffer */
774 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
775 ZSTD_inBuffer in;
776 ZSTD_outBuffer out;
777 size_t cctxSize1;
778 size_t cctxSize2;
779 in.src = CNBuffer;
780 in.size = CNBufferSize;
781 out.dst = compressedBuffer;
782 out.size = compressedBufferSize;
783 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
784 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
785 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
786 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
787 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
788 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
789 { ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
790 in.pos = out.pos = 0;
791 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
792 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
793 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
794 ZSTD_freeCCtx(cctx2);
795 }
796 { ZSTD_CCtx* cctx3 = ZSTD_createCCtx();
797 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
798 size_t cSize3;
799 params.fParams.checksumFlag = 1;
800 cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
801 CHECK_Z(cSize3);
802 CHECK(!(cSize == cSize3), "Must be same compressed size");
803 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx3)), "Must be same CCtx size");
804 ZSTD_freeCCtx(cctx3);
805 }
806 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
807 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
808 DISPLAYLEVEL(3, "OK \n");
809
810 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
811 {
812 int stableInBuffer;
813 int stableOutBuffer;
814 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
815 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
816 CHECK(!(stableInBuffer == 0), "Modified");
817 CHECK(!(stableOutBuffer == 0), "Modified");
818 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
819 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
820 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
821 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
822 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
823 CHECK(!(stableInBuffer == 1), "Modified");
824 CHECK(!(stableOutBuffer == 1), "Modified");
825 }
826 DISPLAYLEVEL(3, "OK \n");
827
828 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
829 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
830 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
831 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
832 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
833 in.pos = out.pos = 0;
834 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
835 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
836 DISPLAYLEVEL(3, "OK \n");
837
838 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
839 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
840 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
841 }
842 DISPLAYLEVEL(3, "OK \n");
843
844 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
845 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
846 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
847 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
848 in.pos = out.pos = 0;
849 out.size = cSize / 4;
850 for (;;) {
851 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
852 CHECK_Z(ret);
853 if (ret == 0)
854 break;
855 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
856 }
857 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
858 DISPLAYLEVEL(3, "OK \n");
859
860 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
861 in.pos = out.pos = 0;
862 out.size = cSize / 4;
863 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
864 in.src = (char const*)in.src + in.pos;
865 in.size -= in.pos;
866 in.pos = 0;
867 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
868 CHECK(!ZSTD_isError(ret), "Must error");
869 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
870 }
871 DISPLAYLEVEL(3, "OK \n");
872
873 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++);
874 in.src = CNBuffer;
875 in.size = CNBufferSize;
876 in.pos = 0;
877 out.pos = 0;
878 out.size = compressedBufferSize;
879 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
880 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
881 CHECK(!ZSTD_isError(ret), "Must error");
882 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
883 }
884 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
885 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
886 CHECK(!ZSTD_isError(ret), "Must error");
887 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
888 }
889 DISPLAYLEVEL(3, "OK \n");
890
891 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer allocated size : ", testNb++);
892 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
893 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
894 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
895 cctxSize1 = cctxSize;
896 }
897 DISPLAYLEVEL(3, "OK \n");
898
899 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
900 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
901 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
902 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
903 in.pos = out.pos = 0;
904 in.size = MIN(CNBufferSize, 10);
905 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
906 in.pos = 0;
907 in.size = CNBufferSize - in.size;
908 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
909 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
910 DISPLAYLEVEL(3, "OK \n");
911
912 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
913 in.pos = out.pos = 0;
914 in.size = CNBufferSize;
915 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
916 in.pos = out.pos = 0;
917 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
918 CHECK(!ZSTD_isError(ret), "Must have errored");
919 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_dstBuffer_wrong), "Must be this error");
920 }
921 DISPLAYLEVEL(3, "OK \n");
922
923 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer allocated size : ", testNb++);
924 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
925 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
926 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
927 }
928 DISPLAYLEVEL(3, "OK \n");
929
930 ZSTD_freeCCtx(cctx);
931 }
932
933 /* CDict scenario */
934 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
935 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
936 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
937 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
938 if (ZSTD_isError(initError)) goto _output_error;
939 outBuff.dst = compressedBuffer;
940 outBuff.size = compressedBufferSize;
941 outBuff.pos = 0;
942 inBuff.src = CNBuffer;
943 inBuff.size = CNBufferSize;
944 inBuff.pos = 0;
945 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
946 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
947 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
948 { size_t const r = ZSTD_endStream(zc, &outBuff);
949 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
950 if (r != 0) goto _output_error; /* error, or some data not flushed */
951 }
952 cSize = outBuff.pos;
953 ZSTD_freeCDict(cdict);
954 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
955 }
956
957 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
958 { size_t const s = ZSTD_sizeof_CStream(zc);
959 if (ZSTD_isError(s)) goto _output_error;
960 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
961 }
962
963 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
964 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
965 if (dID != dictID) goto _output_error;
966 DISPLAYLEVEL(4, "OK (%u) \n", dID);
967 }
968
969 /* DDict scenario */
970 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
971 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
972 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
973 if (ZSTD_isError(initError)) goto _output_error;
974 outBuff.dst = decodedBuffer;
975 outBuff.size = CNBufferSize;
976 outBuff.pos = 0;
977 inBuff.src = compressedBuffer;
978 inBuff.size = cSize;
979 inBuff.pos = 0;
980 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
981 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
982 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
983 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
984 ZSTD_freeDDict(ddict);
985 DISPLAYLEVEL(3, "OK \n");
986 }
987
988 /* Memory restriction */
989 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
990 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
991 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
992 outBuff.dst = decodedBuffer;
993 outBuff.size = CNBufferSize;
994 outBuff.pos = 0;
995 inBuff.src = compressedBuffer;
996 inBuff.size = cSize;
997 inBuff.pos = 0;
998 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
999 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1000 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1001 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1002
1003 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1004 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1005 int const maxLevel = 16; /* first level with zstd_opt */
1006 int level;
1007 assert(maxLevel < ZSTD_maxCLevel());
1008 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1009 for (level = 1; level <= maxLevel; ++level) {
1010 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1011 size_t const maxSize = MIN(1 MB, CNBufferSize);
1012 size_t size;
1013 for (size = 512; size <= maxSize; size <<= 1) {
1014 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1015 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1016 ZSTD_parameters savedParams;
1017 getCCtxParams(cctx, &savedParams);
1018 outBuff.dst = compressedBuffer;
1019 outBuff.size = compressedBufferSize;
1020 outBuff.pos = 0;
1021 inBuff.src = CNBuffer;
1022 inBuff.size = size;
1023 inBuff.pos = 0;
1024 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1025 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1026 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1027 if (inBuff.pos != inBuff.size) goto _output_error;
1028 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1029 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1030 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1031 if (decIn.pos != decIn.size) goto _output_error;
1032 if (decOut.pos != size) goto _output_error;
1033 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1034 if (crcDec != crcOrig) goto _output_error;
1035 } }
1036 ZSTD_freeCCtx(cctx);
1037 }
1038 ZSTD_freeCDict(cdict);
1039 }
1040 ZSTD_freeDCtx(dctx);
1041 }
1042 DISPLAYLEVEL(3, "OK\n");
1043
1044 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1045 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1046 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1047 CHECK_Z(cSize);
1048 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1049 {
1050 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1051 /* We should fail to decompress without a dictionary. */
1052 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1053 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1054 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1055 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1056 if (!ZSTD_isError(ret)) goto _output_error;
1057 }
1058 /* We should succeed to decompress with the dictionary. */
1059 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1060 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1061 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1062 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1063 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1064 if (in.pos != in.size) goto _output_error;
1065 }
1066 /* The dictionary should presist across calls. */
1067 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1068 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1069 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1070 if (in.pos != in.size) goto _output_error;
1071 }
1072 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1073 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1074 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1075 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1076 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1077 if (in.pos != in.size) goto _output_error;
1078 }
1079 /* When we reset the context the dictionary is cleared. */
1080 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1081 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1082 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1083 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1084 if (!ZSTD_isError(ret)) goto _output_error;
1085 }
1086 ZSTD_freeDCtx(dctx);
1087 }
1088 DISPLAYLEVEL(3, "OK \n");
1089
1090 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1091 {
1092 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1093 /* We should succeed to decompress with the dictionary. */
1094 ZSTD_resetDStream(dctx);
1095 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1096 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1097 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1098 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1099 if (in.pos != in.size) goto _output_error;
1100 }
1101 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1102 ZSTD_resetDStream(dctx);
1103 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1104 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1105 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1106 if (in.pos != in.size) goto _output_error;
1107 }
1108 /* The dictionary should be cleared by ZSTD_initDStream(). */
1109 CHECK_Z( ZSTD_initDStream(dctx) );
1110 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1111 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1112 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1113 if (!ZSTD_isError(ret)) goto _output_error;
1114 }
1115 ZSTD_freeDCtx(dctx);
1116 }
1117 DISPLAYLEVEL(3, "OK \n");
1118
1119 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1120 {
1121 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1122 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1123 /* We should succeed to decompress with the ddict. */
1124 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1125 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1126 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1127 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1128 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1129 if (in.pos != in.size) goto _output_error;
1130 }
1131 /* The ddict should presist across calls. */
1132 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1133 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1134 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1135 if (in.pos != in.size) goto _output_error;
1136 }
1137 /* When we reset the context the ddict is cleared. */
1138 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1139 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1140 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1141 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1142 if (!ZSTD_isError(ret)) goto _output_error;
1143 }
1144 ZSTD_freeDCtx(dctx);
1145 ZSTD_freeDDict(ddict);
1146 }
1147 DISPLAYLEVEL(3, "OK \n");
1148
1149 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1150 {
1151 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1152 /* We should succeed to decompress with the prefix. */
1153 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1154 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1155 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1156 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1157 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1158 if (in.pos != in.size) goto _output_error;
1159 }
1160 /* The prefix should be cleared after the first compression. */
1161 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1162 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1163 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1164 if (!ZSTD_isError(ret)) goto _output_error;
1165 }
1166 ZSTD_freeDCtx(dctx);
1167 }
1168 DISPLAYLEVEL(3, "OK \n");
1169
1170 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1171 {
1172 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1173 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1174 size_t ret;
1175 /* We should succeed to decompress with the dictionary. */
1176 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1177 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1178 /* The dictionary should presist across calls. */
1179 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1180 /* We should succeed to decompress with the ddict. */
1181 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1182 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1183 /* The ddict should presist across calls. */
1184 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1185 /* When we reset the context the ddict is cleared. */
1186 CHECK_Z( ZSTD_initDStream(dctx) );
1187 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1188 if (!ZSTD_isError(ret)) goto _output_error;
1189 ZSTD_freeDCtx(dctx);
1190 ZSTD_freeDDict(ddict);
1191 }
1192 DISPLAYLEVEL(3, "OK \n");
1193
1194 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1195 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1196 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1197 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1198 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1199 if (ZSTD_isError(initError)) goto _output_error;
1200 outBuff.dst = compressedBuffer;
1201 outBuff.size = compressedBufferSize;
1202 outBuff.pos = 0;
1203 inBuff.src = CNBuffer;
1204 inBuff.size = CNBufferSize;
1205 inBuff.pos = 0;
1206 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1207 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1208 { size_t const r = ZSTD_endStream(zc, &outBuff);
1209 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1210 cSize = outBuff.pos;
1211 ZSTD_freeCDict(cdict);
1212 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1213 }
1214
1215 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1216 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1217 if (did != 0) goto _output_error;
1218 }
1219 DISPLAYLEVEL(3, "OK (not detected) \n");
1220
1221 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1222 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1223 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1224 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1225 }
1226
1227 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1228 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1229 outBuff.dst = compressedBuffer;
1230 outBuff.size = compressedBufferSize;
1231 outBuff.pos = 0;
1232 inBuff.src = CNBuffer;
1233 inBuff.size = CNBufferSize;
1234 inBuff.pos = 0;
1235 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1236 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1237 cSize = outBuff.pos;
1238 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1239
1240 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1241 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1242 outBuff.dst = decodedBuffer;
1243 outBuff.size = CNBufferSize;
1244 outBuff.pos = 0;
1245 inBuff.src = compressedBuffer;
1246 inBuff.size = cSize;
1247 inBuff.pos = 0;
1248 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1249 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1250 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1251 DISPLAYLEVEL(3, "OK \n");
1252
1253 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1254 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1255 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1256 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1257 }
1258
1259 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1260 outBuff.dst = compressedBuffer;
1261 outBuff.size = compressedBufferSize;
1262 outBuff.pos = 0;
1263 inBuff.src = CNBuffer;
1264 inBuff.size = CNBufferSize;
1265 inBuff.pos = 0;
1266 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1267 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1268 cSize = outBuff.pos;
1269 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1270
1271 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1272 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1273 DISPLAYLEVEL(3, "OK \n");
1274
1275 /* Empty srcSize */
1276 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1277 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1278 params.fParams.contentSizeFlag = 1;
1279 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1280 } /* cstream advanced shall write content size = 0 */
1281 outBuff.dst = compressedBuffer;
1282 outBuff.size = compressedBufferSize;
1283 outBuff.pos = 0;
1284 inBuff.src = CNBuffer;
1285 inBuff.size = 0;
1286 inBuff.pos = 0;
1287 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1288 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1289 cSize = outBuff.pos;
1290 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1291 DISPLAYLEVEL(3, "OK \n");
1292
1293 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1294 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1295 params.fParams.contentSizeFlag = 1;
1296 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1297 } /* cstream advanced shall write content size = 0 */
1298 inBuff.src = CNBuffer;
1299 inBuff.size = 0;
1300 inBuff.pos = 0;
1301 outBuff.dst = compressedBuffer;
1302 outBuff.size = compressedBufferSize;
1303 outBuff.pos = 0;
1304 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1305 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1306 cSize = outBuff.pos;
1307 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1308
1309 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1310 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1311 outBuff.dst = compressedBuffer;
1312 outBuff.size = compressedBufferSize;
1313 outBuff.pos = 0;
1314 inBuff.src = CNBuffer;
1315 inBuff.size = 0;
1316 inBuff.pos = 0;
1317 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1318 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1319 cSize = outBuff.pos;
1320 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1321 DISPLAYLEVEL(3, "OK \n");
1322
1323 /* Basic multithreading compression test */
1324 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1325 { int jobSize;
1326 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1327 CHECK(jobSize != 0, "job size non-zero");
1328 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1329 CHECK(jobSize != 0, "job size non-zero");
1330 }
1331 outBuff.dst = compressedBuffer;
1332 outBuff.size = compressedBufferSize;
1333 outBuff.pos = 0;
1334 inBuff.src = CNBuffer;
1335 inBuff.size = CNBufferSize;
1336 inBuff.pos = 0;
1337 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1338 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1339 }
1340 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1341 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1342 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1343 }
1344 DISPLAYLEVEL(3, "OK \n");
1345
1346 /* Complex multithreading + dictionary test */
1347 { U32 const nbWorkers = 2;
1348 size_t const jobSize = 4 * 1 MB;
1349 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1350 size_t const segLength = 2 KB;
1351 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1352 size_t const start = jobSize + (offset-1);
1353 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1354 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1355 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1356 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1357 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1358 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1359 assert(start > offset);
1360 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1361 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1362 outBuff.dst = compressedBuffer;
1363 outBuff.size = compressedBufferSize;
1364 outBuff.pos = 0;
1365 inBuff.src = CNBuffer;
1366 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1367 inBuff.pos = 0;
1368 }
1369 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1370 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1371 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1372 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1373 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1374 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1375 ZSTD_freeCDict(cdict);
1376 }
1377 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1378 cSize = outBuff.pos;
1379 DISPLAYLEVEL(3, "OK \n");
1380
1381 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1382 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1383 ZSTD_frameHeader zfh;
1384 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1385 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1386 outBuff.dst = decodedBuffer;
1387 outBuff.size = CNBufferSize;
1388 outBuff.pos = 0;
1389 inBuff.src = compressedBuffer;
1390 inBuff.pos = 0;
1391 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1392 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1393 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1394 inBuff.size = cSize;
1395 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1396 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1397 ZSTD_freeDStream(dstream);
1398 }
1399 DISPLAYLEVEL(3, "OK \n");
1400
1401 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1402 { unsigned const kMaxWindowLog = 24;
1403 unsigned value;
1404 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1405 ZSTD_CDict* cdict;
1406 ZSTD_DDict* ddict;
1407 SEQ_stream seq = SEQ_initStream(0x87654321);
1408 SEQ_gen_type type;
1409 XXH64_state_t xxh;
1410
1411 XXH64_reset(&xxh, 0);
1412 cParams.windowLog = kMaxWindowLog;
1413 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1414 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1415
1416 if (!cdict || !ddict) goto _output_error;
1417
1418 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1419 ZSTD_resetDStream(zd);
1420 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1421 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1422 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1423 /* Test all values < 300 */
1424 for (value = 0; value < 300; ++value) {
1425 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1426 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1427 }
1428 }
1429 /* Test values 2^8 to 2^17 */
1430 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1431 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1432 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1433 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1434 }
1435 }
1436 /* Test offset values up to the max window log */
1437 for (value = 8; value <= kMaxWindowLog; ++value) {
1438 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1439 }
1440
1441 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1442 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1443
1444 ZSTD_freeCDict(cdict);
1445 ZSTD_freeDDict(ddict);
1446 }
1447 DISPLAYLEVEL(3, "OK \n");
1448
1449 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1450 { int level;
1451 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1452 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1453 CHECK(level != 11, "Compression level does not match");
1454 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1455 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1456 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1457 CHECK(level != 11, "Compression level does not match");
1458 }
1459 DISPLAYLEVEL(3, "OK \n");
1460
1461 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1462 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1463 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1464 CHECK(badParameters(zc, params), "Compression parameters do not match");
1465 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1466 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1467 CHECK(badParameters(zc, params), "Compression parameters do not match");
1468 }
1469 DISPLAYLEVEL(3, "OK \n");
1470
1471 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1472 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1473 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1474 { int srcSizeHint;
1475 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1476 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1477 }
1478 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1479 DISPLAYLEVEL(3, "OK \n");
1480
1481 /* Overlen overwriting window data bug */
1482 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1483 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1484 1. 'a' repeated 517 times
1485 2. 'b' repeated 516 times
1486 3. a compressed block with no literals and 3 sequence commands:
1487 litlength = 0, offset = 24, match length = 24
1488 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1489 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1490
1491 const char* testCase =
1492 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1493 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1494 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1495 ZSTD_DStream* const zds = ZSTD_createDStream();
1496 if (zds==NULL) goto _output_error;
1497
1498 CHECK_Z( ZSTD_initDStream(zds) );
1499 inBuff.src = testCase;
1500 inBuff.size = 47;
1501 inBuff.pos = 0;
1502 outBuff.dst = decodedBuffer;
1503 outBuff.size = CNBufferSize;
1504 outBuff.pos = 0;
1505
1506 while (inBuff.pos < inBuff.size) {
1507 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1508 }
1509
1510 ZSTD_freeDStream(zds);
1511 }
1512 DISPLAYLEVEL(3, "OK \n");
1513
1514 /* Small Sequence Section bug */
1515 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1516 { /* This test consists of 3 blocks. Each block has one sequence.
1517 The sequence has literal length of 10, match length of 10 and offset of 10.
1518 The sequence value and compression mode for the blocks are following:
1519 The order of values are ll, ml, of.
1520 - First block : (10, 7, 13) (rle, rle, rle)
1521 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1522 - Second block : (10, 7, 1) (repeat, repeat, rle)
1523 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1524 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1525 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1526
1527 unsigned char compressed[] = {
1528 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1529 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1530 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1531 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1532 0x40, 0x0a, 0xa4
1533 };
1534 unsigned int compressedSize = 51;
1535 unsigned char decompressed[] = {
1536 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1537 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1538 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1539 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1540 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1541 };
1542 unsigned int decompressedSize = 60;
1543
1544 ZSTD_DStream* const zds = ZSTD_createDStream();
1545 if (zds==NULL) goto _output_error;
1546
1547 CHECK_Z( ZSTD_initDStream(zds) );
1548 inBuff.src = compressed;
1549 inBuff.size = compressedSize;
1550 inBuff.pos = 0;
1551 outBuff.dst = decodedBuffer;
1552 outBuff.size = CNBufferSize;
1553 outBuff.pos = 0;
1554
1555 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1556 "Decompress did not reach the end of frame");
1557 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1558 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1559 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1560 "Decompressed data does not match");
1561
1562 ZSTD_freeDStream(zds);
1563 }
1564 DISPLAYLEVEL(3, "OK \n");
1565
1566 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1567 { size_t const inputSize = 10000;
1568 size_t const compCapacity = ZSTD_compressBound(inputSize);
1569 BYTE* const input = (BYTE*)malloc(inputSize);
1570 BYTE* const comp = (BYTE*)malloc(compCapacity);
1571 BYTE* const decomp = (BYTE*)malloc(inputSize);
1572
1573 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1574
1575 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1576 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1577 ZSTD_inBuffer in = { comp, 0, 0 };
1578 ZSTD_outBuffer out = { decomp, 0, 0 };
1579 CHECK_Z(compSize);
1580 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1581 while (in.size < compSize) {
1582 in.size = MIN(in.size + 100, compSize);
1583 while (in.pos < in.size) {
1584 size_t const outPos = out.pos;
1585 if (out.pos == out.size) {
1586 out.size = MIN(out.size + 10, inputSize);
1587 }
1588 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1589 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1590 }
1591 }
1592 CHECK(in.pos != compSize, "Not all input consumed!");
1593 CHECK(out.pos != inputSize, "Not all output produced!");
1594 }
1595 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1596
1597 free(input);
1598 free(comp);
1599 free(decomp);
1600 }
1601 DISPLAYLEVEL(3, "OK \n");
1602
1603 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1604 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1605 dictionary.start, dictionary.filled,
1606 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1607 ZSTD_getCParams(3, 0, dictionary.filled),
1608 ZSTD_defaultCMem);
1609 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1610 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1611 size_t inbufpos = 0;
1612 size_t cursegmentlen;
1613 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1614 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1615 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1616 size_t ret;
1617
1618 CHECK(cdict == NULL, "failed to alloc cdict");
1619 CHECK(inbuf == NULL, "failed to alloc input buffer");
1620
1621 /* first block is uncompressible */
1622 cursegmentlen = 128 * 1024;
1623 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1624 inbufpos += cursegmentlen;
1625
1626 /* second block is compressible */
1627 cursegmentlen = 128 * 1024 - 256;
1628 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1629 inbufpos += cursegmentlen;
1630
1631 /* and includes a very long backref */
1632 cursegmentlen = 128;
1633 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1634 inbufpos += cursegmentlen;
1635
1636 /* and includes a very long backref */
1637 cursegmentlen = 128;
1638 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1639 inbufpos += cursegmentlen;
1640
1641 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1642 CHECK_Z(ret);
1643
1644 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1645 CHECK_Z(ret);
1646
1647 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1648
1649 ZSTD_freeCDict(cdict);
1650 free(inbuf);
1651 free(outbuf);
1652 free(checkbuf);
1653 }
1654 DISPLAYLEVEL(3, "OK \n");
1655
1656 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1657 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1658 dictionary.start, dictionary.filled,
1659 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1660 ZSTD_getCParams(3, 0, dictionary.filled),
1661 ZSTD_defaultCMem);
1662 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1663 int remainingInput = 256 * 1024;
1664 int offset;
1665
1666 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1667 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1668 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1669 /* Write a bunch of 6 byte blocks */
1670 while (remainingInput > 0) {
1671 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1672 const size_t kSmallBlockSize = sizeof(testBuffer);
1673 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1674
1675 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1676 CHECK(in.pos != in.size, "input not fully consumed");
1677 remainingInput -= kSmallBlockSize;
1678 }
1679 /* Write several very long offset matches into the dictionary */
1680 for (offset = 1024; offset >= 0; offset -= 128) {
1681 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1682 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1683 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1684 CHECK(in.pos != in.size, "input not fully consumed");
1685 }
1686 /* Ensure decompression works */
1687 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1688
1689 ZSTD_freeCDict(cdict);
1690 }
1691 DISPLAYLEVEL(3, "OK \n");
1692
1693 _end:
1694 FUZ_freeDictionary(dictionary);
1695 ZSTD_freeCStream(zc);
1696 ZSTD_freeDStream(zd);
1697 ZSTD_freeCCtx(mtctx);
1698 free(CNBuffer);
1699 free(compressedBuffer);
1700 free(decodedBuffer);
1701 return testResult;
1702
1703 _output_error:
1704 testResult = 1;
1705 DISPLAY("Error detected in Unit tests ! \n");
1706 goto _end;
1707 }
1708
1709
1710 /* ====== Fuzzer tests ====== */
1711
findDiff(const void * buf1,const void * buf2,size_t max)1712 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
1713 {
1714 const BYTE* b1 = (const BYTE*)buf1;
1715 const BYTE* b2 = (const BYTE*)buf2;
1716 size_t u;
1717 for (u=0; u<max; u++) {
1718 if (b1[u] != b2[u]) break;
1719 }
1720 if (u==max) {
1721 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
1722 return u;
1723 }
1724 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
1725 if (u>=3)
1726 DISPLAY(" %02X %02X %02X ",
1727 b1[u-3], b1[u-2], b1[u-1]);
1728 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
1729 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
1730 if (u>=3)
1731 DISPLAY(" %02X %02X %02X ",
1732 b2[u-3], b2[u-2], b2[u-1]);
1733 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
1734 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
1735 return u;
1736 }
1737
FUZ_rLogLength(U32 * seed,U32 logLength)1738 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
1739 {
1740 size_t const lengthMask = ((size_t)1 << logLength) - 1;
1741 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
1742 }
1743
FUZ_randomLength(U32 * seed,U32 maxLog)1744 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
1745 {
1746 U32 const logLength = FUZ_rand(seed) % maxLog;
1747 return FUZ_rLogLength(seed, logLength);
1748 }
1749
1750 /* Return value in range minVal <= v <= maxVal */
FUZ_randomClampedLength(U32 * seed,U32 minVal,U32 maxVal)1751 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
1752 {
1753 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
1754 return (U32)((FUZ_rand(seed) % mod) + minVal);
1755 }
1756
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,int bigTests)1757 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
1758 {
1759 U32 const maxSrcLog = bigTests ? 24 : 22;
1760 static const U32 maxSampleLog = 19;
1761 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
1762 BYTE* cNoiseBuffer[5];
1763 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
1764 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
1765 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
1766 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
1767 size_t const dstBufferSize = srcBufferSize;
1768 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
1769 U32 result = 0;
1770 unsigned testNb = 0;
1771 U32 coreSeed = seed;
1772 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
1773 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
1774 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
1775 UTIL_time_t const startClock = UTIL_getTime();
1776 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
1777 size_t dictSize = 0;
1778 U32 oldTestLog = 0;
1779 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
1780
1781 /* allocations */
1782 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
1783 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
1784 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
1785 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
1786 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
1787 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
1788 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
1789 "Not enough memory, fuzzer tests cancelled");
1790
1791 /* Create initial samples */
1792 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
1793 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
1794 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
1795 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
1796 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
1797 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
1798 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
1799
1800 /* catch up testNb */
1801 for (testNb=1; testNb < startTest; testNb++)
1802 FUZ_rand(&coreSeed);
1803
1804 /* test loop */
1805 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
1806 U32 lseed;
1807 const BYTE* srcBuffer;
1808 size_t totalTestSize, totalGenSize, cSize;
1809 XXH64_state_t xxhState;
1810 U64 crcOrig;
1811 U32 resetAllowed = 1;
1812 size_t maxTestSize;
1813
1814 /* init */
1815 FUZ_rand(&coreSeed);
1816 lseed = coreSeed ^ prime32;
1817 if (nbTests >= testNb) {
1818 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
1819 } else {
1820 DISPLAYUPDATE(2, "\r%6u ", testNb);
1821 }
1822
1823 /* states full reset (deliberately not synchronized) */
1824 /* some issues can only happen when reusing states */
1825 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
1826 ZSTD_freeCStream(zc);
1827 zc = ZSTD_createCStream();
1828 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
1829 resetAllowed=0;
1830 }
1831 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
1832 ZSTD_freeDStream(zd);
1833 zd = ZSTD_createDStream();
1834 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
1835 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
1836 }
1837
1838 /* srcBuffer selection [0-4] */
1839 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
1840 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
1841 else {
1842 buffNb >>= 3;
1843 if (buffNb & 7) {
1844 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
1845 buffNb = tnb[buffNb >> 3];
1846 } else {
1847 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
1848 buffNb = tnb[buffNb >> 3];
1849 } }
1850 srcBuffer = cNoiseBuffer[buffNb];
1851 }
1852
1853 /* compression init */
1854 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
1855 && oldTestLog /* at least one test happened */ && resetAllowed) {
1856 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
1857 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
1858 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1859 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1860 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
1861 }
1862 } else {
1863 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
1864 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
1865 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
1866 (ZSTD_maxCLevel() -
1867 (MAX(testLog, dictLog) / 3)))
1868 + 1;
1869 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
1870 maxTestSize = FUZ_rLogLength(&lseed, testLog);
1871 oldTestLog = testLog;
1872 /* random dictionary selection */
1873 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
1874 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
1875 dict = srcBuffer + dictStart;
1876 }
1877 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1878 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1879 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
1880 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
1881 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
1882 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
1883 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
1884 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
1885 } }
1886
1887 /* multi-segments compression test */
1888 XXH64_reset(&xxhState, 0);
1889 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
1890 U32 n;
1891 for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
1892 /* compress random chunks into randomly sized dst buffers */
1893 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1894 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
1895 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
1896 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1897 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
1898 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
1899 outBuff.size = outBuff.pos + dstBuffSize;
1900
1901 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1902
1903 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
1904 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
1905 totalTestSize += inBuff.pos;
1906 }
1907
1908 /* random flush operation, to mess around */
1909 if ((FUZ_rand(&lseed) & 15) == 0) {
1910 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1911 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1912 outBuff.size = outBuff.pos + adjustedDstSize;
1913 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
1914 } }
1915
1916 /* final frame epilogue */
1917 { size_t remainingToFlush = (size_t)(-1);
1918 while (remainingToFlush) {
1919 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1920 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1921 outBuff.size = outBuff.pos + adjustedDstSize;
1922 remainingToFlush = ZSTD_endStream(zc, &outBuff);
1923 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
1924 } }
1925 crcOrig = XXH64_digest(&xxhState);
1926 cSize = outBuff.pos;
1927 }
1928
1929 /* multi - fragments decompression test */
1930 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
1931 CHECK_Z ( ZSTD_resetDStream(zd) );
1932 } else {
1933 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
1934 }
1935 { size_t decompressionResult = 1;
1936 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1937 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1938 for (totalGenSize = 0 ; decompressionResult ; ) {
1939 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1940 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1941 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
1942 inBuff.size = inBuff.pos + readCSrcSize;
1943 outBuff.size = outBuff.pos + dstBuffSize;
1944 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1945 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
1946 DISPLAY("checksum error : \n");
1947 findDiff(copyBuffer, dstBuffer, totalTestSize);
1948 }
1949 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
1950 ZSTD_getErrorName(decompressionResult) );
1951 }
1952 CHECK (decompressionResult != 0, "frame not fully decoded");
1953 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
1954 (unsigned)outBuff.pos, (unsigned)totalTestSize);
1955 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
1956 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
1957 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
1958 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
1959 } }
1960
1961 /*===== noisy/erroneous src decompression test =====*/
1962
1963 /* add some noise */
1964 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
1965 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
1966 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
1967 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
1968 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
1969 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
1970 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
1971 } }
1972
1973 /* try decompression on noisy data */
1974 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
1975 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
1976 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1977 while (outBuff.pos < dstBufferSize) {
1978 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1979 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1980 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
1981 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
1982 outBuff.size = outBuff.pos + adjustedDstSize;
1983 inBuff.size = inBuff.pos + adjustedCSrcSize;
1984 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1985 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
1986 /* No forward progress possible */
1987 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
1988 } } } }
1989 DISPLAY("\r%u fuzzer tests completed \n", testNb);
1990
1991 _cleanup:
1992 ZSTD_freeCStream(zc);
1993 ZSTD_freeDStream(zd);
1994 ZSTD_freeDStream(zd_noise);
1995 free(cNoiseBuffer[0]);
1996 free(cNoiseBuffer[1]);
1997 free(cNoiseBuffer[2]);
1998 free(cNoiseBuffer[3]);
1999 free(cNoiseBuffer[4]);
2000 free(copyBuffer);
2001 free(cBuffer);
2002 free(dstBuffer);
2003 return result;
2004
2005 _output_error:
2006 result = 1;
2007 goto _cleanup;
2008 }
2009
2010 /** If useOpaqueAPI, sets param in cctxParams.
2011 * Otherwise, sets the param in zc. */
setCCtxParameter(ZSTD_CCtx * zc,ZSTD_CCtx_params * cctxParams,ZSTD_cParameter param,unsigned value,int useOpaqueAPI)2012 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2013 ZSTD_cParameter param, unsigned value,
2014 int useOpaqueAPI)
2015 {
2016 if (useOpaqueAPI) {
2017 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2018 } else {
2019 return ZSTD_CCtx_setParameter(zc, param, value);
2020 }
2021 }
2022
2023 /* Tests for ZSTD_compress_generic() API */
fuzzerTests_newAPI(U32 seed,int nbTests,int startTest,double compressibility,int bigTests)2024 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2025 double compressibility, int bigTests)
2026 {
2027 U32 const maxSrcLog = bigTests ? 24 : 22;
2028 static const U32 maxSampleLog = 19;
2029 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2030 BYTE* cNoiseBuffer[5];
2031 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2032 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2033 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2034 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2035 size_t const dstBufferSize = srcBufferSize;
2036 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2037 U32 result = 0;
2038 int testNb = 0;
2039 U32 coreSeed = seed;
2040 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2041 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2042 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2043 UTIL_time_t const startClock = UTIL_getTime();
2044 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2045 size_t dictSize = 0;
2046 U32 oldTestLog = 0;
2047 U32 windowLogMalus = 0; /* can survive between 2 loops */
2048 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2049 U32 const nbThreadsMax = bigTests ? 4 : 2;
2050 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2051
2052 /* allocations */
2053 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2054 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2055 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2056 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2057 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2058 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2059 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2060 "Not enough memory, fuzzer tests cancelled");
2061
2062 /* Create initial samples */
2063 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2064 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2065 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2066 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2067 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2068 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2069 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2070
2071 /* catch up testNb */
2072 for (testNb=1; testNb < startTest; testNb++)
2073 FUZ_rand(&coreSeed);
2074
2075 /* test loop */
2076 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2077 U32 lseed;
2078 int opaqueAPI;
2079 const BYTE* srcBuffer;
2080 size_t totalTestSize, totalGenSize, cSize;
2081 XXH64_state_t xxhState;
2082 U64 crcOrig;
2083 U32 resetAllowed = 1;
2084 size_t maxTestSize;
2085 ZSTD_parameters savedParams;
2086 int isRefPrefix = 0;
2087 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2088
2089 /* init */
2090 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2091 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2092 FUZ_rand(&coreSeed);
2093 lseed = coreSeed ^ prime32;
2094 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2095 opaqueAPI = FUZ_rand(&lseed) & 1;
2096
2097 /* states full reset (deliberately not synchronized) */
2098 /* some issues can only happen when reusing states */
2099 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2100 DISPLAYLEVEL(5, "Creating new context \n");
2101 ZSTD_freeCCtx(zc);
2102 zc = ZSTD_createCCtx();
2103 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2104 resetAllowed = 0;
2105 }
2106 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2107 ZSTD_freeDStream(zd);
2108 zd = ZSTD_createDStream();
2109 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2110 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2111 }
2112
2113 /* srcBuffer selection [0-4] */
2114 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2115 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2116 else {
2117 buffNb >>= 3;
2118 if (buffNb & 7) {
2119 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2120 buffNb = tnb[buffNb >> 3];
2121 } else {
2122 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2123 buffNb = tnb[buffNb >> 3];
2124 } }
2125 srcBuffer = cNoiseBuffer[buffNb];
2126 }
2127
2128 /* compression init */
2129 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2130 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2131 && oldTestLog /* at least one test happened */
2132 && resetAllowed) {
2133 /* just set a compression level */
2134 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2135 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2136 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2137 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2138 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2139 }
2140 } else {
2141 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2142 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2143 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2144 (ZSTD_maxCLevel() -
2145 (MAX(testLog, dictLog) / 2))) +
2146 1;
2147 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2148 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2149 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2150 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2151 oldTestLog = testLog;
2152 /* random dictionary selection */
2153 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2154 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2155 dict = srcBuffer + dictStart;
2156 if (!dictSize) dict=NULL;
2157 }
2158 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2159 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2160 const U32 windowLogMax = bigTests ? 24 : 20;
2161 const U32 searchLogMax = bigTests ? 15 : 13;
2162 if (dictSize)
2163 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2164
2165 /* mess with compression parameters */
2166 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2167 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2168 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2169 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2170 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2171 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2172 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2173 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2174 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2175
2176 if (FUZ_rand(&lseed) & 1) {
2177 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2178 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2179 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2180 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2181 }
2182 if (FUZ_rand(&lseed) & 1) {
2183 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2184 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2185 }
2186 if (FUZ_rand(&lseed) & 1) {
2187 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2188 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2189 }
2190 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2191 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2192 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2193
2194 /* mess with long distance matching parameters */
2195 if (bigTests) {
2196 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
2197 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2198 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2199 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2200 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2201 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2202 }
2203
2204 /* mess with frame parameters */
2205 if (FUZ_rand(&lseed) & 1) {
2206 int const checksumFlag = FUZ_rand(&lseed) & 1;
2207 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2208 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2209 }
2210 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2211 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2212 if (FUZ_rand(&lseed) & 1) {
2213 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2214 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2215 } else {
2216 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2217 }
2218
2219 /* multi-threading parameters. Only adjust occasionally for small tests. */
2220 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2221 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2222 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2223 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2224 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2225 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2226 if (nbThreads > 1) {
2227 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2228 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2229 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2230 }
2231 }
2232 /* Enable rsyncable mode 1 in 4 times. */
2233 {
2234 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2235 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2236 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
2237 }
2238
2239 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
2240 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
2241
2242 /* Apply parameters */
2243 if (opaqueAPI) {
2244 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
2245 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
2246 }
2247
2248 if (FUZ_rand(&lseed) & 1) {
2249 if (FUZ_rand(&lseed) & 1) {
2250 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2251 } else {
2252 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
2253 }
2254 } else {
2255 isRefPrefix = 1;
2256 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2257 }
2258 } }
2259
2260 CHECK_Z(getCCtxParams(zc, &savedParams));
2261
2262 /* multi-segments compression test */
2263 { int iter;
2264 int const startSeed = lseed;
2265 XXH64_hash_t compressedCrcs[2];
2266 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
2267 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2268 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
2269 int nbWorkers;
2270
2271 XXH64_reset(&xxhState, 0);
2272
2273 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2274 if (isRefPrefix) {
2275 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
2276 /* Need to reload the prefix because it gets dropped after one compression */
2277 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2278 }
2279
2280 /* Adjust number of workers occassionally - result must be deterministic independent of nbWorkers */
2281 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
2282 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
2283 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
2284 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
2285 }
2286
2287 if (singlePass) {
2288 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
2289 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
2290 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
2291 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2292 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
2293 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
2294 totalTestSize = maxTestSize;
2295 } else {
2296 outBuff.size = 0;
2297 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
2298 /* compress random chunks into randomly sized dst buffers */
2299 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2300 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2301 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2302 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
2303 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2304 int forwardProgress;
2305 do {
2306 size_t const ipos = inBuff.pos;
2307 size_t const opos = outBuff.pos;
2308 size_t ret;
2309 if (outBuff.pos == outBuff.size) {
2310 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2311 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2312 outBuff.size = outBuff.pos + dstBuffSize;
2313 }
2314 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
2315 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
2316 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
2317
2318 /* We've completed the flush */
2319 if (flush == ZSTD_e_flush && ret == 0)
2320 break;
2321
2322 /* Ensure maximal forward progress for determinism */
2323 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
2324 } while (forwardProgress);
2325 assert(inBuff.pos == inBuff.size);
2326
2327 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2328 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2329 totalTestSize += inBuff.pos;
2330 }
2331
2332 /* final frame epilogue */
2333 { size_t remainingToFlush = 1;
2334 while (remainingToFlush) {
2335 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
2336 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2337 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2338 outBuff.size = outBuff.pos + adjustedDstSize;
2339 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
2340 /* ZSTD_e_end guarantees maximal forward progress */
2341 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
2342 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
2343 CHECK( ZSTD_isError(remainingToFlush),
2344 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
2345 ZSTD_getErrorName(remainingToFlush) );
2346 } }
2347 crcOrig = XXH64_digest(&xxhState);
2348 }
2349 cSize = outBuff.pos;
2350 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
2351 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
2352 }
2353 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
2354 }
2355
2356 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
2357
2358 /* multi - fragments decompression test */
2359 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2360 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", dict);
2361 CHECK_Z( ZSTD_resetDStream(zd) );
2362 } else {
2363 if (dictSize)
2364 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
2365 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2366 }
2367 { size_t decompressionResult = 1;
2368 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2369 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2370 for (totalGenSize = 0 ; decompressionResult ; ) {
2371 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2372 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2373 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2374 inBuff.size = inBuff.pos + readCSrcSize;
2375 outBuff.size = outBuff.pos + dstBuffSize;
2376 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
2377 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
2378 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2379 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
2380 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2381 if (ZSTD_isError(decompressionResult)) {
2382 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
2383 findDiff(copyBuffer, dstBuffer, totalTestSize);
2384 }
2385 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
2386 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
2387 }
2388 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
2389 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
2390 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2391 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2392 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2393 } }
2394
2395 /*===== noisy/erroneous src decompression test =====*/
2396
2397 /* add some noise */
2398 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2399 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2400 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2401 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2402 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2403 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2404 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2405 } }
2406
2407 /* try decompression on noisy data */
2408 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2409 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2410 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2411 while (outBuff.pos < dstBufferSize) {
2412 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2413 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2414 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2415 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2416 outBuff.size = outBuff.pos + adjustedDstSize;
2417 inBuff.size = inBuff.pos + adjustedCSrcSize;
2418 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2419 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2420 /* Good so far, but no more progress possible */
2421 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2422 } } } }
2423 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
2424
2425 _cleanup:
2426 ZSTD_freeCCtx(zc);
2427 ZSTD_freeDStream(zd);
2428 ZSTD_freeDStream(zd_noise);
2429 ZSTD_freeCCtxParams(cctxParams);
2430 free(cNoiseBuffer[0]);
2431 free(cNoiseBuffer[1]);
2432 free(cNoiseBuffer[2]);
2433 free(cNoiseBuffer[3]);
2434 free(cNoiseBuffer[4]);
2435 free(copyBuffer);
2436 free(cBuffer);
2437 free(dstBuffer);
2438 return result;
2439
2440 _output_error:
2441 result = 1;
2442 goto _cleanup;
2443 }
2444
2445 /*-*******************************************************
2446 * Command line
2447 *********************************************************/
FUZ_usage(const char * programName)2448 static int FUZ_usage(const char* programName)
2449 {
2450 DISPLAY( "Usage :\n");
2451 DISPLAY( " %s [args]\n", programName);
2452 DISPLAY( "\n");
2453 DISPLAY( "Arguments :\n");
2454 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
2455 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
2456 DISPLAY( " -s# : Select seed (default:prompt user)\n");
2457 DISPLAY( " -t# : Select starting test number (default:0)\n");
2458 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
2459 DISPLAY( " -v : verbose\n");
2460 DISPLAY( " -p : pause at the end\n");
2461 DISPLAY( " -h : display help and exit\n");
2462 return 0;
2463 }
2464
2465 typedef enum { simple_api, advanced_api } e_api;
2466
main(int argc,const char ** argv)2467 int main(int argc, const char** argv)
2468 {
2469 U32 seed = 0;
2470 int seedset = 0;
2471 int nbTests = nbTestsDefault;
2472 int testNb = 0;
2473 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
2474 int result = 0;
2475 int mainPause = 0;
2476 int bigTests = (sizeof(size_t) == 8);
2477 e_api selected_api = simple_api;
2478 const char* const programName = argv[0];
2479 int argNb;
2480
2481 /* Check command line */
2482 for(argNb=1; argNb<argc; argNb++) {
2483 const char* argument = argv[argNb];
2484 assert(argument != NULL);
2485
2486 /* Parsing commands. Aggregated commands are allowed */
2487 if (argument[0]=='-') {
2488
2489 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
2490 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
2491
2492 argument++;
2493 while (*argument!=0) {
2494 switch(*argument)
2495 {
2496 case 'h':
2497 return FUZ_usage(programName);
2498
2499 case 'v':
2500 argument++;
2501 g_displayLevel++;
2502 break;
2503
2504 case 'q':
2505 argument++;
2506 g_displayLevel--;
2507 break;
2508
2509 case 'p': /* pause at the end */
2510 argument++;
2511 mainPause = 1;
2512 break;
2513
2514 case 'i': /* limit tests by nb of iterations (default) */
2515 argument++;
2516 nbTests=0; g_clockTime=0;
2517 while ((*argument>='0') && (*argument<='9')) {
2518 nbTests *= 10;
2519 nbTests += *argument - '0';
2520 argument++;
2521 }
2522 break;
2523
2524 case 'T': /* limit tests by time */
2525 argument++;
2526 nbTests=0; g_clockTime=0;
2527 while ((*argument>='0') && (*argument<='9')) {
2528 g_clockTime *= 10;
2529 g_clockTime += *argument - '0';
2530 argument++;
2531 }
2532 if (*argument=='m') { /* -T1m == -T60 */
2533 g_clockTime *=60, argument++;
2534 if (*argument=='n') argument++; /* -T1mn == -T60 */
2535 } else if (*argument=='s') argument++; /* -T10s == -T10 */
2536 g_clockTime *= SEC_TO_MICRO;
2537 break;
2538
2539 case 's': /* manually select seed */
2540 argument++;
2541 seedset=1;
2542 seed=0;
2543 while ((*argument>='0') && (*argument<='9')) {
2544 seed *= 10;
2545 seed += *argument - '0';
2546 argument++;
2547 }
2548 break;
2549
2550 case 't': /* select starting test number */
2551 argument++;
2552 testNb=0;
2553 while ((*argument>='0') && (*argument<='9')) {
2554 testNb *= 10;
2555 testNb += *argument - '0';
2556 argument++;
2557 }
2558 break;
2559
2560 case 'P': /* compressibility % */
2561 argument++;
2562 proba=0;
2563 while ((*argument>='0') && (*argument<='9')) {
2564 proba *= 10;
2565 proba += *argument - '0';
2566 argument++;
2567 }
2568 if (proba<0) proba=0;
2569 if (proba>100) proba=100;
2570 break;
2571
2572 default:
2573 return FUZ_usage(programName);
2574 }
2575 } } } /* for(argNb=1; argNb<argc; argNb++) */
2576
2577 /* Get Seed */
2578 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
2579
2580 if (!seedset) {
2581 time_t const t = time(NULL);
2582 U32 const h = XXH32(&t, sizeof(t), 1);
2583 seed = h % 10000;
2584 }
2585
2586 DISPLAY("Seed = %u\n", (unsigned)seed);
2587 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
2588
2589 if (nbTests<=0) nbTests=1;
2590
2591 if (testNb==0) {
2592 result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
2593 }
2594
2595 if (!result) {
2596 switch(selected_api)
2597 {
2598 case simple_api :
2599 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2600 break;
2601 case advanced_api :
2602 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2603 break;
2604 default :
2605 assert(0); /* impossible */
2606 }
2607 }
2608
2609 if (mainPause) {
2610 int unused;
2611 DISPLAY("Press Enter \n");
2612 unused = getchar();
2613 (void)unused;
2614 }
2615 return result;
2616 }
2617