1 /*
2  * Copyright (c) Yann Collet, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under both the BSD-style license (found in the
6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7  * in the COPYING file in the root directory of this source tree).
8  * You may select, at your option, one of the above-listed licenses.
9  */
10 
11 
12 /*-************************************
13  *  Compiler specific
14  **************************************/
15 #ifdef _MSC_VER    /* Visual Studio */
16 #  define _CRT_SECURE_NO_WARNINGS   /* fgets */
17 #  pragma warning(disable : 4127)   /* disable: C4127: conditional expression is constant */
18 #  pragma warning(disable : 4146)   /* disable: C4146: minus unsigned expression */
19 #endif
20 
21 
22 /*-************************************
23  *  Includes
24  **************************************/
25 #include <stdlib.h>       /* free */
26 #include <stdio.h>        /* fgets, sscanf */
27 #include <string.h>       /* strcmp */
28 #include <assert.h>       /* assert */
29 #include "timefn.h"       /* UTIL_time_t, UTIL_getTime */
30 #include "mem.h"
31 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
32 #define ZSTD_STATIC_LINKING_ONLY  /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
33 #include "zstd.h"         /* ZSTD_compressBound */
34 #include "zstd_errors.h"  /* ZSTD_error_srcSize_wrong */
35 #include "zdict.h"        /* ZDICT_trainFromBuffer */
36 #include "datagen.h"      /* RDG_genBuffer */
37 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
38 #include "xxhash.h"       /* XXH64_* */
39 #include "seqgen.h"
40 #include "util.h"
41 #include "timefn.h"       /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
42 
43 
44 /*-************************************
45  *  Constants
46  **************************************/
47 #define KB *(1U<<10)
48 #define MB *(1U<<20)
49 #define GB *(1U<<30)
50 
51 static const int nbTestsDefault = 10000;
52 static const U32 g_cLevelMax_smallTests = 10;
53 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
54 #define FUZ_COMPRESSIBILITY_DEFAULT 50
55 static const U32 prime32 = 2654435761U;
56 
57 
58 /*-************************************
59  *  Display Macros
60  **************************************/
61 #define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
62 #define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) {                     \
63                                   DISPLAY(__VA_ARGS__);                    \
64                                   if (g_displayLevel>=4) fflush(stderr); }
65 static U32 g_displayLevel = 2;
66 
67 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
68 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
69 
70 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
71             if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72             { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73             if (g_displayLevel>=4) fflush(stderr); } }
74 
75 static U64 g_clockTime = 0;
76 
77 
78 /*-*******************************************************
79  *  Check macros
80  *********************************************************/
81 #undef MIN
82 #undef MAX
83 #define MIN(a,b) ((a)<(b)?(a):(b))
84 #define MAX(a,b) ((a)>(b)?(a):(b))
85 /*! FUZ_rand() :
86     @return : a 27 bits random value, from a 32-bits `seed`.
87     `seed` is also modified */
88 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * seedPtr)89 static U32 FUZ_rand(U32* seedPtr)
90 {
91     static const U32 prime2 = 2246822519U;
92     U32 rand32 = *seedPtr;
93     rand32 *= prime32;
94     rand32 += prime2;
95     rand32  = FUZ_rotl32(rand32, 13);
96     *seedPtr = rand32;
97     return rand32 >> 5;
98 }
99 
100 #define CHECK(cond, ...) {                                   \
101     if (cond) {                                              \
102         DISPLAY("Error => ");                                \
103         DISPLAY(__VA_ARGS__);                                \
104         DISPLAY(" (seed %u, test nb %u, line %u) \n",        \
105                 (unsigned)seed, testNb, __LINE__);           \
106         goto _output_error;                                  \
107 }   }
108 
109 #define CHECK_Z(f) {                                         \
110     size_t const err = f;                                    \
111     CHECK(ZSTD_isError(err), "%s : %s ",                     \
112           #f, ZSTD_getErrorName(err));                       \
113 }
114 
115 #define CHECK_RET(ret, cond, ...) {                          \
116     if (cond) {                                              \
117         DISPLAY("Error %llu => ", (unsigned long long)ret);  \
118         DISPLAY(__VA_ARGS__);                                \
119         DISPLAY(" (line %u)\n", __LINE__);                   \
120         return ret;                                          \
121 }   }
122 
123 #define CHECK_RET_Z(f) {                                     \
124     size_t const err = f;                                    \
125     CHECK_RET(err, ZSTD_isError(err), "%s : %s ",            \
126           #f, ZSTD_getErrorName(err));                       \
127 }
128 
129 
130 /*======================================================
131  *   Basic Unit tests
132  *======================================================*/
133 
134 typedef struct {
135     void* start;
136     size_t size;
137     size_t filled;
138 } buffer_t;
139 
140 static const buffer_t kBuffNull = { NULL, 0 , 0 };
141 
FUZ_freeDictionary(buffer_t dict)142 static void FUZ_freeDictionary(buffer_t dict)
143 {
144     free(dict.start);
145 }
146 
FUZ_createDictionary(const void * src,size_t srcSize,size_t blockSize,size_t requestedDictSize)147 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
148 {
149     buffer_t dict = kBuffNull;
150     size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
151     size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
152     if (!blockSizes) return kBuffNull;
153     dict.start = malloc(requestedDictSize);
154     if (!dict.start) { free(blockSizes); return kBuffNull; }
155     {   size_t nb;
156         for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
157         blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
158     }
159     {   size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
160         free(blockSizes);
161         if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
162         dict.size = requestedDictSize;
163         dict.filled = dictSize;
164         return dict;
165     }
166 }
167 
168 /* Round trips data and updates xxh with the decompressed data produced */
SEQ_roundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,void * data,size_t size,ZSTD_EndDirective endOp)169 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
170                             XXH64_state_t* xxh, void* data, size_t size,
171                             ZSTD_EndDirective endOp)
172 {
173     static BYTE compressed[1024];
174     static BYTE uncompressed[1024];
175 
176     ZSTD_inBuffer cin = {data, size, 0};
177     size_t cret;
178 
179     do {
180         ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
181         ZSTD_inBuffer din   = { compressed, 0, 0 };
182         ZSTD_outBuffer dout = { uncompressed, 0, 0 };
183 
184         cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
185         if (ZSTD_isError(cret))
186             return cret;
187 
188         din.size = cout.pos;
189         while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
190             size_t dret;
191 
192             dout.pos = 0;
193             dout.size = sizeof(uncompressed);
194             dret = ZSTD_decompressStream(dctx, &dout, &din);
195             if (ZSTD_isError(dret))
196                 return dret;
197             XXH64_update(xxh, dout.dst, dout.pos);
198             if (dret == 0)
199                 break;
200         }
201     } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
202     return 0;
203 }
204 
205 /* Generates some data and round trips it */
SEQ_generateRoundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,SEQ_stream * seq,SEQ_gen_type type,unsigned value)206 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
207                                     XXH64_state_t* xxh, SEQ_stream* seq,
208                                     SEQ_gen_type type, unsigned value)
209 {
210     static BYTE data[1024];
211     size_t gen;
212 
213     do {
214         SEQ_outBuffer sout = {data, sizeof(data), 0};
215         size_t ret;
216         gen = SEQ_gen(seq, type, value, &sout);
217 
218         ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
219         if (ZSTD_isError(ret))
220             return ret;
221     } while (gen != 0);
222 
223     return 0;
224 }
225 
getCCtxParams(ZSTD_CCtx * zc,ZSTD_parameters * savedParams)226 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
227 {
228     int value;
229     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
230     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
231     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
232     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
233     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
234     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
235     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
236     savedParams->cParams.strategy = value;
237 
238     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
239     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
240     CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
241     savedParams->fParams.noDictIDFlag = !value;
242     return 0;
243 }
244 
badParameters(ZSTD_CCtx * zc,ZSTD_parameters const savedParams)245 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
246 {
247     ZSTD_parameters params;
248     if (ZSTD_isError(getCCtxParams(zc, &params))) return 10;
249     CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
250     CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
251     CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
252     CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
253     CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
254     CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
255 
256     CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
257     CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
258     CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
259     return 0;
260 }
261 
basicUnitTests(U32 seed,double compressibility)262 static int basicUnitTests(U32 seed, double compressibility)
263 {
264     size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
265     void* CNBuffer = malloc(CNBufferSize);
266     size_t const skippableFrameSize = 200 KB;
267     size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
268     void* compressedBuffer = malloc(compressedBufferSize);
269     size_t const decodedBufferSize = CNBufferSize;
270     void* decodedBuffer = malloc(decodedBufferSize);
271     size_t cSize;
272     int testResult = 0;
273     int testNb = 1;
274     U32 coreSeed = 0;  /* this name to conform with CHECK_Z macro display */
275     ZSTD_CStream* zc = ZSTD_createCStream();
276     ZSTD_DStream* zd = ZSTD_createDStream();
277     ZSTD_CCtx* mtctx = ZSTD_createCCtx();
278 
279     ZSTD_inBuffer  inBuff, inBuff2;
280     ZSTD_outBuffer outBuff;
281     buffer_t dictionary = kBuffNull;
282     size_t const dictSize = 128 KB;
283     unsigned dictID = 0;
284 
285     /* Create compressible test buffer */
286     if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
287         DISPLAY("Not enough memory, aborting \n");
288         goto _output_error;
289     }
290     RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
291 
292     CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
293 
294     /* Create dictionary */
295     DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
296     dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
297     if (!dictionary.start) {
298         DISPLAY("Error creating dictionary, aborting \n");
299         goto _output_error;
300     }
301     dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
302 
303     /* Basic compression test */
304     DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
305     CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
306     outBuff.dst = (char*)(compressedBuffer);
307     outBuff.size = compressedBufferSize;
308     outBuff.pos = 0;
309     inBuff.src = CNBuffer;
310     inBuff.size = CNBufferSize;
311     inBuff.pos = 0;
312     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
313     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
314     { size_t const r = ZSTD_endStream(zc, &outBuff);
315       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
316     DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
317 
318     /* generate skippable frame */
319     MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
320     MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
321     cSize = skippableFrameSize + 8;
322 
323     /* Basic compression test using dict */
324     DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
325     CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
326     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
327     CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
328     outBuff.dst = (char*)(compressedBuffer)+cSize;
329     assert(compressedBufferSize > cSize);
330     outBuff.size = compressedBufferSize - cSize;
331     outBuff.pos = 0;
332     inBuff.src = CNBuffer;
333     inBuff.size = CNBufferSize;
334     inBuff.pos = 0;
335     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
336     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
337     { size_t const r = ZSTD_endStream(zc, &outBuff);
338       if (r != 0) goto _output_error; }  /* error, or some data not flushed */
339     cSize += outBuff.pos;
340     DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
341                     (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
342 
343     /* context size functions */
344     DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
345     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
346         size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
347         size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
348         if (ZSTD_isError(cstreamSize)) goto _output_error;
349         if (ZSTD_isError(cdictSize)) goto _output_error;
350         DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
351     }
352 
353     /* context size functions */
354     DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
355     {   ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
356         size_t cstreamSize, cctxSize;
357         CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
358         cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
359         CHECK_Z(cstreamSize);
360         cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
361         CHECK_Z(cctxSize);
362         if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
363         ZSTD_freeCCtxParams(params);
364         DISPLAYLEVEL(3, "OK \n");
365     }
366 
367     DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
368     {   size_t const s = ZSTD_sizeof_CStream(zc);
369         if (ZSTD_isError(s)) goto _output_error;
370         DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
371     }
372 
373     /* Attempt bad compression parameters */
374     DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
375     {   size_t r;
376         ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
377         params.cParams.minMatch = 2;
378         r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
379         if (!ZSTD_isError(r)) goto _output_error;
380         DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
381     }
382 
383     /* skippable frame test */
384     DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
385     CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
386     inBuff.src = compressedBuffer;
387     inBuff.size = cSize;
388     inBuff.pos = 0;
389     outBuff.dst = decodedBuffer;
390     outBuff.size = CNBufferSize;
391     outBuff.pos = 0;
392     {   size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
393         DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
394         if (r != 0) goto _output_error;
395     }
396     if (outBuff.pos != 0) goto _output_error;   /* skippable frame output len is 0 */
397     DISPLAYLEVEL(3, "OK \n");
398 
399     /* Basic decompression test */
400     inBuff2 = inBuff;
401     DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
402     ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
403     CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) );  /* large limit */
404     { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
405       if (remaining != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
406     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
407     if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
408     DISPLAYLEVEL(3, "OK \n");
409 
410     /* Re-use without init */
411     DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
412     outBuff.pos = 0;
413     { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
414       if (remaining != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
415     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
416     if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
417     DISPLAYLEVEL(3, "OK \n");
418 
419     /* check regenerated data is byte exact */
420     DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
421     {   size_t i;
422         for (i=0; i<CNBufferSize; i++) {
423             if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
424     }   }
425     DISPLAYLEVEL(3, "OK \n");
426 
427     /* context size functions */
428     DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
429     {   ZSTD_frameHeader fhi;
430         const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
431         size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
432         if (gfhError!=0) goto _output_error;
433         DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
434         {   size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
435                             /* uses ZSTD_initDStream_usingDict() */
436                            + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
437             if (ZSTD_isError(s)) goto _output_error;
438             DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
439     }   }
440 
441     DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
442     { size_t const s = ZSTD_sizeof_DStream(zd);
443       if (ZSTD_isError(s)) goto _output_error;
444       DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
445     }
446 
447     /* Decompression by small increment */
448     DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
449     {   /* skippable frame */
450         size_t r = 1;
451         ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
452         inBuff.src = compressedBuffer;
453         outBuff.dst = decodedBuffer;
454         inBuff.pos = 0;
455         outBuff.pos = 0;
456         while (r) {   /* skippable frame */
457             size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
458             size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
459             inBuff.size = inBuff.pos + inSize;
460             outBuff.size = outBuff.pos + outSize;
461             r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
462             if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
463             if (ZSTD_isError(r)) goto _output_error;
464         }
465         /* normal frame */
466         ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
467         r=1;
468         while (r) {
469             size_t const inSize = FUZ_rand(&coreSeed) & 15;
470             size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize);   /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
471             inBuff.size = inBuff.pos + inSize;
472             outBuff.size = outBuff.pos + outSize;
473             r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
474             if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
475             if (ZSTD_isError(r)) goto _output_error;
476         }
477     }
478     if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
479     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
480     if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
481     if (inBuff.pos != cSize) goto _output_error;   /* should have read the entire frame */
482     DISPLAYLEVEL(3, "OK \n");
483 
484     /* check regenerated data is byte exact */
485     DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
486     {   size_t i;
487         for (i=0; i<CNBufferSize; i++) {
488             if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
489     }   }
490     DISPLAYLEVEL(3, "OK \n");
491 
492     /* Decompression forward progress */
493     DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
494     {   /* skippable frame */
495         size_t r = 0;
496         int decNb = 0;
497         int const maxDec = 100;
498         inBuff.src = compressedBuffer;
499         inBuff.size = cSize;
500         inBuff.pos = 0;
501 
502         outBuff.dst = decodedBuffer;
503         outBuff.pos = 0;
504         outBuff.size = CNBufferSize-1;   /* 1 byte missing */
505 
506         for (decNb=0; decNb<maxDec; decNb++) {
507             if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
508             r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
509             if (ZSTD_isError(r)) break;
510         }
511         if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
512         if (!ZSTD_isError(r)) goto _output_error;   /* should have triggered no_forward_progress error */
513     }
514     DISPLAYLEVEL(3, "OK \n");
515 
516     DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
517     inBuff.src = NULL;
518     inBuff.size = 0;
519     inBuff.pos = 0;
520     outBuff.dst = NULL;
521     outBuff.size = 0;
522     outBuff.pos = 0;
523     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
524     CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
525     CHECK_Z( ZSTD_endStream(zc, &outBuff) );
526     outBuff.dst = (char*)(compressedBuffer);
527     outBuff.size = compressedBufferSize;
528     outBuff.pos = 0;
529     {   size_t const r = ZSTD_endStream(zc, &outBuff);
530         CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
531     }
532     inBuff.src = outBuff.dst;
533     inBuff.size = outBuff.pos;
534     inBuff.pos = 0;
535     outBuff.dst = NULL;
536     outBuff.size = 0;
537     outBuff.pos = 0;
538     CHECK_Z( ZSTD_initDStream(zd) );
539     {   size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
540         if (ret != 0) goto _output_error;
541     }
542     DISPLAYLEVEL(3, "OK\n");
543     /* _srcSize compression test */
544     DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
545     CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
546     CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
547     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
548     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
549     outBuff.dst = (char*)(compressedBuffer);
550     outBuff.size = compressedBufferSize;
551     outBuff.pos = 0;
552     inBuff.src = CNBuffer;
553     inBuff.size = CNBufferSize;
554     inBuff.pos = 0;
555     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
556     CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
557     {   size_t const r = ZSTD_endStream(zc, &outBuff);
558         CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
559     }
560     {   unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
561         CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
562         CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
563     }
564     DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
565 
566     /* wrong _srcSize compression test */
567     DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
568     CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
569     CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
570     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
571     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
572     outBuff.dst = (char*)(compressedBuffer);
573     outBuff.size = compressedBufferSize;
574     outBuff.pos = 0;
575     inBuff.src = CNBuffer;
576     inBuff.size = CNBufferSize;
577     inBuff.pos = 0;
578     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
579     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
580     { size_t const r = ZSTD_endStream(zc, &outBuff);
581       if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
582       DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
583 
584     /* wrong _srcSize compression test */
585     DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
586     CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
587     CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
588     CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
589     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
590     outBuff.dst = (char*)(compressedBuffer);
591     outBuff.size = compressedBufferSize;
592     outBuff.pos = 0;
593     inBuff.src = CNBuffer;
594     inBuff.size = CNBufferSize;
595     inBuff.pos = 0;
596     {   size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
597         if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
598         DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
599     }
600 
601     DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
602     {   CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
603         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
604         CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
605         outBuff.dst = (char*)compressedBuffer;
606         outBuff.size = compressedBufferSize;
607         outBuff.pos = 0;
608         inBuff.src = CNBuffer;
609         inBuff.size = CNBufferSize;
610         inBuff.pos = 0;
611         {   size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
612             if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error;    /* must fail : wrong srcSize */
613             DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
614     }   }
615 
616     /* Complex context re-use scenario */
617     DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
618     ZSTD_freeCStream(zc);
619     zc = ZSTD_createCStream();
620     if (zc==NULL) goto _output_error;   /* memory allocation issue */
621     /* use 1 */
622     {   size_t const inSize = 513;
623         DISPLAYLEVEL(5, "use1 ");
624         CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
625         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
626         CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
627         inBuff.src = CNBuffer;
628         inBuff.size = inSize;
629         inBuff.pos = 0;
630         outBuff.dst = (char*)(compressedBuffer)+cSize;
631         outBuff.size = ZSTD_compressBound(inSize);
632         outBuff.pos = 0;
633         DISPLAYLEVEL(5, "compress1 ");
634         CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
635         if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
636         DISPLAYLEVEL(5, "end1 ");
637         { size_t const r = ZSTD_endStream(zc, &outBuff);
638             if (r != 0) goto _output_error; }  /* error, or some data not flushed */
639     }
640     /* use 2 */
641     {   size_t const inSize = 1025;   /* will not continue, because tables auto-adjust and are therefore different size */
642         DISPLAYLEVEL(5, "use2 ");
643         CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
644         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
645         CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
646         inBuff.src = CNBuffer;
647         inBuff.size = inSize;
648         inBuff.pos = 0;
649         outBuff.dst = (char*)(compressedBuffer)+cSize;
650         outBuff.size = ZSTD_compressBound(inSize);
651         outBuff.pos = 0;
652         DISPLAYLEVEL(5, "compress2 ");
653         CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
654         if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
655         DISPLAYLEVEL(5, "end2 ");
656         { size_t const r = ZSTD_endStream(zc, &outBuff);
657             if (r != 0) goto _output_error; }  /* error, or some data not flushed */
658     }
659     DISPLAYLEVEL(3, "OK \n");
660 
661     /* Decompression single pass with empty frame */
662     cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
663     CHECK_Z(cSize);
664     DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
665     {   ZSTD_DCtx* dctx = ZSTD_createDCtx();
666         size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
667         CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
668 
669         outBuff.dst = decodedBuffer;
670         outBuff.pos = 0;
671         outBuff.size = CNBufferSize;
672 
673         inBuff.src = compressedBuffer;
674         inBuff.size = cSize;
675         inBuff.pos = 0;
676         {   size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
677             CHECK_Z(r);
678             CHECK(r != 0, "Entire frame must be decompressed");
679             CHECK(outBuff.pos != 0, "Wrong size!");
680             CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
681         }
682         CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
683         ZSTD_freeDCtx(dctx);
684     }
685     DISPLAYLEVEL(3, "OK \n");
686 
687     /* Decompression with ZSTD_d_stableOutBuffer */
688     cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
689     CHECK_Z(cSize);
690     {   ZSTD_DCtx* dctx = ZSTD_createDCtx();
691         size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
692         size_t dctxSize1;
693         CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
694 
695         outBuff.dst = decodedBuffer;
696         outBuff.pos = 0;
697         outBuff.size = CNBufferSize;
698 
699         DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
700         inBuff.src = compressedBuffer;
701         inBuff.size = cSize;
702         inBuff.pos = 0;
703         {   size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
704             CHECK_Z(r);
705             CHECK(r != 0, "Entire frame must be decompressed");
706             CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
707             CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
708         }
709         CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
710         DISPLAYLEVEL(3, "OK \n");
711 
712         DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
713         outBuff.pos = 0;
714         inBuff.pos = 0;
715         inBuff.size = 0;
716         while (inBuff.pos < cSize) {
717             inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
718             CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
719         }
720         CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
721         CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
722         dctxSize1 = ZSTD_sizeof_DCtx(dctx);
723         CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
724         DISPLAYLEVEL(3, "OK \n");
725 
726         DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
727         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
728         CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
729         inBuff.src = compressedBuffer;
730         inBuff.size = cSize;
731         inBuff.pos = 0;
732         outBuff.pos = 0;
733         outBuff.size = CNBufferSize - 1;
734         {   size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
735             CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
736         }
737         DISPLAYLEVEL(3, "OK \n");
738 
739         DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
740         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
741         CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
742         inBuff.src = compressedBuffer;
743         inBuff.size = cSize - 1;
744         inBuff.pos = 0;
745         outBuff.pos = 0;
746         outBuff.size = CNBufferSize;
747         CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
748         ++inBuff.size;
749         outBuff.pos = 0;
750         {   size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
751             CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
752         }
753         DISPLAYLEVEL(3, "OK \n");
754 
755         DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
756         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
757         CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
758         outBuff.pos = 0;
759         inBuff.pos = 0;
760         inBuff.size = 0;
761         while (inBuff.pos < cSize) {
762             inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
763             CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
764         }
765         CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
766         CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
767         CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
768         DISPLAYLEVEL(3, "OK \n");
769 
770         ZSTD_freeDCtx(dctx);
771     }
772 
773     /* Compression with ZSTD_c_stable{In,Out}Buffer */
774     {   ZSTD_CCtx* cctx = ZSTD_createCCtx();
775         ZSTD_inBuffer in;
776         ZSTD_outBuffer out;
777         size_t cctxSize1;
778         size_t cctxSize2;
779         in.src = CNBuffer;
780         in.size = CNBufferSize;
781         out.dst = compressedBuffer;
782         out.size = compressedBufferSize;
783         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
784         DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
785         CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
786         CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
787         CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
788         CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
789         {   ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
790             in.pos = out.pos = 0;
791             CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
792             CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
793             CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
794             ZSTD_freeCCtx(cctx2);
795         }
796         {   ZSTD_CCtx* cctx3 = ZSTD_createCCtx();
797             ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
798             size_t cSize3;
799             params.fParams.checksumFlag = 1;
800             cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
801             CHECK_Z(cSize3);
802             CHECK(!(cSize == cSize3), "Must be same compressed size");
803             CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx3)), "Must be same CCtx size");
804             ZSTD_freeCCtx(cctx3);
805         }
806         CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
807         CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
808         DISPLAYLEVEL(3, "OK \n");
809 
810         DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
811         {
812             int stableInBuffer;
813             int stableOutBuffer;
814             CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
815             CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
816             CHECK(!(stableInBuffer == 0), "Modified");
817             CHECK(!(stableOutBuffer == 0), "Modified");
818             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
819             CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
820             CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
821             CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
822             CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
823             CHECK(!(stableInBuffer == 1), "Modified");
824             CHECK(!(stableOutBuffer == 1), "Modified");
825         }
826         DISPLAYLEVEL(3, "OK \n");
827 
828         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
829         CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
830         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
831         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
832         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
833         in.pos = out.pos = 0;
834         CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
835         CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
836         DISPLAYLEVEL(3, "OK \n");
837 
838         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
839         {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
840             CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
841         }
842         DISPLAYLEVEL(3, "OK \n");
843 
844         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
845         CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
846         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
847         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
848         in.pos = out.pos = 0;
849         out.size = cSize / 4;
850         for (;;) {
851             size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
852             CHECK_Z(ret);
853             if (ret == 0)
854                 break;
855             out.size = MIN(out.size + cSize / 4, compressedBufferSize);
856         }
857         CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
858         DISPLAYLEVEL(3, "OK \n");
859 
860         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
861         in.pos = out.pos = 0;
862         out.size = cSize / 4;
863         CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
864         in.src = (char const*)in.src + in.pos;
865         in.size -= in.pos;
866         in.pos = 0;
867         {   size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
868             CHECK(!ZSTD_isError(ret), "Must error");
869             CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
870         }
871         DISPLAYLEVEL(3, "OK \n");
872 
873         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++);
874         in.src = CNBuffer;
875         in.size = CNBufferSize;
876         in.pos = 0;
877         out.pos = 0;
878         out.size = compressedBufferSize;
879         CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
880         {   size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
881             CHECK(!ZSTD_isError(ret), "Must error");
882             CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
883         }
884         CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
885         {   size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
886             CHECK(!ZSTD_isError(ret), "Must error");
887             CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
888         }
889         DISPLAYLEVEL(3, "OK \n");
890 
891         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer allocated size : ", testNb++);
892         {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
893             CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
894             CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
895             cctxSize1 = cctxSize;
896         }
897         DISPLAYLEVEL(3, "OK \n");
898 
899         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
900         CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
901         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
902         CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
903         in.pos = out.pos = 0;
904         in.size = MIN(CNBufferSize, 10);
905         CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
906         in.pos = 0;
907         in.size = CNBufferSize - in.size;
908         CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
909         CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
910         DISPLAYLEVEL(3, "OK \n");
911 
912         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
913         in.pos = out.pos = 0;
914         in.size = CNBufferSize;
915         CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
916         in.pos = out.pos = 0;
917         {   size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
918             CHECK(!ZSTD_isError(ret), "Must have errored");
919             CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_dstBuffer_wrong), "Must be this error");
920         }
921         DISPLAYLEVEL(3, "OK \n");
922 
923         DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer allocated size : ", testNb++);
924         {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
925             CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
926             CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
927         }
928         DISPLAYLEVEL(3, "OK \n");
929 
930         ZSTD_freeCCtx(cctx);
931     }
932 
933     /* CDict scenario */
934     DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
935     {   ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
936         size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
937         DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
938         if (ZSTD_isError(initError)) goto _output_error;
939         outBuff.dst = compressedBuffer;
940         outBuff.size = compressedBufferSize;
941         outBuff.pos = 0;
942         inBuff.src = CNBuffer;
943         inBuff.size = CNBufferSize;
944         inBuff.pos = 0;
945         DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
946         CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
947         if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
948         {   size_t const r = ZSTD_endStream(zc, &outBuff);
949             DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
950             if (r != 0) goto _output_error;  /* error, or some data not flushed */
951         }
952         cSize = outBuff.pos;
953         ZSTD_freeCDict(cdict);
954         DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
955     }
956 
957     DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
958     { size_t const s = ZSTD_sizeof_CStream(zc);
959       if (ZSTD_isError(s)) goto _output_error;
960       DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
961     }
962 
963     DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
964     { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
965       if (dID != dictID) goto _output_error;
966       DISPLAYLEVEL(4, "OK (%u) \n", dID);
967     }
968 
969     /* DDict scenario */
970     DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
971     {   ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
972         size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
973         if (ZSTD_isError(initError)) goto _output_error;
974         outBuff.dst = decodedBuffer;
975         outBuff.size = CNBufferSize;
976         outBuff.pos = 0;
977         inBuff.src = compressedBuffer;
978         inBuff.size = cSize;
979         inBuff.pos = 0;
980         { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
981           if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
982         if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
983         if (inBuff.pos != inBuff.size) goto _output_error;   /* should have read the entire frame */
984         ZSTD_freeDDict(ddict);
985         DISPLAYLEVEL(3, "OK \n");
986     }
987 
988     /* Memory restriction */
989     DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
990     ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
991     CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) );  /* too small limit */
992     outBuff.dst = decodedBuffer;
993     outBuff.size = CNBufferSize;
994     outBuff.pos = 0;
995     inBuff.src = compressedBuffer;
996     inBuff.size = cSize;
997     inBuff.pos = 0;
998     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
999       if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
1000       DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1001     ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters);   /* leave zd in good shape for next tests */
1002 
1003     DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1004     {   ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1005         int const maxLevel = 16;   /* first level with zstd_opt */
1006         int level;
1007         assert(maxLevel < ZSTD_maxCLevel());
1008         CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1009         for (level = 1; level <= maxLevel; ++level) {
1010             ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1011             size_t const maxSize = MIN(1 MB, CNBufferSize);
1012             size_t size;
1013             for (size = 512; size <= maxSize; size <<= 1) {
1014                 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1015                 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1016                 ZSTD_parameters savedParams;
1017                 getCCtxParams(cctx, &savedParams);
1018                 outBuff.dst = compressedBuffer;
1019                 outBuff.size = compressedBufferSize;
1020                 outBuff.pos = 0;
1021                 inBuff.src = CNBuffer;
1022                 inBuff.size = size;
1023                 inBuff.pos = 0;
1024                 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1025                 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1026                 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1027                 if (inBuff.pos != inBuff.size) goto _output_error;
1028                 {   ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1029                     ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1030                     CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1031                     if (decIn.pos != decIn.size) goto _output_error;
1032                     if (decOut.pos != size) goto _output_error;
1033                     {   U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1034                         if (crcDec != crcOrig) goto _output_error;
1035                 }   }
1036                 ZSTD_freeCCtx(cctx);
1037             }
1038             ZSTD_freeCDict(cdict);
1039         }
1040         ZSTD_freeDCtx(dctx);
1041     }
1042     DISPLAYLEVEL(3, "OK\n");
1043 
1044     ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1045     CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1046     cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1047     CHECK_Z(cSize);
1048     DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1049     {
1050         ZSTD_DCtx* dctx = ZSTD_createDCtx();
1051         /* We should fail to decompress without a dictionary. */
1052         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1053         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1054             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1055             size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1056             if (!ZSTD_isError(ret)) goto _output_error;
1057         }
1058         /* We should succeed to decompress with the dictionary. */
1059         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1060         CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1061         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1062             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1063             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1064             if (in.pos != in.size) goto _output_error;
1065         }
1066         /* The dictionary should presist across calls. */
1067         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1068             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1069             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1070             if (in.pos != in.size) goto _output_error;
1071         }
1072         /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1073         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1074         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1075             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1076             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1077             if (in.pos != in.size) goto _output_error;
1078         }
1079         /* When we reset the context the dictionary is cleared. */
1080         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1081         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1082             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1083             size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1084             if (!ZSTD_isError(ret)) goto _output_error;
1085         }
1086         ZSTD_freeDCtx(dctx);
1087     }
1088     DISPLAYLEVEL(3, "OK \n");
1089 
1090     DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1091     {
1092         ZSTD_DCtx* dctx = ZSTD_createDCtx();
1093         /* We should succeed to decompress with the dictionary. */
1094         ZSTD_resetDStream(dctx);
1095         CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1096         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1097             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1098             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1099             if (in.pos != in.size) goto _output_error;
1100         }
1101         /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1102         ZSTD_resetDStream(dctx);
1103         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1104             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1105             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1106             if (in.pos != in.size) goto _output_error;
1107         }
1108         /* The dictionary should be cleared by ZSTD_initDStream(). */
1109         CHECK_Z( ZSTD_initDStream(dctx) );
1110         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1111             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1112             size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1113             if (!ZSTD_isError(ret)) goto _output_error;
1114         }
1115         ZSTD_freeDCtx(dctx);
1116     }
1117     DISPLAYLEVEL(3, "OK \n");
1118 
1119     DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1120     {
1121         ZSTD_DCtx* dctx = ZSTD_createDCtx();
1122         ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1123         /* We should succeed to decompress with the ddict. */
1124         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1125         CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1126         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1127             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1128             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1129             if (in.pos != in.size) goto _output_error;
1130         }
1131         /* The ddict should presist across calls. */
1132         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1133             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1134             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1135             if (in.pos != in.size) goto _output_error;
1136         }
1137         /* When we reset the context the ddict is cleared. */
1138         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1139         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1140             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1141             size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1142             if (!ZSTD_isError(ret)) goto _output_error;
1143         }
1144         ZSTD_freeDCtx(dctx);
1145         ZSTD_freeDDict(ddict);
1146     }
1147     DISPLAYLEVEL(3, "OK \n");
1148 
1149     DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1150     {
1151         ZSTD_DCtx* dctx = ZSTD_createDCtx();
1152         /* We should succeed to decompress with the prefix. */
1153         ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1154         CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1155         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1156             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1157             if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1158             if (in.pos != in.size) goto _output_error;
1159         }
1160         /* The prefix should be cleared after the first compression. */
1161         {   ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1162             ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1163             size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1164             if (!ZSTD_isError(ret)) goto _output_error;
1165         }
1166         ZSTD_freeDCtx(dctx);
1167     }
1168     DISPLAYLEVEL(3, "OK \n");
1169 
1170     DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1171     {
1172         ZSTD_DCtx* dctx = ZSTD_createDCtx();
1173         ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1174         size_t ret;
1175         /* We should succeed to decompress with the dictionary. */
1176         CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1177         CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1178         /* The dictionary should presist across calls. */
1179         CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1180         /* We should succeed to decompress with the ddict. */
1181         CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1182         CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1183         /* The ddict should presist across calls. */
1184         CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1185         /* When we reset the context the ddict is cleared. */
1186         CHECK_Z( ZSTD_initDStream(dctx) );
1187         ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1188         if (!ZSTD_isError(ret)) goto _output_error;
1189         ZSTD_freeDCtx(dctx);
1190         ZSTD_freeDDict(ddict);
1191     }
1192     DISPLAYLEVEL(3, "OK \n");
1193 
1194     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1195     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1196         ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1197         ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1198         size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1199         if (ZSTD_isError(initError)) goto _output_error;
1200         outBuff.dst = compressedBuffer;
1201         outBuff.size = compressedBufferSize;
1202         outBuff.pos = 0;
1203         inBuff.src = CNBuffer;
1204         inBuff.size = CNBufferSize;
1205         inBuff.pos = 0;
1206         CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1207         if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
1208         { size_t const r = ZSTD_endStream(zc, &outBuff);
1209           if (r != 0) goto _output_error; }  /* error, or some data not flushed */
1210         cSize = outBuff.pos;
1211         ZSTD_freeCDict(cdict);
1212         DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1213     }
1214 
1215     DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1216     {   U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1217         if (did != 0) goto _output_error;
1218     }
1219     DISPLAYLEVEL(3, "OK (not detected) \n");
1220 
1221     DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1222     {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1223         if (!ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
1224         DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1225     }
1226 
1227     DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1228     CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1229     outBuff.dst = compressedBuffer;
1230     outBuff.size = compressedBufferSize;
1231     outBuff.pos = 0;
1232     inBuff.src = CNBuffer;
1233     inBuff.size = CNBufferSize;
1234     inBuff.pos = 0;
1235     CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1236     if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
1237     cSize = outBuff.pos;
1238     DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1239 
1240     DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1241     CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1242     outBuff.dst = decodedBuffer;
1243     outBuff.size = CNBufferSize;
1244     outBuff.pos = 0;
1245     inBuff.src = compressedBuffer;
1246     inBuff.size = cSize;
1247     inBuff.pos = 0;
1248     CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1249     if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
1250     if (outBuff.pos != CNBufferSize) goto _output_error;  /* must regenerate whole input */
1251     DISPLAYLEVEL(3, "OK \n");
1252 
1253     DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1254     {   size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1255         if (!ZSTD_isError(r)) goto _output_error;  /* must fail : dictionary not used */
1256         DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1257     }
1258 
1259     DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1260     outBuff.dst = compressedBuffer;
1261     outBuff.size = compressedBufferSize;
1262     outBuff.pos = 0;
1263     inBuff.src = CNBuffer;
1264     inBuff.size = CNBufferSize;
1265     inBuff.pos = 0;
1266     CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1267     if (inBuff.pos != inBuff.size) goto _output_error;  /* entire input should be consumed */
1268     cSize = outBuff.pos;
1269     DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1270 
1271     DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1272     CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1273     DISPLAYLEVEL(3, "OK \n");
1274 
1275     /* Empty srcSize */
1276     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1277     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1278         params.fParams.contentSizeFlag = 1;
1279         CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1280     } /* cstream advanced shall write content size = 0 */
1281     outBuff.dst = compressedBuffer;
1282     outBuff.size = compressedBufferSize;
1283     outBuff.pos = 0;
1284     inBuff.src = CNBuffer;
1285     inBuff.size = 0;
1286     inBuff.pos = 0;
1287     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1288     if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1289     cSize = outBuff.pos;
1290     if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1291     DISPLAYLEVEL(3, "OK \n");
1292 
1293     DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1294     {   ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1295         params.fParams.contentSizeFlag = 1;
1296         CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1297     } /* cstream advanced shall write content size = 0 */
1298     inBuff.src = CNBuffer;
1299     inBuff.size = 0;
1300     inBuff.pos = 0;
1301     outBuff.dst = compressedBuffer;
1302     outBuff.size = compressedBufferSize;
1303     outBuff.pos = 0;
1304     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1305     if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1306     cSize = outBuff.pos;
1307     if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1308 
1309     CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1310     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1311     outBuff.dst = compressedBuffer;
1312     outBuff.size = compressedBufferSize;
1313     outBuff.pos = 0;
1314     inBuff.src = CNBuffer;
1315     inBuff.size = 0;
1316     inBuff.pos = 0;
1317     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1318     if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1319     cSize = outBuff.pos;
1320     if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1321     DISPLAYLEVEL(3, "OK \n");
1322 
1323     /* Basic multithreading compression test */
1324     DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1325     {   int jobSize;
1326         CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1327         CHECK(jobSize != 0, "job size non-zero");
1328         CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1329         CHECK(jobSize != 0, "job size non-zero");
1330     }
1331     outBuff.dst = compressedBuffer;
1332     outBuff.size = compressedBufferSize;
1333     outBuff.pos = 0;
1334     inBuff.src = CNBuffer;
1335     inBuff.size = CNBufferSize;
1336     inBuff.pos = 0;
1337     {   size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1338         if (compressResult != 0) goto _output_error;  /* compression must be completed in a single round */
1339     }
1340     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
1341     {   size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1342         if (compressedSize != outBuff.pos) goto _output_error;  /* must be a full valid frame */
1343     }
1344     DISPLAYLEVEL(3, "OK \n");
1345 
1346     /* Complex multithreading + dictionary test */
1347     {   U32 const nbWorkers = 2;
1348         size_t const jobSize = 4 * 1 MB;
1349         size_t const srcSize = jobSize * nbWorkers;  /* we want each job to have predictable size */
1350         size_t const segLength = 2 KB;
1351         size_t const offset = 600 KB;   /* must be larger than window defined in cdict */
1352         size_t const start = jobSize + (offset-1);
1353         const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1354         BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1355         DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1356         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1357         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1358         CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1359         assert(start > offset);
1360         assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1361         memcpy(dst, srcToCopy, segLength);   /* create a long repetition at long distance for job 2 */
1362         outBuff.dst = compressedBuffer;
1363         outBuff.size = compressedBufferSize;
1364         outBuff.pos = 0;
1365         inBuff.src = CNBuffer;
1366         inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1367         inBuff.pos = 0;
1368     }
1369     {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled);   /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1370         ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1371         DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1372         CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1373         CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1374         CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );  /* do not keep a reference to cdict, as its lifetime ends */
1375         ZSTD_freeCDict(cdict);
1376     }
1377     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
1378     cSize = outBuff.pos;
1379     DISPLAYLEVEL(3, "OK \n");
1380 
1381     DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1382     {   ZSTD_DStream* const dstream = ZSTD_createDCtx();
1383         ZSTD_frameHeader zfh;
1384         ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1385         DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1386         outBuff.dst = decodedBuffer;
1387         outBuff.size = CNBufferSize;
1388         outBuff.pos = 0;
1389         inBuff.src = compressedBuffer;
1390         inBuff.pos = 0;
1391         CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1392         inBuff.size = 1;  /* avoid shortcut to single-pass mode */
1393         CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1394         inBuff.size = cSize;
1395         CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1396         if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
1397         ZSTD_freeDStream(dstream);
1398     }
1399     DISPLAYLEVEL(3, "OK \n");
1400 
1401     DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1402     {   unsigned const kMaxWindowLog = 24;
1403         unsigned value;
1404         ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1405         ZSTD_CDict* cdict;
1406         ZSTD_DDict* ddict;
1407         SEQ_stream seq = SEQ_initStream(0x87654321);
1408         SEQ_gen_type type;
1409         XXH64_state_t xxh;
1410 
1411         XXH64_reset(&xxh, 0);
1412         cParams.windowLog = kMaxWindowLog;
1413         cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1414         ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1415 
1416         if (!cdict || !ddict) goto _output_error;
1417 
1418         ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1419         ZSTD_resetDStream(zd);
1420         CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1421         CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1422         CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1423         /* Test all values < 300 */
1424         for (value = 0; value < 300; ++value) {
1425             for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1426                 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1427             }
1428         }
1429         /* Test values 2^8 to 2^17 */
1430         for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1431             for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1432                 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1433                 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1434             }
1435         }
1436         /* Test offset values up to the max window log */
1437         for (value = 8; value <= kMaxWindowLog; ++value) {
1438             CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1439         }
1440 
1441         CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1442         CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1443 
1444         ZSTD_freeCDict(cdict);
1445         ZSTD_freeDDict(ddict);
1446     }
1447     DISPLAYLEVEL(3, "OK \n");
1448 
1449     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1450     {   int level;
1451         CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1452         CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1453         CHECK(level != 11, "Compression level does not match");
1454         CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1455         CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1456         CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1457         CHECK(level != 11, "Compression level does not match");
1458     }
1459     DISPLAYLEVEL(3, "OK \n");
1460 
1461     DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1462     {   ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1463         CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1464         CHECK(badParameters(zc, params), "Compression parameters do not match");
1465         CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1466         CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1467         CHECK(badParameters(zc, params), "Compression parameters do not match");
1468     }
1469     DISPLAYLEVEL(3, "OK \n");
1470 
1471     DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1472     ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1473     CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1474     {   int srcSizeHint;
1475         CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1476         CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1477     }
1478     CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1479     DISPLAYLEVEL(3, "OK \n");
1480 
1481     /* Overlen overwriting window data bug */
1482     DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1483     {   /* This test has a window size of 1024 bytes and consists of 3 blocks:
1484             1. 'a' repeated 517 times
1485             2. 'b' repeated 516 times
1486             3. a compressed block with no literals and 3 sequence commands:
1487                 litlength = 0, offset = 24, match length = 24
1488                 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1489                 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1490 
1491         const char* testCase =
1492             "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1493             "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1494             "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1495         ZSTD_DStream* const zds = ZSTD_createDStream();
1496         if (zds==NULL) goto _output_error;
1497 
1498         CHECK_Z( ZSTD_initDStream(zds) );
1499         inBuff.src = testCase;
1500         inBuff.size = 47;
1501         inBuff.pos = 0;
1502         outBuff.dst = decodedBuffer;
1503         outBuff.size = CNBufferSize;
1504         outBuff.pos = 0;
1505 
1506         while (inBuff.pos < inBuff.size) {
1507             CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1508         }
1509 
1510         ZSTD_freeDStream(zds);
1511     }
1512     DISPLAYLEVEL(3, "OK \n");
1513 
1514     /* Small Sequence Section bug */
1515     DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1516     {   /* This test consists of 3 blocks. Each block has one sequence.
1517             The sequence has literal length of 10, match length of 10 and offset of 10.
1518             The sequence value and compression mode for the blocks are following:
1519             The order of values are ll, ml, of.
1520               - First block  : (10, 7, 13) (rle, rle, rle)
1521                  - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1522               - Second block : (10, 7, 1) (repeat, repeat, rle)
1523                  - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1524               - Third block  : (10, 7, 1) (repeat, repeat, repeat)
1525                  - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1526 
1527         unsigned char compressed[] = {
1528             0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1529             0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1530             0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1531             0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1532             0x40, 0x0a, 0xa4
1533         };
1534         unsigned int compressedSize = 51;
1535         unsigned char decompressed[] = {
1536             0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1537             0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1538             0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1539             0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1540             0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1541         };
1542         unsigned int decompressedSize = 60;
1543 
1544         ZSTD_DStream* const zds = ZSTD_createDStream();
1545         if (zds==NULL) goto _output_error;
1546 
1547         CHECK_Z( ZSTD_initDStream(zds) );
1548         inBuff.src = compressed;
1549         inBuff.size = compressedSize;
1550         inBuff.pos = 0;
1551         outBuff.dst = decodedBuffer;
1552         outBuff.size = CNBufferSize;
1553         outBuff.pos = 0;
1554 
1555         CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1556               "Decompress did not reach the end of frame");
1557         CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1558         CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1559         CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1560               "Decompressed data does not match");
1561 
1562         ZSTD_freeDStream(zds);
1563     }
1564     DISPLAYLEVEL(3, "OK \n");
1565 
1566     DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1567     {   size_t const inputSize = 10000;
1568         size_t const compCapacity = ZSTD_compressBound(inputSize);
1569         BYTE* const input = (BYTE*)malloc(inputSize);
1570         BYTE* const comp = (BYTE*)malloc(compCapacity);
1571         BYTE* const decomp = (BYTE*)malloc(inputSize);
1572 
1573         CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1574 
1575         RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1576         {   size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1577             ZSTD_inBuffer in = { comp, 0, 0 };
1578             ZSTD_outBuffer out = { decomp, 0, 0 };
1579             CHECK_Z(compSize);
1580             CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1581             while (in.size < compSize) {
1582                 in.size = MIN(in.size + 100, compSize);
1583                 while (in.pos < in.size) {
1584                     size_t const outPos = out.pos;
1585                     if (out.pos == out.size) {
1586                         out.size = MIN(out.size + 10, inputSize);
1587                     }
1588                     CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1589                     CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1590                 }
1591             }
1592             CHECK(in.pos != compSize, "Not all input consumed!");
1593             CHECK(out.pos != inputSize, "Not all output produced!");
1594         }
1595         CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1596 
1597         free(input);
1598         free(comp);
1599         free(decomp);
1600     }
1601     DISPLAYLEVEL(3, "OK \n");
1602 
1603     DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1604     {   ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1605             dictionary.start, dictionary.filled,
1606             ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1607             ZSTD_getCParams(3, 0, dictionary.filled),
1608             ZSTD_defaultCMem);
1609         const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1610         const size_t outbufsize = ZSTD_compressBound(inbufsize);
1611         size_t inbufpos = 0;
1612         size_t cursegmentlen;
1613         BYTE *inbuf = (BYTE *)malloc(inbufsize);
1614         BYTE *outbuf = (BYTE *)malloc(outbufsize);
1615         BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1616         size_t ret;
1617 
1618         CHECK(cdict == NULL, "failed to alloc cdict");
1619         CHECK(inbuf == NULL, "failed to alloc input buffer");
1620 
1621         /* first block is uncompressible */
1622         cursegmentlen = 128 * 1024;
1623         RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1624         inbufpos += cursegmentlen;
1625 
1626         /* second block is compressible */
1627         cursegmentlen = 128 * 1024 - 256;
1628         RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1629         inbufpos += cursegmentlen;
1630 
1631         /* and includes a very long backref */
1632         cursegmentlen = 128;
1633         memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1634         inbufpos += cursegmentlen;
1635 
1636         /* and includes a very long backref */
1637         cursegmentlen = 128;
1638         memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1639         inbufpos += cursegmentlen;
1640 
1641         ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1642         CHECK_Z(ret);
1643 
1644         ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1645         CHECK_Z(ret);
1646 
1647         CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1648 
1649         ZSTD_freeCDict(cdict);
1650         free(inbuf);
1651         free(outbuf);
1652         free(checkbuf);
1653     }
1654     DISPLAYLEVEL(3, "OK \n");
1655 
1656     DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1657     {   ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1658             dictionary.start, dictionary.filled,
1659             ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1660             ZSTD_getCParams(3, 0, dictionary.filled),
1661             ZSTD_defaultCMem);
1662         ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1663         int remainingInput = 256 * 1024;
1664         int offset;
1665 
1666         CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1667         CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1668         CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1669         /* Write a bunch of 6 byte blocks */
1670         while (remainingInput > 0) {
1671           char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1672           const size_t kSmallBlockSize = sizeof(testBuffer);
1673           ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1674 
1675           CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1676           CHECK(in.pos != in.size, "input not fully consumed");
1677           remainingInput -= kSmallBlockSize;
1678         }
1679         /* Write several very long offset matches into the dictionary */
1680         for (offset = 1024; offset >= 0; offset -= 128) {
1681           ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1682           ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1683           CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1684           CHECK(in.pos != in.size, "input not fully consumed");
1685         }
1686         /* Ensure decompression works */
1687         CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1688 
1689         ZSTD_freeCDict(cdict);
1690     }
1691     DISPLAYLEVEL(3, "OK \n");
1692 
1693 _end:
1694     FUZ_freeDictionary(dictionary);
1695     ZSTD_freeCStream(zc);
1696     ZSTD_freeDStream(zd);
1697     ZSTD_freeCCtx(mtctx);
1698     free(CNBuffer);
1699     free(compressedBuffer);
1700     free(decodedBuffer);
1701     return testResult;
1702 
1703 _output_error:
1704     testResult = 1;
1705     DISPLAY("Error detected in Unit tests ! \n");
1706     goto _end;
1707 }
1708 
1709 
1710 /* ======   Fuzzer tests   ====== */
1711 
findDiff(const void * buf1,const void * buf2,size_t max)1712 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
1713 {
1714     const BYTE* b1 = (const BYTE*)buf1;
1715     const BYTE* b2 = (const BYTE*)buf2;
1716     size_t u;
1717     for (u=0; u<max; u++) {
1718         if (b1[u] != b2[u]) break;
1719     }
1720     if (u==max) {
1721         DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
1722         return u;
1723     }
1724     DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
1725     if (u>=3)
1726         DISPLAY(" %02X %02X %02X ",
1727                 b1[u-3], b1[u-2], b1[u-1]);
1728     DISPLAY(" :%02X:  %02X %02X %02X %02X %02X \n",
1729             b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
1730     if (u>=3)
1731         DISPLAY(" %02X %02X %02X ",
1732                 b2[u-3], b2[u-2], b2[u-1]);
1733     DISPLAY(" :%02X:  %02X %02X %02X %02X %02X \n",
1734             b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
1735     return u;
1736 }
1737 
FUZ_rLogLength(U32 * seed,U32 logLength)1738 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
1739 {
1740     size_t const lengthMask = ((size_t)1 << logLength) - 1;
1741     return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
1742 }
1743 
FUZ_randomLength(U32 * seed,U32 maxLog)1744 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
1745 {
1746     U32 const logLength = FUZ_rand(seed) % maxLog;
1747     return FUZ_rLogLength(seed, logLength);
1748 }
1749 
1750 /* Return value in range minVal <= v <= maxVal */
FUZ_randomClampedLength(U32 * seed,U32 minVal,U32 maxVal)1751 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
1752 {
1753     U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
1754     return (U32)((FUZ_rand(seed) % mod) + minVal);
1755 }
1756 
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,int bigTests)1757 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
1758 {
1759     U32 const maxSrcLog = bigTests ? 24 : 22;
1760     static const U32 maxSampleLog = 19;
1761     size_t const srcBufferSize = (size_t)1<<maxSrcLog;
1762     BYTE* cNoiseBuffer[5];
1763     size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
1764     BYTE*  const copyBuffer = (BYTE*)malloc (copyBufferSize);
1765     size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
1766     BYTE*  const cBuffer = (BYTE*)malloc (cBufferSize);
1767     size_t const dstBufferSize = srcBufferSize;
1768     BYTE*  const dstBuffer = (BYTE*)malloc (dstBufferSize);
1769     U32 result = 0;
1770     unsigned testNb = 0;
1771     U32 coreSeed = seed;
1772     ZSTD_CStream* zc = ZSTD_createCStream();   /* will be re-created sometimes */
1773     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be re-created sometimes */
1774     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
1775     UTIL_time_t const startClock = UTIL_getTime();
1776     const BYTE* dict = NULL;  /* can keep same dict on 2 consecutive tests */
1777     size_t dictSize = 0;
1778     U32 oldTestLog = 0;
1779     U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
1780 
1781     /* allocations */
1782     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
1783     cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
1784     cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
1785     cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
1786     cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
1787     CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
1788            !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
1789            "Not enough memory, fuzzer tests cancelled");
1790 
1791     /* Create initial samples */
1792     RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
1793     RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
1794     RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
1795     RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
1796     RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
1797     memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
1798     ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
1799 
1800     /* catch up testNb */
1801     for (testNb=1; testNb < startTest; testNb++)
1802         FUZ_rand(&coreSeed);
1803 
1804     /* test loop */
1805     for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
1806         U32 lseed;
1807         const BYTE* srcBuffer;
1808         size_t totalTestSize, totalGenSize, cSize;
1809         XXH64_state_t xxhState;
1810         U64 crcOrig;
1811         U32 resetAllowed = 1;
1812         size_t maxTestSize;
1813 
1814         /* init */
1815         FUZ_rand(&coreSeed);
1816         lseed = coreSeed ^ prime32;
1817         if (nbTests >= testNb) {
1818             DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests);
1819         } else {
1820             DISPLAYUPDATE(2, "\r%6u        ", testNb);
1821         }
1822 
1823         /* states full reset (deliberately not synchronized) */
1824         /* some issues can only happen when reusing states */
1825         if ((FUZ_rand(&lseed) & 0xFF) == 131) {
1826             ZSTD_freeCStream(zc);
1827             zc = ZSTD_createCStream();
1828             CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
1829             resetAllowed=0;
1830         }
1831         if ((FUZ_rand(&lseed) & 0xFF) == 132) {
1832             ZSTD_freeDStream(zd);
1833             zd = ZSTD_createDStream();
1834             CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
1835             CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) );  /* ensure at least one init */
1836         }
1837 
1838         /* srcBuffer selection [0-4] */
1839         {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
1840             if (buffNb & 7) buffNb=2;   /* most common : compressible (P) */
1841             else {
1842                 buffNb >>= 3;
1843                 if (buffNb & 7) {
1844                     const U32 tnb[2] = { 1, 3 };   /* barely/highly compressible */
1845                     buffNb = tnb[buffNb >> 3];
1846                 } else {
1847                     const U32 tnb[2] = { 0, 4 };   /* not compressible / sparse */
1848                     buffNb = tnb[buffNb >> 3];
1849             }   }
1850             srcBuffer = cNoiseBuffer[buffNb];
1851         }
1852 
1853         /* compression init */
1854         if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
1855             && oldTestLog /* at least one test happened */ && resetAllowed) {
1856             maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
1857             maxTestSize = MIN(maxTestSize, srcBufferSize-16);
1858             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1859                 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1860                 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
1861             }
1862         } else {
1863             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
1864             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
1865             U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
1866                                 (ZSTD_maxCLevel() -
1867                                 (MAX(testLog, dictLog) / 3)))
1868                                  + 1;
1869             U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
1870             maxTestSize = FUZ_rLogLength(&lseed, testLog);
1871             oldTestLog = testLog;
1872             /* random dictionary selection */
1873             dictSize  = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
1874             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
1875                 dict = srcBuffer + dictStart;
1876             }
1877             {   U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
1878                 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1879                 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
1880                 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
1881                 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
1882                 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
1883                 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
1884                 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
1885         }   }
1886 
1887         /* multi-segments compression test */
1888         XXH64_reset(&xxhState, 0);
1889         {   ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
1890             U32 n;
1891             for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
1892                 /* compress random chunks into randomly sized dst buffers */
1893                 {   size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1894                     size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
1895                     size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
1896                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1897                     size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
1898                     ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
1899                     outBuff.size = outBuff.pos + dstBuffSize;
1900 
1901                     CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1902 
1903                     XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
1904                     memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
1905                     totalTestSize += inBuff.pos;
1906                 }
1907 
1908                 /* random flush operation, to mess around */
1909                 if ((FUZ_rand(&lseed) & 15) == 0) {
1910                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1911                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1912                     outBuff.size = outBuff.pos + adjustedDstSize;
1913                     CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
1914             }   }
1915 
1916             /* final frame epilogue */
1917             {   size_t remainingToFlush = (size_t)(-1);
1918                 while (remainingToFlush) {
1919                     size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1920                     size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
1921                     outBuff.size = outBuff.pos + adjustedDstSize;
1922                     remainingToFlush = ZSTD_endStream(zc, &outBuff);
1923                     CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
1924             }   }
1925             crcOrig = XXH64_digest(&xxhState);
1926             cSize = outBuff.pos;
1927         }
1928 
1929         /* multi - fragments decompression test */
1930         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
1931             CHECK_Z ( ZSTD_resetDStream(zd) );
1932         } else {
1933             CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
1934         }
1935         {   size_t decompressionResult = 1;
1936             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
1937             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1938             for (totalGenSize = 0 ; decompressionResult ; ) {
1939                 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1940                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1941                 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
1942                 inBuff.size = inBuff.pos + readCSrcSize;
1943                 outBuff.size = outBuff.pos + dstBuffSize;
1944                 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1945                 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
1946                     DISPLAY("checksum error : \n");
1947                     findDiff(copyBuffer, dstBuffer, totalTestSize);
1948                 }
1949                 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
1950                        ZSTD_getErrorName(decompressionResult) );
1951             }
1952             CHECK (decompressionResult != 0, "frame not fully decoded");
1953             CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
1954                     (unsigned)outBuff.pos, (unsigned)totalTestSize);
1955             CHECK (inBuff.pos != cSize, "compressed data should be fully read")
1956             {   U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
1957                 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
1958                 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
1959         }   }
1960 
1961         /*=====   noisy/erroneous src decompression test   =====*/
1962 
1963         /* add some noise */
1964         {   U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
1965             U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
1966                 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
1967                 size_t const noiseSize  = MIN((cSize/3) , randomNoiseSize);
1968                 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
1969                 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
1970                 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
1971         }   }
1972 
1973         /* try decompression on noisy data */
1974         CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
1975         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
1976             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
1977             while (outBuff.pos < dstBufferSize) {
1978                 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
1979                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
1980                 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
1981                 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
1982                 outBuff.size = outBuff.pos + adjustedDstSize;
1983                 inBuff.size  = inBuff.pos + adjustedCSrcSize;
1984                 {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1985                     if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
1986                     /* No forward progress possible */
1987                     if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
1988     }   }   }   }
1989     DISPLAY("\r%u fuzzer tests completed   \n", testNb);
1990 
1991 _cleanup:
1992     ZSTD_freeCStream(zc);
1993     ZSTD_freeDStream(zd);
1994     ZSTD_freeDStream(zd_noise);
1995     free(cNoiseBuffer[0]);
1996     free(cNoiseBuffer[1]);
1997     free(cNoiseBuffer[2]);
1998     free(cNoiseBuffer[3]);
1999     free(cNoiseBuffer[4]);
2000     free(copyBuffer);
2001     free(cBuffer);
2002     free(dstBuffer);
2003     return result;
2004 
2005 _output_error:
2006     result = 1;
2007     goto _cleanup;
2008 }
2009 
2010 /** If useOpaqueAPI, sets param in cctxParams.
2011  *  Otherwise, sets the param in zc. */
setCCtxParameter(ZSTD_CCtx * zc,ZSTD_CCtx_params * cctxParams,ZSTD_cParameter param,unsigned value,int useOpaqueAPI)2012 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2013                                ZSTD_cParameter param, unsigned value,
2014                                int useOpaqueAPI)
2015 {
2016     if (useOpaqueAPI) {
2017         return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2018     } else {
2019         return ZSTD_CCtx_setParameter(zc, param, value);
2020     }
2021 }
2022 
2023 /* Tests for ZSTD_compress_generic() API */
fuzzerTests_newAPI(U32 seed,int nbTests,int startTest,double compressibility,int bigTests)2024 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2025                               double compressibility, int bigTests)
2026 {
2027     U32 const maxSrcLog = bigTests ? 24 : 22;
2028     static const U32 maxSampleLog = 19;
2029     size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2030     BYTE* cNoiseBuffer[5];
2031     size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2032     BYTE*  const copyBuffer = (BYTE*)malloc (copyBufferSize);
2033     size_t const cBufferSize   = ZSTD_compressBound(srcBufferSize);
2034     BYTE*  const cBuffer = (BYTE*)malloc (cBufferSize);
2035     size_t const dstBufferSize = srcBufferSize;
2036     BYTE*  const dstBuffer = (BYTE*)malloc (dstBufferSize);
2037     U32 result = 0;
2038     int testNb = 0;
2039     U32 coreSeed = seed;
2040     ZSTD_CCtx* zc = ZSTD_createCCtx();   /* will be reset sometimes */
2041     ZSTD_DStream* zd = ZSTD_createDStream();   /* will be reset sometimes */
2042     ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2043     UTIL_time_t const startClock = UTIL_getTime();
2044     const BYTE* dict = NULL;   /* can keep same dict on 2 consecutive tests */
2045     size_t dictSize = 0;
2046     U32 oldTestLog = 0;
2047     U32 windowLogMalus = 0;   /* can survive between 2 loops */
2048     U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2049     U32 const nbThreadsMax = bigTests ? 4 : 2;
2050     ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2051 
2052     /* allocations */
2053     cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2054     cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2055     cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2056     cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2057     cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2058     CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2059            !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2060            "Not enough memory, fuzzer tests cancelled");
2061 
2062     /* Create initial samples */
2063     RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed);    /* pure noise */
2064     RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed);    /* barely compressible */
2065     RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2066     RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed);    /* highly compressible */
2067     RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed);    /* sparse content */
2068     memset(copyBuffer, 0x65, copyBufferSize);                             /* make copyBuffer considered initialized */
2069     CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) );   /* ensure at least one init */
2070 
2071     /* catch up testNb */
2072     for (testNb=1; testNb < startTest; testNb++)
2073         FUZ_rand(&coreSeed);
2074 
2075     /* test loop */
2076     for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2077         U32 lseed;
2078         int opaqueAPI;
2079         const BYTE* srcBuffer;
2080         size_t totalTestSize, totalGenSize, cSize;
2081         XXH64_state_t xxhState;
2082         U64 crcOrig;
2083         U32 resetAllowed = 1;
2084         size_t maxTestSize;
2085         ZSTD_parameters savedParams;
2086         int isRefPrefix = 0;
2087         U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2088 
2089         /* init */
2090         if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u    ", testNb, nbTests); }
2091         else { DISPLAYUPDATE(2, "\r%6u          ", testNb); }
2092         FUZ_rand(&coreSeed);
2093         lseed = coreSeed ^ prime32;
2094         DISPLAYLEVEL(5, " ***  Test %u  *** \n", testNb);
2095         opaqueAPI = FUZ_rand(&lseed) & 1;
2096 
2097         /* states full reset (deliberately not synchronized) */
2098         /* some issues can only happen when reusing states */
2099         if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2100             DISPLAYLEVEL(5, "Creating new context \n");
2101             ZSTD_freeCCtx(zc);
2102             zc = ZSTD_createCCtx();
2103             CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2104             resetAllowed = 0;
2105         }
2106         if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2107             ZSTD_freeDStream(zd);
2108             zd = ZSTD_createDStream();
2109             CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2110             ZSTD_initDStream_usingDict(zd, NULL, 0);  /* ensure at least one init */
2111         }
2112 
2113         /* srcBuffer selection [0-4] */
2114         {   U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2115             if (buffNb & 7) buffNb=2;   /* most common : compressible (P) */
2116             else {
2117                 buffNb >>= 3;
2118                 if (buffNb & 7) {
2119                     const U32 tnb[2] = { 1, 3 };   /* barely/highly compressible */
2120                     buffNb = tnb[buffNb >> 3];
2121                 } else {
2122                     const U32 tnb[2] = { 0, 4 };   /* not compressible / sparse */
2123                     buffNb = tnb[buffNb >> 3];
2124             }   }
2125             srcBuffer = cNoiseBuffer[buffNb];
2126         }
2127 
2128         /* compression init */
2129         CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* cancel previous dict /*/
2130         if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2131           && oldTestLog   /* at least one test happened */
2132           && resetAllowed) {
2133             /* just set a compression level */
2134             maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2135             if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2136             {   int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2137                 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2138                 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2139             }
2140         } else {
2141             U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2142             U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2143             U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2144                                (ZSTD_maxCLevel() -
2145                                (MAX(testLog, dictLog) / 2))) +
2146                                1;
2147             int const cLevel = MIN(cLevelCandidate, cLevelMax);
2148             DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2149             maxTestSize = FUZ_rLogLength(&lseed, testLog);
2150             DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2151             oldTestLog = testLog;
2152             /* random dictionary selection */
2153             dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2154             {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2155                 dict = srcBuffer + dictStart;
2156                 if (!dictSize) dict=NULL;
2157             }
2158             pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2159             {   ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2160                 const U32 windowLogMax = bigTests ? 24 : 20;
2161                 const U32 searchLogMax = bigTests ? 15 : 13;
2162                 if (dictSize)
2163                     DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2164 
2165                 /* mess with compression parameters */
2166                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2167                 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2168                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2169                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2170                 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2171                 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2172                 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2173                 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2174                 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2175 
2176                 if (FUZ_rand(&lseed) & 1) {
2177                     DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2178                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2179                     assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN);   /* guaranteed by ZSTD_adjustCParams() */
2180                     windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2181                 }
2182                 if (FUZ_rand(&lseed) & 1) {
2183                     DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2184                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2185                 }
2186                 if (FUZ_rand(&lseed) & 1) {
2187                     DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2188                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2189                 }
2190                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2191                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2192                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2193 
2194                 /* mess with long distance matching parameters */
2195                 if (bigTests) {
2196                     if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) );
2197                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2198                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2199                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2200                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2201                     if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2202                 }
2203 
2204                 /* mess with frame parameters */
2205                 if (FUZ_rand(&lseed) & 1) {
2206                     int const checksumFlag = FUZ_rand(&lseed) & 1;
2207                     DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2208                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2209                 }
2210                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2211                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2212                 if (FUZ_rand(&lseed) & 1) {
2213                     DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2214                     CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2215                 } else {
2216                     pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2217                 }
2218 
2219                 /* multi-threading parameters. Only adjust occasionally for small tests. */
2220                 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2221                     U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2222                     U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2223                     int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2224                     DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2225                     CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2226                     if (nbThreads > 1) {
2227                         U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2228                         CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2229                         CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2230                     }
2231                 }
2232                 /* Enable rsyncable mode 1 in 4 times. */
2233                 {
2234                     int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2235                     DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2236                     setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
2237                 }
2238 
2239                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
2240                 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
2241 
2242                 /* Apply parameters */
2243                 if (opaqueAPI) {
2244                     DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
2245                     CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
2246                 }
2247 
2248                 if (FUZ_rand(&lseed) & 1) {
2249                     if (FUZ_rand(&lseed) & 1) {
2250                         CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2251                     } else {
2252                         CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
2253                     }
2254                 } else {
2255                     isRefPrefix = 1;
2256                     CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2257                 }
2258         }   }
2259 
2260         CHECK_Z(getCCtxParams(zc, &savedParams));
2261 
2262         /* multi-segments compression test */
2263         {   int iter;
2264             int const startSeed = lseed;
2265             XXH64_hash_t compressedCrcs[2];
2266             for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
2267                 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2268                 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
2269                 int nbWorkers;
2270 
2271                 XXH64_reset(&xxhState, 0);
2272 
2273                 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2274                 if (isRefPrefix) {
2275                     DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
2276                     /* Need to reload the prefix because it gets dropped after one compression */
2277                     CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2278                 }
2279 
2280                 /* Adjust number of workers occassionally - result must be deterministic independent of nbWorkers */
2281                 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
2282                 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
2283                     DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
2284                     CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
2285                 }
2286 
2287                 if (singlePass) {
2288                     ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
2289                     CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
2290                     DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
2291                         testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2292                     CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
2293                     crcOrig = XXH64(srcBuffer, maxTestSize, 0);
2294                     totalTestSize = maxTestSize;
2295                 } else {
2296                     outBuff.size = 0;
2297                     for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
2298                         /* compress random chunks into randomly sized dst buffers */
2299                         size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2300                         size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2301                         size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2302                         ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
2303                         ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2304                         int forwardProgress;
2305                         do {
2306                             size_t const ipos = inBuff.pos;
2307                             size_t const opos = outBuff.pos;
2308                             size_t ret;
2309                             if (outBuff.pos == outBuff.size) {
2310                                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2311                                 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2312                                 outBuff.size = outBuff.pos + dstBuffSize;
2313                             }
2314                             CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
2315                             DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
2316                                 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
2317 
2318                             /* We've completed the flush */
2319                             if (flush == ZSTD_e_flush && ret == 0)
2320                                 break;
2321 
2322                             /* Ensure maximal forward progress for determinism */
2323                             forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
2324                         } while (forwardProgress);
2325                         assert(inBuff.pos == inBuff.size);
2326 
2327                         XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2328                         memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2329                         totalTestSize += inBuff.pos;
2330                     }
2331 
2332                     /* final frame epilogue */
2333                     {   size_t remainingToFlush = 1;
2334                         while (remainingToFlush) {
2335                             ZSTD_inBuffer inBuff = { NULL, 0, 0 };
2336                             size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2337                             size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2338                             outBuff.size = outBuff.pos + adjustedDstSize;
2339                             DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
2340                             /* ZSTD_e_end guarantees maximal forward progress */
2341                             remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
2342                             DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
2343                             CHECK( ZSTD_isError(remainingToFlush),
2344                                 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
2345                                 ZSTD_getErrorName(remainingToFlush) );
2346                     }   }
2347                     crcOrig = XXH64_digest(&xxhState);
2348                 }
2349                 cSize = outBuff.pos;
2350                 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
2351                 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
2352             }
2353             CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
2354         }
2355 
2356         CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
2357 
2358         /* multi - fragments decompression test */
2359         if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2360             DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", dict);
2361             CHECK_Z( ZSTD_resetDStream(zd) );
2362         } else {
2363             if (dictSize)
2364                 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
2365             CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2366         }
2367         {   size_t decompressionResult = 1;
2368             ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
2369             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2370             for (totalGenSize = 0 ; decompressionResult ; ) {
2371                 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2372                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2373                 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2374                 inBuff.size = inBuff.pos + readCSrcSize;
2375                 outBuff.size = outBuff.pos + dstBuffSize;
2376                 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
2377                                 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
2378                 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2379                 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
2380                                 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2381                 if (ZSTD_isError(decompressionResult)) {
2382                     DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
2383                     findDiff(copyBuffer, dstBuffer, totalTestSize);
2384                 }
2385                 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
2386                 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
2387             }
2388             CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
2389             CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
2390             {   U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2391                 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2392                 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2393         }   }
2394 
2395         /*=====   noisy/erroneous src decompression test   =====*/
2396 
2397         /* add some noise */
2398         {   U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2399             U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2400                 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2401                 size_t const noiseSize  = MIN((cSize/3) , randomNoiseSize);
2402                 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2403                 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2404                 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2405         }   }
2406 
2407         /* try decompression on noisy data */
2408         CHECK_Z( ZSTD_initDStream(zd_noise) );   /* note : no dictionary */
2409         {   ZSTD_inBuffer  inBuff = { cBuffer, cSize, 0 };
2410             ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2411             while (outBuff.pos < dstBufferSize) {
2412                 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2413                 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2414                 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2415                 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2416                 outBuff.size = outBuff.pos + adjustedDstSize;
2417                 inBuff.size  = inBuff.pos + adjustedCSrcSize;
2418                 {   size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2419                     if (ZSTD_isError(decompressError)) break;   /* error correctly detected */
2420                     /* Good so far, but no more progress possible */
2421                     if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2422     }   }   }   }
2423     DISPLAY("\r%u fuzzer tests completed   \n", testNb-1);
2424 
2425 _cleanup:
2426     ZSTD_freeCCtx(zc);
2427     ZSTD_freeDStream(zd);
2428     ZSTD_freeDStream(zd_noise);
2429     ZSTD_freeCCtxParams(cctxParams);
2430     free(cNoiseBuffer[0]);
2431     free(cNoiseBuffer[1]);
2432     free(cNoiseBuffer[2]);
2433     free(cNoiseBuffer[3]);
2434     free(cNoiseBuffer[4]);
2435     free(copyBuffer);
2436     free(cBuffer);
2437     free(dstBuffer);
2438     return result;
2439 
2440 _output_error:
2441     result = 1;
2442     goto _cleanup;
2443 }
2444 
2445 /*-*******************************************************
2446 *  Command line
2447 *********************************************************/
FUZ_usage(const char * programName)2448 static int FUZ_usage(const char* programName)
2449 {
2450     DISPLAY( "Usage :\n");
2451     DISPLAY( "      %s [args]\n", programName);
2452     DISPLAY( "\n");
2453     DISPLAY( "Arguments :\n");
2454     DISPLAY( " -i#    : Number of tests (default:%u)\n", nbTestsDefault);
2455     DISPLAY( " -T#    : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
2456     DISPLAY( " -s#    : Select seed (default:prompt user)\n");
2457     DISPLAY( " -t#    : Select starting test number (default:0)\n");
2458     DISPLAY( " -P#    : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
2459     DISPLAY( " -v     : verbose\n");
2460     DISPLAY( " -p     : pause at the end\n");
2461     DISPLAY( " -h     : display help and exit\n");
2462     return 0;
2463 }
2464 
2465 typedef enum { simple_api, advanced_api } e_api;
2466 
main(int argc,const char ** argv)2467 int main(int argc, const char** argv)
2468 {
2469     U32 seed = 0;
2470     int seedset = 0;
2471     int nbTests = nbTestsDefault;
2472     int testNb = 0;
2473     int proba = FUZ_COMPRESSIBILITY_DEFAULT;
2474     int result = 0;
2475     int mainPause = 0;
2476     int bigTests = (sizeof(size_t) == 8);
2477     e_api selected_api = simple_api;
2478     const char* const programName = argv[0];
2479     int argNb;
2480 
2481     /* Check command line */
2482     for(argNb=1; argNb<argc; argNb++) {
2483         const char* argument = argv[argNb];
2484         assert(argument != NULL);
2485 
2486         /* Parsing commands. Aggregated commands are allowed */
2487         if (argument[0]=='-') {
2488 
2489             if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
2490             if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
2491 
2492             argument++;
2493             while (*argument!=0) {
2494                 switch(*argument)
2495                 {
2496                 case 'h':
2497                     return FUZ_usage(programName);
2498 
2499                 case 'v':
2500                     argument++;
2501                     g_displayLevel++;
2502                     break;
2503 
2504                 case 'q':
2505                     argument++;
2506                     g_displayLevel--;
2507                     break;
2508 
2509                 case 'p': /* pause at the end */
2510                     argument++;
2511                     mainPause = 1;
2512                     break;
2513 
2514                 case 'i':   /* limit tests by nb of iterations (default) */
2515                     argument++;
2516                     nbTests=0; g_clockTime=0;
2517                     while ((*argument>='0') && (*argument<='9')) {
2518                         nbTests *= 10;
2519                         nbTests += *argument - '0';
2520                         argument++;
2521                     }
2522                     break;
2523 
2524                 case 'T':   /* limit tests by time */
2525                     argument++;
2526                     nbTests=0; g_clockTime=0;
2527                     while ((*argument>='0') && (*argument<='9')) {
2528                         g_clockTime *= 10;
2529                         g_clockTime += *argument - '0';
2530                         argument++;
2531                     }
2532                     if (*argument=='m') {    /* -T1m == -T60 */
2533                         g_clockTime *=60, argument++;
2534                         if (*argument=='n') argument++; /* -T1mn == -T60 */
2535                     } else if (*argument=='s') argument++; /* -T10s == -T10 */
2536                     g_clockTime *= SEC_TO_MICRO;
2537                     break;
2538 
2539                 case 's':   /* manually select seed */
2540                     argument++;
2541                     seedset=1;
2542                     seed=0;
2543                     while ((*argument>='0') && (*argument<='9')) {
2544                         seed *= 10;
2545                         seed += *argument - '0';
2546                         argument++;
2547                     }
2548                     break;
2549 
2550                 case 't':   /* select starting test number */
2551                     argument++;
2552                     testNb=0;
2553                     while ((*argument>='0') && (*argument<='9')) {
2554                         testNb *= 10;
2555                         testNb += *argument - '0';
2556                         argument++;
2557                     }
2558                     break;
2559 
2560                 case 'P':   /* compressibility % */
2561                     argument++;
2562                     proba=0;
2563                     while ((*argument>='0') && (*argument<='9')) {
2564                         proba *= 10;
2565                         proba += *argument - '0';
2566                         argument++;
2567                     }
2568                     if (proba<0) proba=0;
2569                     if (proba>100) proba=100;
2570                     break;
2571 
2572                 default:
2573                     return FUZ_usage(programName);
2574                 }
2575     }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
2576 
2577     /* Get Seed */
2578     DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
2579 
2580     if (!seedset) {
2581         time_t const t = time(NULL);
2582         U32 const h = XXH32(&t, sizeof(t), 1);
2583         seed = h % 10000;
2584     }
2585 
2586     DISPLAY("Seed = %u\n", (unsigned)seed);
2587     if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
2588 
2589     if (nbTests<=0) nbTests=1;
2590 
2591     if (testNb==0) {
2592         result = basicUnitTests(0, ((double)proba) / 100);  /* constant seed for predictability */
2593     }
2594 
2595     if (!result) {
2596         switch(selected_api)
2597         {
2598         case simple_api :
2599             result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2600             break;
2601         case advanced_api :
2602             result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
2603             break;
2604         default :
2605             assert(0);   /* impossible */
2606         }
2607     }
2608 
2609     if (mainPause) {
2610         int unused;
2611         DISPLAY("Press Enter \n");
2612         unused = getchar();
2613         (void)unused;
2614     }
2615     return result;
2616 }
2617