1 /* 2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /* *************************************************************** 13 * Tuning parameters 14 *****************************************************************/ 15 /*! 16 * HEAPMODE : 17 * Select how default decompression function ZSTD_decompress() allocates its context, 18 * on stack (0), or into heap (1, default; requires malloc()). 19 * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. 20 */ 21 #ifndef ZSTD_HEAPMODE 22 # define ZSTD_HEAPMODE 1 23 #endif 24 25 /*! 26 * LEGACY_SUPPORT : 27 * if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) 28 */ 29 #ifndef ZSTD_LEGACY_SUPPORT 30 # define ZSTD_LEGACY_SUPPORT 0 31 #endif 32 33 /*! 34 * MAXWINDOWSIZE_DEFAULT : 35 * maximum window size accepted by DStream __by default__. 36 * Frames requiring more memory will be rejected. 37 * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). 38 */ 39 #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT 40 # define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) 41 #endif 42 43 /*! 44 * NO_FORWARD_PROGRESS_MAX : 45 * maximum allowed nb of calls to ZSTD_decompressStream() 46 * without any forward progress 47 * (defined as: no byte read from input, and no byte flushed to output) 48 * before triggering an error. 49 */ 50 #ifndef ZSTD_NO_FORWARD_PROGRESS_MAX 51 # define ZSTD_NO_FORWARD_PROGRESS_MAX 16 52 #endif 53 54 55 /*-******************************************************* 56 * Dependencies 57 *********************************************************/ 58 #include <string.h> /* memcpy, memmove, memset */ 59 #include "cpu.h" /* bmi2 */ 60 #include "mem.h" /* low level memory routines */ 61 #define FSE_STATIC_LINKING_ONLY 62 #include "fse.h" 63 #define HUF_STATIC_LINKING_ONLY 64 #include "huf.h" 65 #include "zstd_internal.h" /* blockProperties_t */ 66 #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ 67 #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ 68 #include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ 69 70 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 71 # include "zstd_legacy.h" 72 #endif 73 74 75 /*-************************************************************* 76 * Context management 77 ***************************************************************/ 78 size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) 79 { 80 if (dctx==NULL) return 0; /* support sizeof NULL */ 81 return sizeof(*dctx) 82 + ZSTD_sizeof_DDict(dctx->ddictLocal) 83 + dctx->inBuffSize + dctx->outBuffSize; 84 } 85 86 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } 87 88 89 static size_t ZSTD_startingInputLength(ZSTD_format_e format) 90 { 91 size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? 92 ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE : 93 ZSTD_FRAMEHEADERSIZE_PREFIX; 94 ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); 95 /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ 96 assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); 97 return startingInputLength; 98 } 99 100 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) 101 { 102 dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */ 103 dctx->staticSize = 0; 104 dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; 105 dctx->ddict = NULL; 106 dctx->ddictLocal = NULL; 107 dctx->dictEnd = NULL; 108 dctx->ddictIsCold = 0; 109 dctx->dictUses = ZSTD_dont_use; 110 dctx->inBuff = NULL; 111 dctx->inBuffSize = 0; 112 dctx->outBuffSize = 0; 113 dctx->streamStage = zdss_init; 114 dctx->legacyContext = NULL; 115 dctx->previousLegacyVersion = 0; 116 dctx->noForwardProgress = 0; 117 dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); 118 } 119 120 ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) 121 { 122 ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; 123 124 if ((size_t)workspace & 7) return NULL; /* 8-aligned */ 125 if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ 126 127 ZSTD_initDCtx_internal(dctx); 128 dctx->staticSize = workspaceSize; 129 dctx->inBuff = (char*)(dctx+1); 130 return dctx; 131 } 132 133 ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) 134 { 135 if (!customMem.customAlloc ^ !customMem.customFree) return NULL; 136 137 { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); 138 if (!dctx) return NULL; 139 dctx->customMem = customMem; 140 ZSTD_initDCtx_internal(dctx); 141 return dctx; 142 } 143 } 144 145 ZSTD_DCtx* ZSTD_createDCtx(void) 146 { 147 DEBUGLOG(3, "ZSTD_createDCtx"); 148 return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); 149 } 150 151 static void ZSTD_clearDict(ZSTD_DCtx* dctx) 152 { 153 ZSTD_freeDDict(dctx->ddictLocal); 154 dctx->ddictLocal = NULL; 155 dctx->ddict = NULL; 156 dctx->dictUses = ZSTD_dont_use; 157 } 158 159 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) 160 { 161 if (dctx==NULL) return 0; /* support free on NULL */ 162 RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); 163 { ZSTD_customMem const cMem = dctx->customMem; 164 ZSTD_clearDict(dctx); 165 ZSTD_free(dctx->inBuff, cMem); 166 dctx->inBuff = NULL; 167 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 168 if (dctx->legacyContext) 169 ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); 170 #endif 171 ZSTD_free(dctx, cMem); 172 return 0; 173 } 174 } 175 176 /* no longer useful */ 177 void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) 178 { 179 size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); 180 memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ 181 } 182 183 184 /*-************************************************************* 185 * Frame header decoding 186 ***************************************************************/ 187 188 /*! ZSTD_isFrame() : 189 * Tells if the content of `buffer` starts with a valid Frame Identifier. 190 * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. 191 * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. 192 * Note 3 : Skippable Frame Identifiers are considered valid. */ 193 unsigned ZSTD_isFrame(const void* buffer, size_t size) 194 { 195 if (size < ZSTD_FRAMEIDSIZE) return 0; 196 { U32 const magic = MEM_readLE32(buffer); 197 if (magic == ZSTD_MAGICNUMBER) return 1; 198 if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; 199 } 200 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 201 if (ZSTD_isLegacy(buffer, size)) return 1; 202 #endif 203 return 0; 204 } 205 206 /** ZSTD_frameHeaderSize_internal() : 207 * srcSize must be large enough to reach header size fields. 208 * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. 209 * @return : size of the Frame Header 210 * or an error code, which can be tested with ZSTD_isError() */ 211 static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) 212 { 213 size_t const minInputSize = ZSTD_startingInputLength(format); 214 RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong); 215 216 { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; 217 U32 const dictID= fhd & 3; 218 U32 const singleSegment = (fhd >> 5) & 1; 219 U32 const fcsId = fhd >> 6; 220 return minInputSize + !singleSegment 221 + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] 222 + (singleSegment && !fcsId); 223 } 224 } 225 226 /** ZSTD_frameHeaderSize() : 227 * srcSize must be >= ZSTD_frameHeaderSize_prefix. 228 * @return : size of the Frame Header, 229 * or an error code (if srcSize is too small) */ 230 size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) 231 { 232 return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); 233 } 234 235 236 /** ZSTD_getFrameHeader_advanced() : 237 * decode Frame Header, or require larger `srcSize`. 238 * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless 239 * @return : 0, `zfhPtr` is correctly filled, 240 * >0, `srcSize` is too small, value is wanted `srcSize` amount, 241 * or an error code, which can be tested using ZSTD_isError() */ 242 size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) 243 { 244 const BYTE* ip = (const BYTE*)src; 245 size_t const minInputSize = ZSTD_startingInputLength(format); 246 247 memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ 248 if (srcSize < minInputSize) return minInputSize; 249 RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); 250 251 if ( (format != ZSTD_f_zstd1_magicless) 252 && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { 253 if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 254 /* skippable frame */ 255 if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) 256 return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ 257 memset(zfhPtr, 0, sizeof(*zfhPtr)); 258 zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); 259 zfhPtr->frameType = ZSTD_skippableFrame; 260 return 0; 261 } 262 RETURN_ERROR(prefix_unknown); 263 } 264 265 /* ensure there is enough `srcSize` to fully read/decode frame header */ 266 { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); 267 if (srcSize < fhsize) return fhsize; 268 zfhPtr->headerSize = (U32)fhsize; 269 } 270 271 { BYTE const fhdByte = ip[minInputSize-1]; 272 size_t pos = minInputSize; 273 U32 const dictIDSizeCode = fhdByte&3; 274 U32 const checksumFlag = (fhdByte>>2)&1; 275 U32 const singleSegment = (fhdByte>>5)&1; 276 U32 const fcsID = fhdByte>>6; 277 U64 windowSize = 0; 278 U32 dictID = 0; 279 U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; 280 RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, 281 "reserved bits, must be zero"); 282 283 if (!singleSegment) { 284 BYTE const wlByte = ip[pos++]; 285 U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; 286 RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge); 287 windowSize = (1ULL << windowLog); 288 windowSize += (windowSize >> 3) * (wlByte&7); 289 } 290 switch(dictIDSizeCode) 291 { 292 default: assert(0); /* impossible */ 293 case 0 : break; 294 case 1 : dictID = ip[pos]; pos++; break; 295 case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; 296 case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; 297 } 298 switch(fcsID) 299 { 300 default: assert(0); /* impossible */ 301 case 0 : if (singleSegment) frameContentSize = ip[pos]; break; 302 case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; 303 case 2 : frameContentSize = MEM_readLE32(ip+pos); break; 304 case 3 : frameContentSize = MEM_readLE64(ip+pos); break; 305 } 306 if (singleSegment) windowSize = frameContentSize; 307 308 zfhPtr->frameType = ZSTD_frame; 309 zfhPtr->frameContentSize = frameContentSize; 310 zfhPtr->windowSize = windowSize; 311 zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 312 zfhPtr->dictID = dictID; 313 zfhPtr->checksumFlag = checksumFlag; 314 } 315 return 0; 316 } 317 318 /** ZSTD_getFrameHeader() : 319 * decode Frame Header, or require larger `srcSize`. 320 * note : this function does not consume input, it only reads it. 321 * @return : 0, `zfhPtr` is correctly filled, 322 * >0, `srcSize` is too small, value is wanted `srcSize` amount, 323 * or an error code, which can be tested using ZSTD_isError() */ 324 size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) 325 { 326 return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); 327 } 328 329 330 /** ZSTD_getFrameContentSize() : 331 * compatible with legacy mode 332 * @return : decompressed size of the single frame pointed to be `src` if known, otherwise 333 * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined 334 * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ 335 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) 336 { 337 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 338 if (ZSTD_isLegacy(src, srcSize)) { 339 unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); 340 return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; 341 } 342 #endif 343 { ZSTD_frameHeader zfh; 344 if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) 345 return ZSTD_CONTENTSIZE_ERROR; 346 if (zfh.frameType == ZSTD_skippableFrame) { 347 return 0; 348 } else { 349 return zfh.frameContentSize; 350 } } 351 } 352 353 static size_t readSkippableFrameSize(void const* src, size_t srcSize) 354 { 355 size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; 356 U32 sizeU32; 357 358 RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong); 359 360 sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); 361 RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, 362 frameParameter_unsupported); 363 { 364 size_t const skippableSize = skippableHeaderSize + sizeU32; 365 RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong); 366 return skippableSize; 367 } 368 } 369 370 /** ZSTD_findDecompressedSize() : 371 * compatible with legacy mode 372 * `srcSize` must be the exact length of some number of ZSTD compressed and/or 373 * skippable frames 374 * @return : decompressed size of the frames contained */ 375 unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) 376 { 377 unsigned long long totalDstSize = 0; 378 379 while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { 380 U32 const magicNumber = MEM_readLE32(src); 381 382 if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 383 size_t const skippableSize = readSkippableFrameSize(src, srcSize); 384 if (ZSTD_isError(skippableSize)) { 385 return ZSTD_CONTENTSIZE_ERROR; 386 } 387 assert(skippableSize <= srcSize); 388 389 src = (const BYTE *)src + skippableSize; 390 srcSize -= skippableSize; 391 continue; 392 } 393 394 { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); 395 if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; 396 397 /* check for overflow */ 398 if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; 399 totalDstSize += ret; 400 } 401 { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); 402 if (ZSTD_isError(frameSrcSize)) { 403 return ZSTD_CONTENTSIZE_ERROR; 404 } 405 406 src = (const BYTE *)src + frameSrcSize; 407 srcSize -= frameSrcSize; 408 } 409 } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ 410 411 if (srcSize) return ZSTD_CONTENTSIZE_ERROR; 412 413 return totalDstSize; 414 } 415 416 /** ZSTD_getDecompressedSize() : 417 * compatible with legacy mode 418 * @return : decompressed size if known, 0 otherwise 419 note : 0 can mean any of the following : 420 - frame content is empty 421 - decompressed size field is not present in frame header 422 - frame header unknown / not supported 423 - frame header not complete (`srcSize` too small) */ 424 unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) 425 { 426 unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); 427 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); 428 return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; 429 } 430 431 432 /** ZSTD_decodeFrameHeader() : 433 * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). 434 * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ 435 static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) 436 { 437 size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); 438 if (ZSTD_isError(result)) return result; /* invalid header */ 439 RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); 440 #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 441 /* Skip the dictID check in fuzzing mode, because it makes the search 442 * harder. 443 */ 444 RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), 445 dictionary_wrong); 446 #endif 447 if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); 448 return 0; 449 } 450 451 static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) 452 { 453 ZSTD_frameSizeInfo frameSizeInfo; 454 frameSizeInfo.compressedSize = ret; 455 frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; 456 return frameSizeInfo; 457 } 458 459 static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) 460 { 461 ZSTD_frameSizeInfo frameSizeInfo; 462 memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); 463 464 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 465 if (ZSTD_isLegacy(src, srcSize)) 466 return ZSTD_findFrameSizeInfoLegacy(src, srcSize); 467 #endif 468 469 if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) 470 && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 471 frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); 472 assert(ZSTD_isError(frameSizeInfo.compressedSize) || 473 frameSizeInfo.compressedSize <= srcSize); 474 return frameSizeInfo; 475 } else { 476 const BYTE* ip = (const BYTE*)src; 477 const BYTE* const ipstart = ip; 478 size_t remainingSize = srcSize; 479 size_t nbBlocks = 0; 480 ZSTD_frameHeader zfh; 481 482 /* Extract Frame Header */ 483 { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); 484 if (ZSTD_isError(ret)) 485 return ZSTD_errorFrameSizeInfo(ret); 486 if (ret > 0) 487 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 488 } 489 490 ip += zfh.headerSize; 491 remainingSize -= zfh.headerSize; 492 493 /* Iterate over each block */ 494 while (1) { 495 blockProperties_t blockProperties; 496 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); 497 if (ZSTD_isError(cBlockSize)) 498 return ZSTD_errorFrameSizeInfo(cBlockSize); 499 500 if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) 501 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 502 503 ip += ZSTD_blockHeaderSize + cBlockSize; 504 remainingSize -= ZSTD_blockHeaderSize + cBlockSize; 505 nbBlocks++; 506 507 if (blockProperties.lastBlock) break; 508 } 509 510 /* Final frame content checksum */ 511 if (zfh.checksumFlag) { 512 if (remainingSize < 4) 513 return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); 514 ip += 4; 515 } 516 517 frameSizeInfo.compressedSize = ip - ipstart; 518 frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) 519 ? zfh.frameContentSize 520 : nbBlocks * zfh.blockSizeMax; 521 return frameSizeInfo; 522 } 523 } 524 525 /** ZSTD_findFrameCompressedSize() : 526 * compatible with legacy mode 527 * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame 528 * `srcSize` must be at least as large as the frame contained 529 * @return : the compressed size of the frame starting at `src` */ 530 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) 531 { 532 ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); 533 return frameSizeInfo.compressedSize; 534 } 535 536 /** ZSTD_decompressBound() : 537 * compatible with legacy mode 538 * `src` must point to the start of a ZSTD frame or a skippeable frame 539 * `srcSize` must be at least as large as the frame contained 540 * @return : the maximum decompressed size of the compressed source 541 */ 542 unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) 543 { 544 unsigned long long bound = 0; 545 /* Iterate over each frame */ 546 while (srcSize > 0) { 547 ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); 548 size_t const compressedSize = frameSizeInfo.compressedSize; 549 unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; 550 if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) 551 return ZSTD_CONTENTSIZE_ERROR; 552 assert(srcSize >= compressedSize); 553 src = (const BYTE*)src + compressedSize; 554 srcSize -= compressedSize; 555 bound += decompressedBound; 556 } 557 return bound; 558 } 559 560 561 /*-************************************************************* 562 * Frame decoding 563 ***************************************************************/ 564 565 566 void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) 567 { 568 if (dst != dctx->previousDstEnd) { /* not contiguous */ 569 dctx->dictEnd = dctx->previousDstEnd; 570 dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); 571 dctx->prefixStart = dst; 572 dctx->previousDstEnd = dst; 573 } 574 } 575 576 /** ZSTD_insertBlock() : 577 insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ 578 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) 579 { 580 ZSTD_checkContinuity(dctx, blockStart); 581 dctx->previousDstEnd = (const char*)blockStart + blockSize; 582 return blockSize; 583 } 584 585 586 static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, 587 const void* src, size_t srcSize) 588 { 589 DEBUGLOG(5, "ZSTD_copyRawBlock"); 590 if (dst == NULL) { 591 if (srcSize == 0) return 0; 592 RETURN_ERROR(dstBuffer_null); 593 } 594 RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall); 595 memcpy(dst, src, srcSize); 596 return srcSize; 597 } 598 599 static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, 600 BYTE b, 601 size_t regenSize) 602 { 603 if (dst == NULL) { 604 if (regenSize == 0) return 0; 605 RETURN_ERROR(dstBuffer_null); 606 } 607 RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall); 608 memset(dst, b, regenSize); 609 return regenSize; 610 } 611 612 613 /*! ZSTD_decompressFrame() : 614 * @dctx must be properly initialized 615 * will update *srcPtr and *srcSizePtr, 616 * to make *srcPtr progress by one frame. */ 617 static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, 618 void* dst, size_t dstCapacity, 619 const void** srcPtr, size_t *srcSizePtr) 620 { 621 const BYTE* ip = (const BYTE*)(*srcPtr); 622 BYTE* const ostart = (BYTE* const)dst; 623 BYTE* const oend = ostart + dstCapacity; 624 BYTE* op = ostart; 625 size_t remainingSrcSize = *srcSizePtr; 626 627 DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); 628 629 /* check */ 630 RETURN_ERROR_IF( 631 remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize, 632 srcSize_wrong); 633 634 /* Frame Header */ 635 { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); 636 if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; 637 RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, 638 srcSize_wrong); 639 FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); 640 ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; 641 } 642 643 /* Loop on each block */ 644 while (1) { 645 size_t decodedSize; 646 blockProperties_t blockProperties; 647 size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); 648 if (ZSTD_isError(cBlockSize)) return cBlockSize; 649 650 ip += ZSTD_blockHeaderSize; 651 remainingSrcSize -= ZSTD_blockHeaderSize; 652 RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong); 653 654 switch(blockProperties.blockType) 655 { 656 case bt_compressed: 657 decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1); 658 break; 659 case bt_raw : 660 decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); 661 break; 662 case bt_rle : 663 decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize); 664 break; 665 case bt_reserved : 666 default: 667 RETURN_ERROR(corruption_detected); 668 } 669 670 if (ZSTD_isError(decodedSize)) return decodedSize; 671 if (dctx->fParams.checksumFlag) 672 XXH64_update(&dctx->xxhState, op, decodedSize); 673 op += decodedSize; 674 ip += cBlockSize; 675 remainingSrcSize -= cBlockSize; 676 if (blockProperties.lastBlock) break; 677 } 678 679 if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { 680 RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, 681 corruption_detected); 682 } 683 if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ 684 U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); 685 U32 checkRead; 686 RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong); 687 checkRead = MEM_readLE32(ip); 688 RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong); 689 ip += 4; 690 remainingSrcSize -= 4; 691 } 692 693 /* Allow caller to get size read */ 694 *srcPtr = ip; 695 *srcSizePtr = remainingSrcSize; 696 return op-ostart; 697 } 698 699 static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, 700 void* dst, size_t dstCapacity, 701 const void* src, size_t srcSize, 702 const void* dict, size_t dictSize, 703 const ZSTD_DDict* ddict) 704 { 705 void* const dststart = dst; 706 int moreThan1Frame = 0; 707 708 DEBUGLOG(5, "ZSTD_decompressMultiFrame"); 709 assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ 710 711 if (ddict) { 712 dict = ZSTD_DDict_dictContent(ddict); 713 dictSize = ZSTD_DDict_dictSize(ddict); 714 } 715 716 while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { 717 718 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) 719 if (ZSTD_isLegacy(src, srcSize)) { 720 size_t decodedSize; 721 size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); 722 if (ZSTD_isError(frameSize)) return frameSize; 723 RETURN_ERROR_IF(dctx->staticSize, memory_allocation, 724 "legacy support is not compatible with static dctx"); 725 726 decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); 727 if (ZSTD_isError(decodedSize)) return decodedSize; 728 729 assert(decodedSize <=- dstCapacity); 730 dst = (BYTE*)dst + decodedSize; 731 dstCapacity -= decodedSize; 732 733 src = (const BYTE*)src + frameSize; 734 srcSize -= frameSize; 735 736 continue; 737 } 738 #endif 739 740 { U32 const magicNumber = MEM_readLE32(src); 741 DEBUGLOG(4, "reading magic number %08X (expecting %08X)", 742 (unsigned)magicNumber, ZSTD_MAGICNUMBER); 743 if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 744 size_t const skippableSize = readSkippableFrameSize(src, srcSize); 745 FORWARD_IF_ERROR(skippableSize); 746 assert(skippableSize <= srcSize); 747 748 src = (const BYTE *)src + skippableSize; 749 srcSize -= skippableSize; 750 continue; 751 } } 752 753 if (ddict) { 754 /* we were called from ZSTD_decompress_usingDDict */ 755 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict)); 756 } else { 757 /* this will initialize correctly with no dict if dict == NULL, so 758 * use this in all cases but ddict */ 759 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); 760 } 761 ZSTD_checkContinuity(dctx, dst); 762 763 { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, 764 &src, &srcSize); 765 RETURN_ERROR_IF( 766 (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) 767 && (moreThan1Frame==1), 768 srcSize_wrong, 769 "at least one frame successfully completed, but following " 770 "bytes are garbage: it's more likely to be a srcSize error, " 771 "specifying more bytes than compressed size of frame(s). This " 772 "error message replaces ERROR(prefix_unknown), which would be " 773 "confusing, as the first header is actually correct. Note that " 774 "one could be unlucky, it might be a corruption error instead, " 775 "happening right at the place where we expect zstd magic " 776 "bytes. But this is _much_ less likely than a srcSize field " 777 "error."); 778 if (ZSTD_isError(res)) return res; 779 assert(res <= dstCapacity); 780 dst = (BYTE*)dst + res; 781 dstCapacity -= res; 782 } 783 moreThan1Frame = 1; 784 } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ 785 786 RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); 787 788 return (BYTE*)dst - (BYTE*)dststart; 789 } 790 791 size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, 792 void* dst, size_t dstCapacity, 793 const void* src, size_t srcSize, 794 const void* dict, size_t dictSize) 795 { 796 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); 797 } 798 799 800 static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) 801 { 802 switch (dctx->dictUses) { 803 default: 804 assert(0 /* Impossible */); 805 /* fall-through */ 806 case ZSTD_dont_use: 807 ZSTD_clearDict(dctx); 808 return NULL; 809 case ZSTD_use_indefinitely: 810 return dctx->ddict; 811 case ZSTD_use_once: 812 dctx->dictUses = ZSTD_dont_use; 813 return dctx->ddict; 814 } 815 } 816 817 size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 818 { 819 return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); 820 } 821 822 823 size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) 824 { 825 #if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) 826 size_t regenSize; 827 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 828 RETURN_ERROR_IF(dctx==NULL, memory_allocation); 829 regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); 830 ZSTD_freeDCtx(dctx); 831 return regenSize; 832 #else /* stack mode */ 833 ZSTD_DCtx dctx; 834 ZSTD_initDCtx_internal(&dctx); 835 return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); 836 #endif 837 } 838 839 840 /*-************************************** 841 * Advanced Streaming Decompression API 842 * Bufferless and synchronous 843 ****************************************/ 844 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } 845 846 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { 847 switch(dctx->stage) 848 { 849 default: /* should not happen */ 850 assert(0); 851 case ZSTDds_getFrameHeaderSize: 852 case ZSTDds_decodeFrameHeader: 853 return ZSTDnit_frameHeader; 854 case ZSTDds_decodeBlockHeader: 855 return ZSTDnit_blockHeader; 856 case ZSTDds_decompressBlock: 857 return ZSTDnit_block; 858 case ZSTDds_decompressLastBlock: 859 return ZSTDnit_lastBlock; 860 case ZSTDds_checkChecksum: 861 return ZSTDnit_checksum; 862 case ZSTDds_decodeSkippableHeader: 863 case ZSTDds_skipFrame: 864 return ZSTDnit_skippableFrame; 865 } 866 } 867 868 static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } 869 870 /** ZSTD_decompressContinue() : 871 * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) 872 * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) 873 * or an error code, which can be tested using ZSTD_isError() */ 874 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 875 { 876 DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); 877 /* Sanity check */ 878 RETURN_ERROR_IF(srcSize != dctx->expected, srcSize_wrong, "not allowed"); 879 if (dstCapacity) ZSTD_checkContinuity(dctx, dst); 880 881 switch (dctx->stage) 882 { 883 case ZSTDds_getFrameHeaderSize : 884 assert(src != NULL); 885 if (dctx->format == ZSTD_f_zstd1) { /* allows header */ 886 assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ 887 if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ 888 memcpy(dctx->headerBuffer, src, srcSize); 889 dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ 890 dctx->stage = ZSTDds_decodeSkippableHeader; 891 return 0; 892 } } 893 dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); 894 if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; 895 memcpy(dctx->headerBuffer, src, srcSize); 896 dctx->expected = dctx->headerSize - srcSize; 897 dctx->stage = ZSTDds_decodeFrameHeader; 898 return 0; 899 900 case ZSTDds_decodeFrameHeader: 901 assert(src != NULL); 902 memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); 903 FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); 904 dctx->expected = ZSTD_blockHeaderSize; 905 dctx->stage = ZSTDds_decodeBlockHeader; 906 return 0; 907 908 case ZSTDds_decodeBlockHeader: 909 { blockProperties_t bp; 910 size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); 911 if (ZSTD_isError(cBlockSize)) return cBlockSize; 912 RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); 913 dctx->expected = cBlockSize; 914 dctx->bType = bp.blockType; 915 dctx->rleSize = bp.origSize; 916 if (cBlockSize) { 917 dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; 918 return 0; 919 } 920 /* empty block */ 921 if (bp.lastBlock) { 922 if (dctx->fParams.checksumFlag) { 923 dctx->expected = 4; 924 dctx->stage = ZSTDds_checkChecksum; 925 } else { 926 dctx->expected = 0; /* end of frame */ 927 dctx->stage = ZSTDds_getFrameHeaderSize; 928 } 929 } else { 930 dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ 931 dctx->stage = ZSTDds_decodeBlockHeader; 932 } 933 return 0; 934 } 935 936 case ZSTDds_decompressLastBlock: 937 case ZSTDds_decompressBlock: 938 DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); 939 { size_t rSize; 940 switch(dctx->bType) 941 { 942 case bt_compressed: 943 DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); 944 rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); 945 break; 946 case bt_raw : 947 rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); 948 break; 949 case bt_rle : 950 rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); 951 break; 952 case bt_reserved : /* should never happen */ 953 default: 954 RETURN_ERROR(corruption_detected); 955 } 956 if (ZSTD_isError(rSize)) return rSize; 957 RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); 958 DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); 959 dctx->decodedSize += rSize; 960 if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); 961 962 if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ 963 DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); 964 RETURN_ERROR_IF( 965 dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN 966 && dctx->decodedSize != dctx->fParams.frameContentSize, 967 corruption_detected); 968 if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ 969 dctx->expected = 4; 970 dctx->stage = ZSTDds_checkChecksum; 971 } else { 972 dctx->expected = 0; /* ends here */ 973 dctx->stage = ZSTDds_getFrameHeaderSize; 974 } 975 } else { 976 dctx->stage = ZSTDds_decodeBlockHeader; 977 dctx->expected = ZSTD_blockHeaderSize; 978 dctx->previousDstEnd = (char*)dst + rSize; 979 } 980 return rSize; 981 } 982 983 case ZSTDds_checkChecksum: 984 assert(srcSize == 4); /* guaranteed by dctx->expected */ 985 { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); 986 U32 const check32 = MEM_readLE32(src); 987 DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); 988 RETURN_ERROR_IF(check32 != h32, checksum_wrong); 989 dctx->expected = 0; 990 dctx->stage = ZSTDds_getFrameHeaderSize; 991 return 0; 992 } 993 994 case ZSTDds_decodeSkippableHeader: 995 assert(src != NULL); 996 assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); 997 memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ 998 dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ 999 dctx->stage = ZSTDds_skipFrame; 1000 return 0; 1001 1002 case ZSTDds_skipFrame: 1003 dctx->expected = 0; 1004 dctx->stage = ZSTDds_getFrameHeaderSize; 1005 return 0; 1006 1007 default: 1008 assert(0); /* impossible */ 1009 RETURN_ERROR(GENERIC); /* some compiler require default to do something */ 1010 } 1011 } 1012 1013 1014 static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1015 { 1016 dctx->dictEnd = dctx->previousDstEnd; 1017 dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); 1018 dctx->prefixStart = dict; 1019 dctx->previousDstEnd = (const char*)dict + dictSize; 1020 return 0; 1021 } 1022 1023 /*! ZSTD_loadDEntropy() : 1024 * dict : must point at beginning of a valid zstd dictionary. 1025 * @return : size of entropy tables read */ 1026 size_t 1027 ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, 1028 const void* const dict, size_t const dictSize) 1029 { 1030 const BYTE* dictPtr = (const BYTE*)dict; 1031 const BYTE* const dictEnd = dictPtr + dictSize; 1032 1033 RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted); 1034 assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ 1035 dictPtr += 8; /* skip header = magic + dictID */ 1036 1037 ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); 1038 ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); 1039 ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); 1040 { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ 1041 size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); 1042 #ifdef HUF_FORCE_DECOMPRESS_X1 1043 /* in minimal huffman, we always use X1 variants */ 1044 size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, 1045 dictPtr, dictEnd - dictPtr, 1046 workspace, workspaceSize); 1047 #else 1048 size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, 1049 dictPtr, dictEnd - dictPtr, 1050 workspace, workspaceSize); 1051 #endif 1052 RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted); 1053 dictPtr += hSize; 1054 } 1055 1056 { short offcodeNCount[MaxOff+1]; 1057 unsigned offcodeMaxValue = MaxOff, offcodeLog; 1058 size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); 1059 RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted); 1060 RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted); 1061 RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted); 1062 ZSTD_buildFSETable( entropy->OFTable, 1063 offcodeNCount, offcodeMaxValue, 1064 OF_base, OF_bits, 1065 offcodeLog); 1066 dictPtr += offcodeHeaderSize; 1067 } 1068 1069 { short matchlengthNCount[MaxML+1]; 1070 unsigned matchlengthMaxValue = MaxML, matchlengthLog; 1071 size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); 1072 RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted); 1073 RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted); 1074 RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted); 1075 ZSTD_buildFSETable( entropy->MLTable, 1076 matchlengthNCount, matchlengthMaxValue, 1077 ML_base, ML_bits, 1078 matchlengthLog); 1079 dictPtr += matchlengthHeaderSize; 1080 } 1081 1082 { short litlengthNCount[MaxLL+1]; 1083 unsigned litlengthMaxValue = MaxLL, litlengthLog; 1084 size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); 1085 RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted); 1086 RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted); 1087 RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted); 1088 ZSTD_buildFSETable( entropy->LLTable, 1089 litlengthNCount, litlengthMaxValue, 1090 LL_base, LL_bits, 1091 litlengthLog); 1092 dictPtr += litlengthHeaderSize; 1093 } 1094 1095 RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted); 1096 { int i; 1097 size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); 1098 for (i=0; i<3; i++) { 1099 U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; 1100 RETURN_ERROR_IF(rep==0 || rep >= dictContentSize, 1101 dictionary_corrupted); 1102 entropy->rep[i] = rep; 1103 } } 1104 1105 return dictPtr - (const BYTE*)dict; 1106 } 1107 1108 static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1109 { 1110 if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); 1111 { U32 const magic = MEM_readLE32(dict); 1112 if (magic != ZSTD_MAGIC_DICTIONARY) { 1113 return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ 1114 } } 1115 dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); 1116 1117 /* load entropy tables */ 1118 { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); 1119 RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted); 1120 dict = (const char*)dict + eSize; 1121 dictSize -= eSize; 1122 } 1123 dctx->litEntropy = dctx->fseEntropy = 1; 1124 1125 /* reference dictionary content */ 1126 return ZSTD_refDictContent(dctx, dict, dictSize); 1127 } 1128 1129 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) 1130 { 1131 assert(dctx != NULL); 1132 dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ 1133 dctx->stage = ZSTDds_getFrameHeaderSize; 1134 dctx->decodedSize = 0; 1135 dctx->previousDstEnd = NULL; 1136 dctx->prefixStart = NULL; 1137 dctx->virtualStart = NULL; 1138 dctx->dictEnd = NULL; 1139 dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ 1140 dctx->litEntropy = dctx->fseEntropy = 0; 1141 dctx->dictID = 0; 1142 ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); 1143 memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ 1144 dctx->LLTptr = dctx->entropy.LLTable; 1145 dctx->MLTptr = dctx->entropy.MLTable; 1146 dctx->OFTptr = dctx->entropy.OFTable; 1147 dctx->HUFptr = dctx->entropy.hufTable; 1148 return 0; 1149 } 1150 1151 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1152 { 1153 FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); 1154 if (dict && dictSize) 1155 RETURN_ERROR_IF( 1156 ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), 1157 dictionary_corrupted); 1158 return 0; 1159 } 1160 1161 1162 /* ====== ZSTD_DDict ====== */ 1163 1164 size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 1165 { 1166 DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); 1167 assert(dctx != NULL); 1168 if (ddict) { 1169 const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); 1170 size_t const dictSize = ZSTD_DDict_dictSize(ddict); 1171 const void* const dictEnd = dictStart + dictSize; 1172 dctx->ddictIsCold = (dctx->dictEnd != dictEnd); 1173 DEBUGLOG(4, "DDict is %s", 1174 dctx->ddictIsCold ? "~cold~" : "hot!"); 1175 } 1176 FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) ); 1177 if (ddict) { /* NULL ddict is equivalent to no dictionary */ 1178 ZSTD_copyDDictParameters(dctx, ddict); 1179 } 1180 return 0; 1181 } 1182 1183 /*! ZSTD_getDictID_fromDict() : 1184 * Provides the dictID stored within dictionary. 1185 * if @return == 0, the dictionary is not conformant with Zstandard specification. 1186 * It can still be loaded, but as a content-only dictionary. */ 1187 unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) 1188 { 1189 if (dictSize < 8) return 0; 1190 if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; 1191 return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); 1192 } 1193 1194 /*! ZSTD_getDictID_fromFrame() : 1195 * Provides the dictID required to decompress frame stored within `src`. 1196 * If @return == 0, the dictID could not be decoded. 1197 * This could for one of the following reasons : 1198 * - The frame does not require a dictionary (most common case). 1199 * - The frame was built with dictID intentionally removed. 1200 * Needed dictionary is a hidden information. 1201 * Note : this use case also happens when using a non-conformant dictionary. 1202 * - `srcSize` is too small, and as a result, frame header could not be decoded. 1203 * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. 1204 * - This is not a Zstandard frame. 1205 * When identifying the exact failure cause, it's possible to use 1206 * ZSTD_getFrameHeader(), which will provide a more precise error code. */ 1207 unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) 1208 { 1209 ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; 1210 size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); 1211 if (ZSTD_isError(hError)) return 0; 1212 return zfp.dictID; 1213 } 1214 1215 1216 /*! ZSTD_decompress_usingDDict() : 1217 * Decompression using a pre-digested Dictionary 1218 * Use dictionary without significant overhead. */ 1219 size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, 1220 void* dst, size_t dstCapacity, 1221 const void* src, size_t srcSize, 1222 const ZSTD_DDict* ddict) 1223 { 1224 /* pass content and size in case legacy frames are encountered */ 1225 return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, 1226 NULL, 0, 1227 ddict); 1228 } 1229 1230 1231 /*===================================== 1232 * Streaming decompression 1233 *====================================*/ 1234 1235 ZSTD_DStream* ZSTD_createDStream(void) 1236 { 1237 DEBUGLOG(3, "ZSTD_createDStream"); 1238 return ZSTD_createDStream_advanced(ZSTD_defaultCMem); 1239 } 1240 1241 ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) 1242 { 1243 return ZSTD_initStaticDCtx(workspace, workspaceSize); 1244 } 1245 1246 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) 1247 { 1248 return ZSTD_createDCtx_advanced(customMem); 1249 } 1250 1251 size_t ZSTD_freeDStream(ZSTD_DStream* zds) 1252 { 1253 return ZSTD_freeDCtx(zds); 1254 } 1255 1256 1257 /* *** Initialization *** */ 1258 1259 size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } 1260 size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } 1261 1262 size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, 1263 const void* dict, size_t dictSize, 1264 ZSTD_dictLoadMethod_e dictLoadMethod, 1265 ZSTD_dictContentType_e dictContentType) 1266 { 1267 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); 1268 ZSTD_clearDict(dctx); 1269 if (dict && dictSize >= 8) { 1270 dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); 1271 RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); 1272 dctx->ddict = dctx->ddictLocal; 1273 dctx->dictUses = ZSTD_use_indefinitely; 1274 } 1275 return 0; 1276 } 1277 1278 size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1279 { 1280 return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); 1281 } 1282 1283 size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) 1284 { 1285 return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); 1286 } 1287 1288 size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) 1289 { 1290 FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType)); 1291 dctx->dictUses = ZSTD_use_once; 1292 return 0; 1293 } 1294 1295 size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) 1296 { 1297 return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); 1298 } 1299 1300 1301 /* ZSTD_initDStream_usingDict() : 1302 * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. 1303 * this function cannot fail */ 1304 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) 1305 { 1306 DEBUGLOG(4, "ZSTD_initDStream_usingDict"); 1307 FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) ); 1308 FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); 1309 return ZSTD_FRAMEHEADERSIZE_PREFIX; 1310 } 1311 1312 /* note : this variant can't fail */ 1313 size_t ZSTD_initDStream(ZSTD_DStream* zds) 1314 { 1315 DEBUGLOG(4, "ZSTD_initDStream"); 1316 return ZSTD_initDStream_usingDDict(zds, NULL); 1317 } 1318 1319 /* ZSTD_initDStream_usingDDict() : 1320 * ddict will just be referenced, and must outlive decompression session 1321 * this function cannot fail */ 1322 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) 1323 { 1324 FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) ); 1325 FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) ); 1326 return ZSTD_FRAMEHEADERSIZE_PREFIX; 1327 } 1328 1329 /* ZSTD_resetDStream() : 1330 * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. 1331 * this function cannot fail */ 1332 size_t ZSTD_resetDStream(ZSTD_DStream* dctx) 1333 { 1334 FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); 1335 return ZSTD_FRAMEHEADERSIZE_PREFIX; 1336 } 1337 1338 1339 size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) 1340 { 1341 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); 1342 ZSTD_clearDict(dctx); 1343 if (ddict) { 1344 dctx->ddict = ddict; 1345 dctx->dictUses = ZSTD_use_indefinitely; 1346 } 1347 return 0; 1348 } 1349 1350 /* ZSTD_DCtx_setMaxWindowSize() : 1351 * note : no direct equivalence in ZSTD_DCtx_setParameter, 1352 * since this version sets windowSize, and the other sets windowLog */ 1353 size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) 1354 { 1355 ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); 1356 size_t const min = (size_t)1 << bounds.lowerBound; 1357 size_t const max = (size_t)1 << bounds.upperBound; 1358 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); 1359 RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound); 1360 RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound); 1361 dctx->maxWindowSize = maxWindowSize; 1362 return 0; 1363 } 1364 1365 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) 1366 { 1367 return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format); 1368 } 1369 1370 ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) 1371 { 1372 ZSTD_bounds bounds = { 0, 0, 0 }; 1373 switch(dParam) { 1374 case ZSTD_d_windowLogMax: 1375 bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; 1376 bounds.upperBound = ZSTD_WINDOWLOG_MAX; 1377 return bounds; 1378 case ZSTD_d_format: 1379 bounds.lowerBound = (int)ZSTD_f_zstd1; 1380 bounds.upperBound = (int)ZSTD_f_zstd1_magicless; 1381 ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); 1382 return bounds; 1383 default:; 1384 } 1385 bounds.error = ERROR(parameter_unsupported); 1386 return bounds; 1387 } 1388 1389 /* ZSTD_dParam_withinBounds: 1390 * @return 1 if value is within dParam bounds, 1391 * 0 otherwise */ 1392 static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) 1393 { 1394 ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); 1395 if (ZSTD_isError(bounds.error)) return 0; 1396 if (value < bounds.lowerBound) return 0; 1397 if (value > bounds.upperBound) return 0; 1398 return 1; 1399 } 1400 1401 #define CHECK_DBOUNDS(p,v) { \ 1402 RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound); \ 1403 } 1404 1405 size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) 1406 { 1407 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); 1408 switch(dParam) { 1409 case ZSTD_d_windowLogMax: 1410 if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; 1411 CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); 1412 dctx->maxWindowSize = ((size_t)1) << value; 1413 return 0; 1414 case ZSTD_d_format: 1415 CHECK_DBOUNDS(ZSTD_d_format, value); 1416 dctx->format = (ZSTD_format_e)value; 1417 return 0; 1418 default:; 1419 } 1420 RETURN_ERROR(parameter_unsupported); 1421 } 1422 1423 size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) 1424 { 1425 if ( (reset == ZSTD_reset_session_only) 1426 || (reset == ZSTD_reset_session_and_parameters) ) { 1427 dctx->streamStage = zdss_init; 1428 dctx->noForwardProgress = 0; 1429 } 1430 if ( (reset == ZSTD_reset_parameters) 1431 || (reset == ZSTD_reset_session_and_parameters) ) { 1432 RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); 1433 ZSTD_clearDict(dctx); 1434 dctx->format = ZSTD_f_zstd1; 1435 dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; 1436 } 1437 return 0; 1438 } 1439 1440 1441 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) 1442 { 1443 return ZSTD_sizeof_DCtx(dctx); 1444 } 1445 1446 size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) 1447 { 1448 size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 1449 unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); 1450 unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); 1451 size_t const minRBSize = (size_t) neededSize; 1452 RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, 1453 frameParameter_windowTooLarge); 1454 return minRBSize; 1455 } 1456 1457 size_t ZSTD_estimateDStreamSize(size_t windowSize) 1458 { 1459 size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); 1460 size_t const inBuffSize = blockSize; /* no block can be larger */ 1461 size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); 1462 return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; 1463 } 1464 1465 size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) 1466 { 1467 U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ 1468 ZSTD_frameHeader zfh; 1469 size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); 1470 if (ZSTD_isError(err)) return err; 1471 RETURN_ERROR_IF(err>0, srcSize_wrong); 1472 RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, 1473 frameParameter_windowTooLarge); 1474 return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); 1475 } 1476 1477 1478 /* ***** Decompression ***** */ 1479 1480 MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) 1481 { 1482 size_t const length = MIN(dstCapacity, srcSize); 1483 memcpy(dst, src, length); 1484 return length; 1485 } 1486 1487 1488 size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) 1489 { 1490 const char* const istart = (const char*)(input->src) + input->pos; 1491 const char* const iend = (const char*)(input->src) + input->size; 1492 const char* ip = istart; 1493 char* const ostart = (char*)(output->dst) + output->pos; 1494 char* const oend = (char*)(output->dst) + output->size; 1495 char* op = ostart; 1496 U32 someMoreWork = 1; 1497 1498 DEBUGLOG(5, "ZSTD_decompressStream"); 1499 RETURN_ERROR_IF( 1500 input->pos > input->size, 1501 srcSize_wrong, 1502 "forbidden. in: pos: %u vs size: %u", 1503 (U32)input->pos, (U32)input->size); 1504 RETURN_ERROR_IF( 1505 output->pos > output->size, 1506 dstSize_tooSmall, 1507 "forbidden. out: pos: %u vs size: %u", 1508 (U32)output->pos, (U32)output->size); 1509 DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); 1510 1511 while (someMoreWork) { 1512 switch(zds->streamStage) 1513 { 1514 case zdss_init : 1515 DEBUGLOG(5, "stage zdss_init => transparent reset "); 1516 zds->streamStage = zdss_loadHeader; 1517 zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; 1518 zds->legacyVersion = 0; 1519 zds->hostageByte = 0; 1520 /* fall-through */ 1521 1522 case zdss_loadHeader : 1523 DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); 1524 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 1525 if (zds->legacyVersion) { 1526 RETURN_ERROR_IF(zds->staticSize, memory_allocation, 1527 "legacy support is incompatible with static dctx"); 1528 { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); 1529 if (hint==0) zds->streamStage = zdss_init; 1530 return hint; 1531 } } 1532 #endif 1533 { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); 1534 DEBUGLOG(5, "header size : %u", (U32)hSize); 1535 if (ZSTD_isError(hSize)) { 1536 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) 1537 U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); 1538 if (legacyVersion) { 1539 ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); 1540 const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; 1541 size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; 1542 DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); 1543 RETURN_ERROR_IF(zds->staticSize, memory_allocation, 1544 "legacy support is incompatible with static dctx"); 1545 FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, 1546 zds->previousLegacyVersion, legacyVersion, 1547 dict, dictSize)); 1548 zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; 1549 { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); 1550 if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ 1551 return hint; 1552 } } 1553 #endif 1554 return hSize; /* error */ 1555 } 1556 if (hSize != 0) { /* need more input */ 1557 size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ 1558 size_t const remainingInput = (size_t)(iend-ip); 1559 assert(iend >= ip); 1560 if (toLoad > remainingInput) { /* not enough input to load full header */ 1561 if (remainingInput > 0) { 1562 memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); 1563 zds->lhSize += remainingInput; 1564 } 1565 input->pos = input->size; 1566 return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ 1567 } 1568 assert(ip != NULL); 1569 memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; 1570 break; 1571 } } 1572 1573 /* check for single-pass mode opportunity */ 1574 if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ 1575 && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { 1576 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); 1577 if (cSize <= (size_t)(iend-istart)) { 1578 /* shortcut : using single-pass mode */ 1579 size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, ZSTD_getDDict(zds)); 1580 if (ZSTD_isError(decompressedSize)) return decompressedSize; 1581 DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") 1582 ip = istart + cSize; 1583 op += decompressedSize; 1584 zds->expected = 0; 1585 zds->streamStage = zdss_init; 1586 someMoreWork = 0; 1587 break; 1588 } } 1589 1590 /* Consume header (see ZSTDds_decodeFrameHeader) */ 1591 DEBUGLOG(4, "Consume header"); 1592 FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds))); 1593 1594 if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ 1595 zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); 1596 zds->stage = ZSTDds_skipFrame; 1597 } else { 1598 FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); 1599 zds->expected = ZSTD_blockHeaderSize; 1600 zds->stage = ZSTDds_decodeBlockHeader; 1601 } 1602 1603 /* control buffer memory usage */ 1604 DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", 1605 (U32)(zds->fParams.windowSize >>10), 1606 (U32)(zds->maxWindowSize >> 10) ); 1607 zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); 1608 RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, 1609 frameParameter_windowTooLarge); 1610 1611 /* Adapt buffer sizes to frame header instructions */ 1612 { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); 1613 size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); 1614 if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) { 1615 size_t const bufferSize = neededInBuffSize + neededOutBuffSize; 1616 DEBUGLOG(4, "inBuff : from %u to %u", 1617 (U32)zds->inBuffSize, (U32)neededInBuffSize); 1618 DEBUGLOG(4, "outBuff : from %u to %u", 1619 (U32)zds->outBuffSize, (U32)neededOutBuffSize); 1620 if (zds->staticSize) { /* static DCtx */ 1621 DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); 1622 assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ 1623 RETURN_ERROR_IF( 1624 bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), 1625 memory_allocation); 1626 } else { 1627 ZSTD_free(zds->inBuff, zds->customMem); 1628 zds->inBuffSize = 0; 1629 zds->outBuffSize = 0; 1630 zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); 1631 RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation); 1632 } 1633 zds->inBuffSize = neededInBuffSize; 1634 zds->outBuff = zds->inBuff + zds->inBuffSize; 1635 zds->outBuffSize = neededOutBuffSize; 1636 } } 1637 zds->streamStage = zdss_read; 1638 /* fall-through */ 1639 1640 case zdss_read: 1641 DEBUGLOG(5, "stage zdss_read"); 1642 { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); 1643 DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); 1644 if (neededInSize==0) { /* end of frame */ 1645 zds->streamStage = zdss_init; 1646 someMoreWork = 0; 1647 break; 1648 } 1649 if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ 1650 int const isSkipFrame = ZSTD_isSkipFrame(zds); 1651 size_t const decodedSize = ZSTD_decompressContinue(zds, 1652 zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), 1653 ip, neededInSize); 1654 if (ZSTD_isError(decodedSize)) return decodedSize; 1655 ip += neededInSize; 1656 if (!decodedSize && !isSkipFrame) break; /* this was just a header */ 1657 zds->outEnd = zds->outStart + decodedSize; 1658 zds->streamStage = zdss_flush; 1659 break; 1660 } } 1661 if (ip==iend) { someMoreWork = 0; break; } /* no more input */ 1662 zds->streamStage = zdss_load; 1663 /* fall-through */ 1664 1665 case zdss_load: 1666 { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); 1667 size_t const toLoad = neededInSize - zds->inPos; 1668 int const isSkipFrame = ZSTD_isSkipFrame(zds); 1669 size_t loadedSize; 1670 if (isSkipFrame) { 1671 loadedSize = MIN(toLoad, (size_t)(iend-ip)); 1672 } else { 1673 RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, 1674 corruption_detected, 1675 "should never happen"); 1676 loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); 1677 } 1678 ip += loadedSize; 1679 zds->inPos += loadedSize; 1680 if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ 1681 1682 /* decode loaded input */ 1683 { size_t const decodedSize = ZSTD_decompressContinue(zds, 1684 zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, 1685 zds->inBuff, neededInSize); 1686 if (ZSTD_isError(decodedSize)) return decodedSize; 1687 zds->inPos = 0; /* input is consumed */ 1688 if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */ 1689 zds->outEnd = zds->outStart + decodedSize; 1690 } } 1691 zds->streamStage = zdss_flush; 1692 /* fall-through */ 1693 1694 case zdss_flush: 1695 { size_t const toFlushSize = zds->outEnd - zds->outStart; 1696 size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); 1697 op += flushedSize; 1698 zds->outStart += flushedSize; 1699 if (flushedSize == toFlushSize) { /* flush completed */ 1700 zds->streamStage = zdss_read; 1701 if ( (zds->outBuffSize < zds->fParams.frameContentSize) 1702 && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { 1703 DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", 1704 (int)(zds->outBuffSize - zds->outStart), 1705 (U32)zds->fParams.blockSizeMax); 1706 zds->outStart = zds->outEnd = 0; 1707 } 1708 break; 1709 } } 1710 /* cannot complete flush */ 1711 someMoreWork = 0; 1712 break; 1713 1714 default: 1715 assert(0); /* impossible */ 1716 RETURN_ERROR(GENERIC); /* some compiler require default to do something */ 1717 } } 1718 1719 /* result */ 1720 input->pos = (size_t)(ip - (const char*)(input->src)); 1721 output->pos = (size_t)(op - (char*)(output->dst)); 1722 if ((ip==istart) && (op==ostart)) { /* no forward progress */ 1723 zds->noForwardProgress ++; 1724 if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { 1725 RETURN_ERROR_IF(op==oend, dstSize_tooSmall); 1726 RETURN_ERROR_IF(ip==iend, srcSize_wrong); 1727 assert(0); 1728 } 1729 } else { 1730 zds->noForwardProgress = 0; 1731 } 1732 { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); 1733 if (!nextSrcSizeHint) { /* frame fully decoded */ 1734 if (zds->outEnd == zds->outStart) { /* output fully flushed */ 1735 if (zds->hostageByte) { 1736 if (input->pos >= input->size) { 1737 /* can't release hostage (not present) */ 1738 zds->streamStage = zdss_read; 1739 return 1; 1740 } 1741 input->pos++; /* release hostage */ 1742 } /* zds->hostageByte */ 1743 return 0; 1744 } /* zds->outEnd == zds->outStart */ 1745 if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ 1746 input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ 1747 zds->hostageByte=1; 1748 } 1749 return 1; 1750 } /* nextSrcSizeHint==0 */ 1751 nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ 1752 assert(zds->inPos <= nextSrcSizeHint); 1753 nextSrcSizeHint -= zds->inPos; /* part already loaded*/ 1754 return nextSrcSizeHint; 1755 } 1756 } 1757 1758 size_t ZSTD_decompressStream_simpleArgs ( 1759 ZSTD_DCtx* dctx, 1760 void* dst, size_t dstCapacity, size_t* dstPos, 1761 const void* src, size_t srcSize, size_t* srcPos) 1762 { 1763 ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; 1764 ZSTD_inBuffer input = { src, srcSize, *srcPos }; 1765 /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ 1766 size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); 1767 *dstPos = output.pos; 1768 *srcPos = input.pos; 1769 return cErr; 1770 } 1771