1 /* 2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /* ************************************* 13 * Compiler Options 14 ***************************************/ 15 #ifdef _MSC_VER /* Visual */ 16 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 17 # pragma warning(disable : 4204) /* non-constant aggregate initializer */ 18 #endif 19 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) 20 # define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ 21 #endif 22 23 /*-************************************* 24 * Includes 25 ***************************************/ 26 #include "platform.h" /* Large Files support, SET_BINARY_MODE */ 27 #include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile */ 28 #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ 29 #include <stdlib.h> /* malloc, free */ 30 #include <string.h> /* strcmp, strlen */ 31 #include <assert.h> 32 #include <errno.h> /* errno */ 33 #include <signal.h> 34 35 #if defined (_MSC_VER) 36 # include <sys/stat.h> 37 # include <io.h> 38 #endif 39 40 #include "mem.h" /* U32, U64 */ 41 #include "fileio.h" 42 43 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ 44 #include "zstd.h" 45 #include "zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ 46 47 #if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) 48 # include <zlib.h> 49 # if !defined(z_const) 50 # define z_const 51 # endif 52 #endif 53 54 #if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) 55 # include <lzma.h> 56 #endif 57 58 #define LZ4_MAGICNUMBER 0x184D2204 59 #if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) 60 # define LZ4F_ENABLE_OBSOLETE_ENUMS 61 # include <lz4frame.h> 62 # include <lz4.h> 63 #endif 64 65 66 /*-************************************* 67 * Constants 68 ***************************************/ 69 #define KB *(1<<10) 70 #define MB *(1<<20) 71 #define GB *(1U<<30) 72 73 #define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ 74 #define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ 75 76 #define FNSPACE 30 77 78 79 /*-************************************* 80 * Macros 81 ***************************************/ 82 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 83 #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) 84 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } 85 static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ 86 void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } 87 88 static const U64 g_refreshRate = SEC_TO_MICRO / 6; 89 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 90 91 #define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) 92 #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } 93 #define DISPLAYUPDATE(l, ...) { \ 94 if (g_displayLevel>=l && !g_noProgress) { \ 95 if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \ 96 DELAY_NEXT_UPDATE(); \ 97 DISPLAY(__VA_ARGS__); \ 98 if (g_displayLevel>=4) fflush(stderr); \ 99 } } } 100 101 #undef MIN /* in case it would be already defined */ 102 #define MIN(a,b) ((a) < (b) ? (a) : (b)) 103 104 105 #define EXM_THROW(error, ...) \ 106 { \ 107 DISPLAYLEVEL(1, "zstd: "); \ 108 DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ 109 DISPLAYLEVEL(1, "error %i : ", error); \ 110 DISPLAYLEVEL(1, __VA_ARGS__); \ 111 DISPLAYLEVEL(1, " \n"); \ 112 exit(error); \ 113 } 114 115 #define CHECK_V(v, f) \ 116 v = f; \ 117 if (ZSTD_isError(v)) { \ 118 DISPLAYLEVEL(5, "%s \n", #f); \ 119 EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ 120 } 121 #define CHECK(f) { size_t err; CHECK_V(err, f); } 122 123 124 /*-************************************ 125 * Signal (Ctrl-C trapping) 126 **************************************/ 127 static const char* g_artefact = NULL; 128 static void INThandler(int sig) 129 { 130 assert(sig==SIGINT); (void)sig; 131 #if !defined(_MSC_VER) 132 signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ 133 #endif 134 if (g_artefact) { 135 assert(UTIL_isRegularFile(g_artefact)); 136 remove(g_artefact); 137 } 138 DISPLAY("\n"); 139 exit(2); 140 } 141 static void addHandler(char const* dstFileName) 142 { 143 if (UTIL_isRegularFile(dstFileName)) { 144 g_artefact = dstFileName; 145 signal(SIGINT, INThandler); 146 } else { 147 g_artefact = NULL; 148 } 149 } 150 /* Idempotent */ 151 static void clearHandler(void) 152 { 153 if (g_artefact) signal(SIGINT, SIG_DFL); 154 g_artefact = NULL; 155 } 156 157 158 /*-********************************************************* 159 * Termination signal trapping (Print debug stack trace) 160 ***********************************************************/ 161 #if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */ 162 # if (__has_feature(address_sanitizer)) 163 # define BACKTRACE_ENABLE 0 164 # endif /* __has_feature(address_sanitizer) */ 165 #elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */ 166 # define BACKTRACE_ENABLE 0 167 #endif 168 169 #if !defined(BACKTRACE_ENABLE) 170 /* automatic detector : backtrace enabled by default on linux+glibc and osx */ 171 # if (defined(__linux__) && defined(__GLIBC__)) \ 172 || (defined(__APPLE__) && defined(__MACH__)) 173 # define BACKTRACE_ENABLE 1 174 # else 175 # define BACKTRACE_ENABLE 0 176 # endif 177 #endif 178 179 /* note : after this point, BACKTRACE_ENABLE is necessarily defined */ 180 181 182 #if BACKTRACE_ENABLE 183 184 #include <execinfo.h> /* backtrace, backtrace_symbols */ 185 186 #define MAX_STACK_FRAMES 50 187 188 static void ABRThandler(int sig) { 189 const char* name; 190 void* addrlist[MAX_STACK_FRAMES]; 191 char** symbollist; 192 U32 addrlen, i; 193 194 switch (sig) { 195 case SIGABRT: name = "SIGABRT"; break; 196 case SIGFPE: name = "SIGFPE"; break; 197 case SIGILL: name = "SIGILL"; break; 198 case SIGINT: name = "SIGINT"; break; 199 case SIGSEGV: name = "SIGSEGV"; break; 200 default: name = "UNKNOWN"; 201 } 202 203 DISPLAY("Caught %s signal, printing stack:\n", name); 204 /* Retrieve current stack addresses. */ 205 addrlen = backtrace(addrlist, MAX_STACK_FRAMES); 206 if (addrlen == 0) { 207 DISPLAY("\n"); 208 return; 209 } 210 /* Create readable strings to each frame. */ 211 symbollist = backtrace_symbols(addrlist, addrlen); 212 /* Print the stack trace, excluding calls handling the signal. */ 213 for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) { 214 DISPLAY("%s\n", symbollist[i]); 215 } 216 free(symbollist); 217 /* Reset and raise the signal so default handler runs. */ 218 signal(sig, SIG_DFL); 219 raise(sig); 220 } 221 #endif 222 223 void FIO_addAbortHandler() 224 { 225 #if BACKTRACE_ENABLE 226 signal(SIGABRT, ABRThandler); 227 signal(SIGFPE, ABRThandler); 228 signal(SIGILL, ABRThandler); 229 signal(SIGSEGV, ABRThandler); 230 signal(SIGBUS, ABRThandler); 231 #endif 232 } 233 234 235 /*-************************************************************ 236 * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW 237 ***************************************************************/ 238 #if defined(_MSC_VER) && _MSC_VER >= 1400 239 # define LONG_SEEK _fseeki64 240 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 241 # define LONG_SEEK fseeko 242 #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) 243 # define LONG_SEEK fseeko64 244 #elif defined(_WIN32) && !defined(__DJGPP__) 245 # include <windows.h> 246 static int LONG_SEEK(FILE* file, __int64 offset, int origin) { 247 LARGE_INTEGER off; 248 DWORD method; 249 off.QuadPart = offset; 250 if (origin == SEEK_END) 251 method = FILE_END; 252 else if (origin == SEEK_CUR) 253 method = FILE_CURRENT; 254 else 255 method = FILE_BEGIN; 256 257 if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) 258 return 0; 259 else 260 return -1; 261 } 262 #else 263 # define LONG_SEEK fseek 264 #endif 265 266 267 /*-************************************* 268 * Local Parameters - Not thread safe 269 ***************************************/ 270 static FIO_compressionType_t g_compressionType = FIO_zstdCompression; 271 void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } 272 static U32 g_overwrite = 0; 273 void FIO_overwriteMode(void) { g_overwrite = 1; } 274 static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ 275 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; } 276 static U32 g_dictIDFlag = 1; 277 void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } 278 static U32 g_checksumFlag = 1; 279 void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } 280 static U32 g_removeSrcFile = 0; 281 void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } 282 static unsigned g_memLimit = 0; 283 void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } 284 static unsigned g_nbWorkers = 1; 285 void FIO_setNbWorkers(unsigned nbWorkers) { 286 #ifndef ZSTD_MULTITHREAD 287 if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); 288 #endif 289 g_nbWorkers = nbWorkers; 290 } 291 static U32 g_blockSize = 0; 292 void FIO_setBlockSize(unsigned blockSize) { 293 if (blockSize && g_nbWorkers==0) 294 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); 295 g_blockSize = blockSize; 296 } 297 #define FIO_OVERLAP_LOG_NOTSET 9999 298 static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET; 299 void FIO_setOverlapLog(unsigned overlapLog){ 300 if (overlapLog && g_nbWorkers==0) 301 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); 302 g_overlapLog = overlapLog; 303 } 304 static U32 g_adaptiveMode = 0; 305 void FIO_setAdaptiveMode(unsigned adapt) { 306 if ((adapt>0) && (g_nbWorkers==0)) 307 EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); 308 g_adaptiveMode = adapt; 309 } 310 static U32 g_rsyncable = 0; 311 void FIO_setRsyncable(unsigned rsyncable) { 312 if ((rsyncable>0) && (g_nbWorkers==0)) 313 EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); 314 g_rsyncable = rsyncable; 315 } 316 static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ 317 void FIO_setAdaptMin(int minCLevel) 318 { 319 #ifndef ZSTD_NOCOMPRESS 320 assert(minCLevel >= ZSTD_minCLevel()); 321 #endif 322 g_minAdaptLevel = minCLevel; 323 } 324 static int g_maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ 325 void FIO_setAdaptMax(int maxCLevel) 326 { 327 g_maxAdaptLevel = maxCLevel; 328 } 329 330 static U32 g_ldmFlag = 0; 331 void FIO_setLdmFlag(unsigned ldmFlag) { 332 g_ldmFlag = (ldmFlag>0); 333 } 334 static U32 g_ldmHashLog = 0; 335 void FIO_setLdmHashLog(unsigned ldmHashLog) { 336 g_ldmHashLog = ldmHashLog; 337 } 338 static U32 g_ldmMinMatch = 0; 339 void FIO_setLdmMinMatch(unsigned ldmMinMatch) { 340 g_ldmMinMatch = ldmMinMatch; 341 } 342 343 #define FIO_LDM_PARAM_NOTSET 9999 344 static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; 345 void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) { 346 g_ldmBucketSizeLog = ldmBucketSizeLog; 347 } 348 349 static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET; 350 void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) { 351 g_ldmHashRateLog = ldmHashRateLog; 352 } 353 static U32 g_noProgress = 0; 354 void FIO_setNoProgress(unsigned noProgress) { 355 g_noProgress = noProgress; 356 } 357 358 359 360 /*-************************************* 361 * Functions 362 ***************************************/ 363 /** FIO_remove() : 364 * @result : Unlink `fileName`, even if it's read-only */ 365 static int FIO_remove(const char* path) 366 { 367 if (!UTIL_isRegularFile(path)) { 368 DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s \n", path); 369 return 0; 370 } 371 #if defined(_WIN32) || defined(WIN32) 372 /* windows doesn't allow remove read-only files, 373 * so try to make it writable first */ 374 chmod(path, _S_IWRITE); 375 #endif 376 return remove(path); 377 } 378 379 /** FIO_openSrcFile() : 380 * condition : `srcFileName` must be non-NULL. 381 * @result : FILE* to `srcFileName`, or NULL if it fails */ 382 static FILE* FIO_openSrcFile(const char* srcFileName) 383 { 384 assert(srcFileName != NULL); 385 if (!strcmp (srcFileName, stdinmark)) { 386 DISPLAYLEVEL(4,"Using stdin for input \n"); 387 SET_BINARY_MODE(stdin); 388 return stdin; 389 } 390 391 if (!UTIL_fileExist(srcFileName)) { 392 DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", 393 srcFileName, strerror(errno)); 394 return NULL; 395 } 396 397 if (!UTIL_isRegularFile(srcFileName)) { 398 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", 399 srcFileName); 400 return NULL; 401 } 402 403 { FILE* const f = fopen(srcFileName, "rb"); 404 if (f == NULL) 405 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 406 return f; 407 } 408 } 409 410 /** FIO_openDstFile() : 411 * condition : `dstFileName` must be non-NULL. 412 * @result : FILE* to `dstFileName`, or NULL if it fails */ 413 static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName) 414 { 415 assert(dstFileName != NULL); 416 if (!strcmp (dstFileName, stdoutmark)) { 417 DISPLAYLEVEL(4,"Using stdout for output \n"); 418 SET_BINARY_MODE(stdout); 419 if (g_sparseFileSupport == 1) { 420 g_sparseFileSupport = 0; 421 DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); 422 } 423 return stdout; 424 } 425 426 /* ensure dst is not the same file as src */ 427 if (srcFileName != NULL) { 428 #ifdef _MSC_VER 429 /* note : Visual does not support file identification by inode. 430 * The following work-around is limited to detecting exact name repetition only, 431 * aka `filename` is considered different from `subdir/../filename` */ 432 if (!strcmp(srcFileName, dstFileName)) { 433 DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); 434 return NULL; 435 } 436 #else 437 stat_t srcStat; 438 stat_t dstStat; 439 if (UTIL_getFileStat(srcFileName, &srcStat) 440 && UTIL_getFileStat(dstFileName, &dstStat)) { 441 if (srcStat.st_dev == dstStat.st_dev 442 && srcStat.st_ino == dstStat.st_ino) { 443 DISPLAYLEVEL(1, "zstd: Refusing to open a output file which will overwrite the input file \n"); 444 return NULL; 445 } 446 } 447 #endif 448 } 449 450 if (g_sparseFileSupport == 1) { 451 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; 452 } 453 454 if (UTIL_isRegularFile(dstFileName)) { 455 /* Check if destination file already exists */ 456 FILE* const fCheck = fopen( dstFileName, "rb" ); 457 if (!strcmp(dstFileName, nulmark)) { 458 EXM_THROW(40, "%s is unexpectedly categorized as a regular file", 459 dstFileName); 460 } 461 if (fCheck != NULL) { /* dst file exists, authorization prompt */ 462 fclose(fCheck); 463 if (!g_overwrite) { 464 if (g_displayLevel <= 1) { 465 /* No interaction possible */ 466 DISPLAY("zstd: %s already exists; not overwritten \n", 467 dstFileName); 468 return NULL; 469 } 470 DISPLAY("zstd: %s already exists; overwrite (y/N) ? ", 471 dstFileName); 472 { int ch = getchar(); 473 if ((ch!='Y') && (ch!='y')) { 474 DISPLAY(" not overwritten \n"); 475 return NULL; 476 } 477 /* flush rest of input line */ 478 while ((ch!=EOF) && (ch!='\n')) ch = getchar(); 479 } } 480 /* need to unlink */ 481 FIO_remove(dstFileName); 482 } } 483 484 { FILE* const f = fopen( dstFileName, "wb" ); 485 if (f == NULL) 486 DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); 487 return f; 488 } 489 } 490 491 492 /*! FIO_createDictBuffer() : 493 * creates a buffer, pointed by `*bufferPtr`, 494 * loads `filename` content into it, up to DICTSIZE_MAX bytes. 495 * @return : loaded size 496 * if fileName==NULL, returns 0 and a NULL pointer 497 */ 498 static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName) 499 { 500 FILE* fileHandle; 501 U64 fileSize; 502 503 assert(bufferPtr != NULL); 504 *bufferPtr = NULL; 505 if (fileName == NULL) return 0; 506 507 DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); 508 fileHandle = fopen(fileName, "rb"); 509 if (fileHandle==NULL) EXM_THROW(31, "%s: %s", fileName, strerror(errno)); 510 511 fileSize = UTIL_getFileSize(fileName); 512 if (fileSize > DICTSIZE_MAX) { 513 EXM_THROW(32, "Dictionary file %s is too large (> %u MB)", 514 fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */ 515 } 516 *bufferPtr = malloc((size_t)fileSize); 517 if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); 518 { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); 519 if (readSize != fileSize) 520 EXM_THROW(35, "Error reading dictionary file %s : %s", 521 fileName, strerror(errno)); 522 } 523 fclose(fileHandle); 524 return (size_t)fileSize; 525 } 526 527 #ifndef ZSTD_NOCOMPRESS 528 529 /* ********************************************************************** 530 * Compression 531 ************************************************************************/ 532 typedef struct { 533 FILE* srcFile; 534 FILE* dstFile; 535 void* srcBuffer; 536 size_t srcBufferSize; 537 void* dstBuffer; 538 size_t dstBufferSize; 539 ZSTD_CStream* cctx; 540 } cRess_t; 541 542 static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, 543 U64 srcSize, 544 ZSTD_compressionParameters comprParams) { 545 cRess_t ress; 546 memset(&ress, 0, sizeof(ress)); 547 548 DISPLAYLEVEL(6, "FIO_createCResources \n"); 549 ress.cctx = ZSTD_createCCtx(); 550 if (ress.cctx == NULL) 551 EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", 552 strerror(errno)); 553 ress.srcBufferSize = ZSTD_CStreamInSize(); 554 ress.srcBuffer = malloc(ress.srcBufferSize); 555 ress.dstBufferSize = ZSTD_CStreamOutSize(); 556 ress.dstBuffer = malloc(ress.dstBufferSize); 557 if (!ress.srcBuffer || !ress.dstBuffer) 558 EXM_THROW(31, "allocation error : not enough memory"); 559 560 /* Advanced parameters, including dictionary */ 561 { void* dictBuffer; 562 size_t const dictBuffSize = FIO_createDictBuffer(&dictBuffer, dictFileName); /* works with dictFileName==NULL */ 563 if (dictFileName && (dictBuffer==NULL)) 564 EXM_THROW(32, "allocation error : can't create dictBuffer"); 565 566 if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog) 567 comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; 568 569 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */ 570 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) ); 571 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) ); 572 /* compression level */ 573 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); 574 /* long distance matching */ 575 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) ); 576 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) ); 577 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) ); 578 if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { 579 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) ); 580 } 581 if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { 582 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) ); 583 } 584 /* compression parameters */ 585 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) ); 586 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, comprParams.chainLog) ); 587 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, comprParams.hashLog) ); 588 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, comprParams.searchLog) ); 589 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, comprParams.minMatch) ); 590 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, comprParams.targetLength) ); 591 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); 592 /* multi-threading */ 593 #ifdef ZSTD_MULTITHREAD 594 DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers); 595 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) ); 596 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) ); 597 if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) { 598 DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog); 599 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) ); 600 } 601 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) ); 602 #endif 603 /* dictionary */ 604 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */ 605 CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); 606 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */ 607 608 free(dictBuffer); 609 } 610 611 return ress; 612 } 613 614 static void FIO_freeCResources(cRess_t ress) 615 { 616 free(ress.srcBuffer); 617 free(ress.dstBuffer); 618 ZSTD_freeCStream(ress.cctx); /* never fails */ 619 } 620 621 622 #ifdef ZSTD_GZCOMPRESS 623 static unsigned long long 624 FIO_compressGzFrame(cRess_t* ress, 625 const char* srcFileName, U64 const srcFileSize, 626 int compressionLevel, U64* readsize) 627 { 628 unsigned long long inFileSize = 0, outFileSize = 0; 629 z_stream strm; 630 int ret; 631 632 if (compressionLevel > Z_BEST_COMPRESSION) 633 compressionLevel = Z_BEST_COMPRESSION; 634 635 strm.zalloc = Z_NULL; 636 strm.zfree = Z_NULL; 637 strm.opaque = Z_NULL; 638 639 ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 640 15 /* maxWindowLogSize */ + 16 /* gzip only */, 641 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ 642 if (ret != Z_OK) 643 EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); 644 645 strm.next_in = 0; 646 strm.avail_in = 0; 647 strm.next_out = (Bytef*)ress->dstBuffer; 648 strm.avail_out = (uInt)ress->dstBufferSize; 649 650 while (1) { 651 if (strm.avail_in == 0) { 652 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 653 if (inSize == 0) break; 654 inFileSize += inSize; 655 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 656 strm.avail_in = (uInt)inSize; 657 } 658 ret = deflate(&strm, Z_NO_FLUSH); 659 if (ret != Z_OK) 660 EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); 661 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 662 if (decompBytes) { 663 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 664 EXM_THROW(73, "Write error : cannot write to output file"); 665 outFileSize += decompBytes; 666 strm.next_out = (Bytef*)ress->dstBuffer; 667 strm.avail_out = (uInt)ress->dstBufferSize; 668 } 669 } 670 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 671 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 672 (unsigned)(inFileSize>>20), 673 (double)outFileSize/inFileSize*100) 674 else 675 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 676 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 677 (double)outFileSize/inFileSize*100); 678 } 679 680 while (1) { 681 ret = deflate(&strm, Z_FINISH); 682 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 683 if (decompBytes) { 684 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) 685 EXM_THROW(75, "Write error : %s", strerror(errno)); 686 outFileSize += decompBytes; 687 strm.next_out = (Bytef*)ress->dstBuffer; 688 strm.avail_out = (uInt)ress->dstBufferSize; 689 } } 690 if (ret == Z_STREAM_END) break; 691 if (ret != Z_BUF_ERROR) 692 EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); 693 } 694 695 ret = deflateEnd(&strm); 696 if (ret != Z_OK) 697 EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); 698 *readsize = inFileSize; 699 700 return outFileSize; 701 } 702 #endif 703 704 705 #ifdef ZSTD_LZMACOMPRESS 706 static unsigned long long 707 FIO_compressLzmaFrame(cRess_t* ress, 708 const char* srcFileName, U64 const srcFileSize, 709 int compressionLevel, U64* readsize, int plain_lzma) 710 { 711 unsigned long long inFileSize = 0, outFileSize = 0; 712 lzma_stream strm = LZMA_STREAM_INIT; 713 lzma_action action = LZMA_RUN; 714 lzma_ret ret; 715 716 if (compressionLevel < 0) compressionLevel = 0; 717 if (compressionLevel > 9) compressionLevel = 9; 718 719 if (plain_lzma) { 720 lzma_options_lzma opt_lzma; 721 if (lzma_lzma_preset(&opt_lzma, compressionLevel)) 722 EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); 723 ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ 724 if (ret != LZMA_OK) 725 EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); 726 } else { 727 ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ 728 if (ret != LZMA_OK) 729 EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); 730 } 731 732 strm.next_in = 0; 733 strm.avail_in = 0; 734 strm.next_out = (BYTE*)ress->dstBuffer; 735 strm.avail_out = ress->dstBufferSize; 736 737 while (1) { 738 if (strm.avail_in == 0) { 739 size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); 740 if (inSize == 0) action = LZMA_FINISH; 741 inFileSize += inSize; 742 strm.next_in = (BYTE const*)ress->srcBuffer; 743 strm.avail_in = inSize; 744 } 745 746 ret = lzma_code(&strm, action); 747 748 if (ret != LZMA_OK && ret != LZMA_STREAM_END) 749 EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); 750 { size_t const compBytes = ress->dstBufferSize - strm.avail_out; 751 if (compBytes) { 752 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) 753 EXM_THROW(73, "Write error : %s", strerror(errno)); 754 outFileSize += compBytes; 755 strm.next_out = (BYTE*)ress->dstBuffer; 756 strm.avail_out = ress->dstBufferSize; 757 } } 758 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) 759 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 760 (unsigned)(inFileSize>>20), 761 (double)outFileSize/inFileSize*100) 762 else 763 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 764 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 765 (double)outFileSize/inFileSize*100); 766 if (ret == LZMA_STREAM_END) break; 767 } 768 769 lzma_end(&strm); 770 *readsize = inFileSize; 771 772 return outFileSize; 773 } 774 #endif 775 776 #ifdef ZSTD_LZ4COMPRESS 777 #if LZ4_VERSION_NUMBER <= 10600 778 #define LZ4F_blockLinked blockLinked 779 #define LZ4F_max64KB max64KB 780 #endif 781 static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } 782 static unsigned long long 783 FIO_compressLz4Frame(cRess_t* ress, 784 const char* srcFileName, U64 const srcFileSize, 785 int compressionLevel, U64* readsize) 786 { 787 const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); 788 unsigned long long inFileSize = 0, outFileSize = 0; 789 790 LZ4F_preferences_t prefs; 791 LZ4F_compressionContext_t ctx; 792 793 LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 794 if (LZ4F_isError(errorCode)) 795 EXM_THROW(31, "zstd: failed to create lz4 compression context"); 796 797 memset(&prefs, 0, sizeof(prefs)); 798 799 assert(blockSize <= ress->srcBufferSize); 800 801 prefs.autoFlush = 1; 802 prefs.compressionLevel = compressionLevel; 803 prefs.frameInfo.blockMode = LZ4F_blockLinked; 804 prefs.frameInfo.blockSizeID = LZ4F_max64KB; 805 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)g_checksumFlag; 806 #if LZ4_VERSION_NUMBER >= 10600 807 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; 808 #endif 809 assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); 810 811 { 812 size_t readSize; 813 size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); 814 if (LZ4F_isError(headerSize)) 815 EXM_THROW(33, "File header generation failed : %s", 816 LZ4F_getErrorName(headerSize)); 817 if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) 818 EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); 819 outFileSize += headerSize; 820 821 /* Read first block */ 822 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 823 inFileSize += readSize; 824 825 /* Main Loop */ 826 while (readSize>0) { 827 size_t const outSize = LZ4F_compressUpdate(ctx, 828 ress->dstBuffer, ress->dstBufferSize, 829 ress->srcBuffer, readSize, NULL); 830 if (LZ4F_isError(outSize)) 831 EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", 832 srcFileName, LZ4F_getErrorName(outSize)); 833 outFileSize += outSize; 834 if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { 835 DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", 836 (unsigned)(inFileSize>>20), 837 (double)outFileSize/inFileSize*100) 838 } else { 839 DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", 840 (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), 841 (double)outFileSize/inFileSize*100); 842 } 843 844 /* Write Block */ 845 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); 846 if (sizeCheck != outSize) 847 EXM_THROW(36, "Write error : %s", strerror(errno)); 848 } 849 850 /* Read next block */ 851 readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); 852 inFileSize += readSize; 853 } 854 if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); 855 856 /* End of Stream mark */ 857 headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); 858 if (LZ4F_isError(headerSize)) 859 EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", 860 srcFileName, LZ4F_getErrorName(headerSize)); 861 862 { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); 863 if (sizeCheck != headerSize) 864 EXM_THROW(39, "Write error : %s (cannot write end of stream)", 865 strerror(errno)); 866 } 867 outFileSize += headerSize; 868 } 869 870 *readsize = inFileSize; 871 LZ4F_freeCompressionContext(ctx); 872 873 return outFileSize; 874 } 875 #endif 876 877 878 static unsigned long long 879 FIO_compressZstdFrame(const cRess_t* ressPtr, 880 const char* srcFileName, U64 fileSize, 881 int compressionLevel, U64* readsize) 882 { 883 cRess_t const ress = *ressPtr; 884 FILE* const srcFile = ress.srcFile; 885 FILE* const dstFile = ress.dstFile; 886 U64 compressedfilesize = 0; 887 ZSTD_EndDirective directive = ZSTD_e_continue; 888 889 /* stats */ 890 ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 }; 891 ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 }; 892 typedef enum { noChange, slower, faster } speedChange_e; 893 speedChange_e speedChange = noChange; 894 unsigned flushWaiting = 0; 895 unsigned inputPresented = 0; 896 unsigned inputBlocked = 0; 897 unsigned lastJobID = 0; 898 899 DISPLAYLEVEL(6, "compression using zstd format \n"); 900 901 /* init */ 902 if (fileSize != UTIL_FILESIZE_UNKNOWN) { 903 CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); 904 } 905 (void)srcFileName; 906 907 /* Main compression loop */ 908 do { 909 size_t stillToFlush; 910 /* Fill input Buffer */ 911 size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); 912 ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; 913 DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); 914 *readsize += inSize; 915 916 if ((inSize == 0) || (*readsize == fileSize)) 917 directive = ZSTD_e_end; 918 919 stillToFlush = 1; 920 while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ 921 || (directive == ZSTD_e_end && stillToFlush != 0) ) { 922 923 size_t const oldIPos = inBuff.pos; 924 ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; 925 size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); 926 CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); 927 928 /* count stats */ 929 inputPresented++; 930 if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */ 931 if (!toFlushNow) flushWaiting = 1; 932 933 /* Write compressed stream */ 934 DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", 935 (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); 936 if (outBuff.pos) { 937 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); 938 if (sizeCheck != outBuff.pos) 939 EXM_THROW(25, "Write error : %s (cannot write compressed block)", 940 strerror(errno)); 941 compressedfilesize += outBuff.pos; 942 } 943 944 /* display notification; and adapt compression level */ 945 if (READY_FOR_UPDATE()) { 946 ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); 947 double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; 948 949 /* display progress notifications */ 950 if (g_displayLevel >= 3) { 951 DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", 952 compressionLevel, 953 (unsigned)((zfp.ingested - zfp.consumed) >> 20), 954 (unsigned)(zfp.consumed >> 20), 955 (unsigned)(zfp.produced >> 20), 956 cShare ); 957 } else { /* summarized notifications if == 2; */ 958 DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20)); 959 if (fileSize != UTIL_FILESIZE_UNKNOWN) 960 DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); 961 DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); 962 DELAY_NEXT_UPDATE(); 963 } 964 965 /* adaptive mode : statistics measurement and speed correction */ 966 if (g_adaptiveMode) { 967 968 /* check output speed */ 969 if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ 970 971 unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; 972 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; 973 assert(zfp.produced >= previous_zfp_update.produced); 974 assert(g_nbWorkers >= 1); 975 976 /* test if compression is blocked 977 * either because output is slow and all buffers are full 978 * or because input is slow and no job can start while waiting for at least one buffer to be filled. 979 * note : excluse starting part, since currentJobID > 1 */ 980 if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ 981 && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ 982 ) { 983 DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") 984 speedChange = slower; 985 } 986 987 previous_zfp_update = zfp; 988 989 if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ 990 && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ 991 ) { 992 DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed); 993 speedChange = slower; 994 } 995 flushWaiting = 0; 996 } 997 998 /* course correct only if there is at least one new job completed */ 999 if (zfp.currentJobID > lastJobID) { 1000 DISPLAYLEVEL(6, "compression level adaptation check \n") 1001 1002 /* check input speed */ 1003 if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ 1004 if (inputBlocked <= 0) { 1005 DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); 1006 speedChange = slower; 1007 } else if (speedChange == noChange) { 1008 unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; 1009 unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; 1010 unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; 1011 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; 1012 previous_zfp_correction = zfp; 1013 assert(inputPresented > 0); 1014 DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", 1015 inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, 1016 (unsigned)newlyIngested, (unsigned)newlyConsumed, 1017 (unsigned)newlyFlushed, (unsigned)newlyProduced); 1018 if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ 1019 && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ 1020 && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ 1021 ) { 1022 DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", 1023 newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); 1024 speedChange = faster; 1025 } 1026 } 1027 inputBlocked = 0; 1028 inputPresented = 0; 1029 } 1030 1031 if (speedChange == slower) { 1032 DISPLAYLEVEL(6, "slower speed , higher compression \n") 1033 compressionLevel ++; 1034 if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); 1035 if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel; 1036 compressionLevel += (compressionLevel == 0); /* skip 0 */ 1037 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1038 } 1039 if (speedChange == faster) { 1040 DISPLAYLEVEL(6, "faster speed , lighter compression \n") 1041 compressionLevel --; 1042 if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel; 1043 compressionLevel -= (compressionLevel == 0); /* skip 0 */ 1044 ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); 1045 } 1046 speedChange = noChange; 1047 1048 lastJobID = zfp.currentJobID; 1049 } /* if (zfp.currentJobID > lastJobID) */ 1050 } /* if (g_adaptiveMode) */ 1051 } /* if (READY_FOR_UPDATE()) */ 1052 } /* while ((inBuff.pos != inBuff.size) */ 1053 } while (directive != ZSTD_e_end); 1054 1055 if (ferror(srcFile)) { 1056 EXM_THROW(26, "Read error : I/O error"); 1057 } 1058 if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { 1059 EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", 1060 (unsigned long long)*readsize, (unsigned long long)fileSize); 1061 } 1062 1063 return compressedfilesize; 1064 } 1065 1066 /*! FIO_compressFilename_internal() : 1067 * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. 1068 * @return : 0 : compression completed correctly, 1069 * 1 : missing or pb opening srcFileName 1070 */ 1071 static int 1072 FIO_compressFilename_internal(cRess_t ress, 1073 const char* dstFileName, const char* srcFileName, 1074 int compressionLevel) 1075 { 1076 U64 readsize = 0; 1077 U64 compressedfilesize = 0; 1078 U64 const fileSize = UTIL_getFileSize(srcFileName); 1079 DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize); 1080 1081 /* compression format selection */ 1082 switch (g_compressionType) { 1083 default: 1084 case FIO_zstdCompression: 1085 compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 1086 break; 1087 1088 case FIO_gzipCompression: 1089 #ifdef ZSTD_GZCOMPRESS 1090 compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 1091 #else 1092 (void)compressionLevel; 1093 EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", 1094 srcFileName); 1095 #endif 1096 break; 1097 1098 case FIO_xzCompression: 1099 case FIO_lzmaCompression: 1100 #ifdef ZSTD_LZMACOMPRESS 1101 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); 1102 #else 1103 (void)compressionLevel; 1104 EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", 1105 srcFileName); 1106 #endif 1107 break; 1108 1109 case FIO_lz4Compression: 1110 #ifdef ZSTD_LZ4COMPRESS 1111 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, &readsize); 1112 #else 1113 (void)compressionLevel; 1114 EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", 1115 srcFileName); 1116 #endif 1117 break; 1118 } 1119 1120 /* Status */ 1121 DISPLAYLEVEL(2, "\r%79s\r", ""); 1122 DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", 1123 srcFileName, 1124 (double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100, 1125 (unsigned long long)readsize, (unsigned long long) compressedfilesize, 1126 dstFileName); 1127 1128 return 0; 1129 } 1130 1131 1132 /*! FIO_compressFilename_dstFile() : 1133 * open dstFileName, or pass-through if ress.dstFile != NULL, 1134 * then start compression with FIO_compressFilename_internal(). 1135 * Manages source removal (--rm) and file permissions transfer. 1136 * note : ress.srcFile must be != NULL, 1137 * so reach this function through FIO_compressFilename_srcFile(). 1138 * @return : 0 : compression completed correctly, 1139 * 1 : pb 1140 */ 1141 static int FIO_compressFilename_dstFile(cRess_t ress, 1142 const char* dstFileName, 1143 const char* srcFileName, 1144 int compressionLevel) 1145 { 1146 int closeDstFile = 0; 1147 int result; 1148 stat_t statbuf; 1149 int transfer_permissions = 0; 1150 1151 assert(ress.srcFile != NULL); 1152 1153 if (ress.dstFile == NULL) { 1154 closeDstFile = 1; 1155 DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); 1156 ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); 1157 if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ 1158 /* Must only be added after FIO_openDstFile() succeeds. 1159 * Otherwise we may delete the destination file if it already exists, 1160 * and the user presses Ctrl-C when asked if they wish to overwrite. 1161 */ 1162 addHandler(dstFileName); 1163 1164 if ( strcmp (srcFileName, stdinmark) 1165 && UTIL_getFileStat(srcFileName, &statbuf)) 1166 transfer_permissions = 1; 1167 } 1168 1169 result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); 1170 1171 if (closeDstFile) { 1172 FILE* const dstFile = ress.dstFile; 1173 ress.dstFile = NULL; 1174 1175 clearHandler(); 1176 1177 if (fclose(dstFile)) { /* error closing dstFile */ 1178 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1179 result=1; 1180 } 1181 if ( (result != 0) /* operation failure */ 1182 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null */ 1183 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 1184 ) { 1185 FIO_remove(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ 1186 } else if ( strcmp(dstFileName, stdoutmark) 1187 && strcmp(dstFileName, nulmark) 1188 && transfer_permissions) { 1189 UTIL_setFileStat(dstFileName, &statbuf); 1190 } 1191 } 1192 1193 return result; 1194 } 1195 1196 1197 /*! FIO_compressFilename_srcFile() : 1198 * @return : 0 : compression completed correctly, 1199 * 1 : missing or pb opening srcFileName 1200 */ 1201 static int 1202 FIO_compressFilename_srcFile(cRess_t ress, 1203 const char* dstFileName, 1204 const char* srcFileName, 1205 int compressionLevel) 1206 { 1207 int result; 1208 1209 /* File check */ 1210 if (UTIL_isDirectory(srcFileName)) { 1211 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1212 return 1; 1213 } 1214 1215 ress.srcFile = FIO_openSrcFile(srcFileName); 1216 if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ 1217 1218 result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); 1219 1220 fclose(ress.srcFile); 1221 ress.srcFile = NULL; 1222 if ( g_removeSrcFile /* --rm */ 1223 && result == 0 /* success */ 1224 && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ 1225 ) { 1226 /* We must clear the handler, since after this point calling it would 1227 * delete both the source and destination files. 1228 */ 1229 clearHandler(); 1230 if (FIO_remove(srcFileName)) 1231 EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); 1232 } 1233 return result; 1234 } 1235 1236 1237 int FIO_compressFilename(const char* dstFileName, const char* srcFileName, 1238 const char* dictFileName, int compressionLevel, 1239 ZSTD_compressionParameters comprParams) 1240 { 1241 clock_t const start = clock(); 1242 U64 const fileSize = UTIL_getFileSize(srcFileName); 1243 U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; 1244 1245 cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 1246 int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); 1247 1248 double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; 1249 DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); 1250 1251 FIO_freeCResources(ress); 1252 return result; 1253 } 1254 1255 1256 /* FIO_determineCompressedName() : 1257 * create a destination filename for compressed srcFileName. 1258 * @return a pointer to it. 1259 * This function never returns an error (it may abort() in case of pb) 1260 */ 1261 static const char* 1262 FIO_determineCompressedName(const char* srcFileName, const char* suffix) 1263 { 1264 static size_t dfnbCapacity = 0; 1265 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 1266 1267 size_t const sfnSize = strlen(srcFileName); 1268 size_t const suffixSize = strlen(suffix); 1269 1270 if (dfnbCapacity <= sfnSize+suffixSize+1) { 1271 /* resize buffer for dstName */ 1272 free(dstFileNameBuffer); 1273 dfnbCapacity = sfnSize + suffixSize + 30; 1274 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 1275 if (!dstFileNameBuffer) { 1276 EXM_THROW(30, "zstd: %s", strerror(errno)); 1277 } } 1278 assert(dstFileNameBuffer != NULL); 1279 memcpy(dstFileNameBuffer, srcFileName, sfnSize); 1280 memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */); 1281 1282 return dstFileNameBuffer; 1283 } 1284 1285 1286 /* FIO_compressMultipleFilenames() : 1287 * compress nbFiles files 1288 * into one destination (outFileName) 1289 * or into one file each (outFileName == NULL, but suffix != NULL). 1290 */ 1291 int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, 1292 const char* outFileName, const char* suffix, 1293 const char* dictFileName, int compressionLevel, 1294 ZSTD_compressionParameters comprParams) 1295 { 1296 int error = 0; 1297 U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); 1298 U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; 1299 U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; 1300 cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); 1301 1302 /* init */ 1303 assert(outFileName != NULL || suffix != NULL); 1304 1305 if (outFileName != NULL) { /* output into a single destination (stdout typically) */ 1306 ress.dstFile = FIO_openDstFile(NULL, outFileName); 1307 if (ress.dstFile == NULL) { /* could not open outFileName */ 1308 error = 1; 1309 } else { 1310 unsigned u; 1311 for (u=0; u<nbFiles; u++) 1312 error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); 1313 if (fclose(ress.dstFile)) 1314 EXM_THROW(29, "Write error (%s) : cannot properly close %s", 1315 strerror(errno), outFileName); 1316 ress.dstFile = NULL; 1317 } 1318 } else { 1319 unsigned u; 1320 for (u=0; u<nbFiles; u++) { 1321 const char* const srcFileName = inFileNamesTable[u]; 1322 const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */ 1323 error |= FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); 1324 } } 1325 1326 FIO_freeCResources(ress); 1327 return error; 1328 } 1329 1330 #endif /* #ifndef ZSTD_NOCOMPRESS */ 1331 1332 1333 1334 #ifndef ZSTD_NODECOMPRESS 1335 1336 /* ************************************************************************** 1337 * Decompression 1338 ***************************************************************************/ 1339 typedef struct { 1340 void* srcBuffer; 1341 size_t srcBufferSize; 1342 size_t srcBufferLoaded; 1343 void* dstBuffer; 1344 size_t dstBufferSize; 1345 ZSTD_DStream* dctx; 1346 FILE* dstFile; 1347 } dRess_t; 1348 1349 static dRess_t FIO_createDResources(const char* dictFileName) 1350 { 1351 dRess_t ress; 1352 memset(&ress, 0, sizeof(ress)); 1353 1354 /* Allocation */ 1355 ress.dctx = ZSTD_createDStream(); 1356 if (ress.dctx==NULL) 1357 EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); 1358 CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) ); 1359 ress.srcBufferSize = ZSTD_DStreamInSize(); 1360 ress.srcBuffer = malloc(ress.srcBufferSize); 1361 ress.dstBufferSize = ZSTD_DStreamOutSize(); 1362 ress.dstBuffer = malloc(ress.dstBufferSize); 1363 if (!ress.srcBuffer || !ress.dstBuffer) 1364 EXM_THROW(61, "Allocation error : not enough memory"); 1365 1366 /* dictionary */ 1367 { void* dictBuffer; 1368 size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName); 1369 CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); 1370 free(dictBuffer); 1371 } 1372 1373 return ress; 1374 } 1375 1376 static void FIO_freeDResources(dRess_t ress) 1377 { 1378 CHECK( ZSTD_freeDStream(ress.dctx) ); 1379 free(ress.srcBuffer); 1380 free(ress.dstBuffer); 1381 } 1382 1383 1384 /** FIO_fwriteSparse() : 1385 * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ 1386 static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) 1387 { 1388 const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ 1389 size_t bufferSizeT = bufferSize / sizeof(size_t); 1390 const size_t* const bufferTEnd = bufferT + bufferSizeT; 1391 const size_t* ptrT = bufferT; 1392 static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ 1393 1394 if (!g_sparseFileSupport) { /* normal write */ 1395 size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); 1396 if (sizeCheck != bufferSize) 1397 EXM_THROW(70, "Write error : %s (cannot write decoded block)", 1398 strerror(errno)); 1399 return 0; 1400 } 1401 1402 /* avoid int overflow */ 1403 if (storedSkips > 1 GB) { 1404 int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); 1405 if (seekResult != 0) 1406 EXM_THROW(71, "1 GB skip error (sparse file support)"); 1407 storedSkips -= 1 GB; 1408 } 1409 1410 while (ptrT < bufferTEnd) { 1411 size_t seg0SizeT = segmentSizeT; 1412 size_t nb0T; 1413 1414 /* count leading zeros */ 1415 if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; 1416 bufferSizeT -= seg0SizeT; 1417 for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; 1418 storedSkips += (unsigned)(nb0T * sizeof(size_t)); 1419 1420 if (nb0T != seg0SizeT) { /* not all 0s */ 1421 int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1422 if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); 1423 storedSkips = 0; 1424 seg0SizeT -= nb0T; 1425 ptrT += nb0T; 1426 { size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file); 1427 if (sizeCheck != seg0SizeT) 1428 EXM_THROW(73, "Write error : cannot write decoded block"); 1429 } } 1430 ptrT += seg0SizeT; 1431 } 1432 1433 { static size_t const maskT = sizeof(size_t)-1; 1434 if (bufferSize & maskT) { 1435 /* size not multiple of sizeof(size_t) : implies end of block */ 1436 const char* const restStart = (const char*)bufferTEnd; 1437 const char* restPtr = restStart; 1438 size_t restSize = bufferSize & maskT; 1439 const char* const restEnd = restStart + restSize; 1440 for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; 1441 storedSkips += (unsigned) (restPtr - restStart); 1442 if (restPtr != restEnd) { 1443 int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); 1444 if (seekResult) 1445 EXM_THROW(74, "Sparse skip error ; try --no-sparse"); 1446 storedSkips = 0; 1447 { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); 1448 if (sizeCheck != (size_t)(restEnd - restPtr)) 1449 EXM_THROW(75, "Write error : cannot write decoded end of block"); 1450 } } } } 1451 1452 return storedSkips; 1453 } 1454 1455 static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) 1456 { 1457 if (storedSkips>0) { 1458 assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ 1459 if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) 1460 EXM_THROW(69, "Final skip error (sparse file support)"); 1461 /* last zero must be explicitly written, 1462 * so that skipped ones get implicitly translated as zero by FS */ 1463 { const char lastZeroByte[1] = { 0 }; 1464 if (fwrite(lastZeroByte, 1, 1, file) != 1) 1465 EXM_THROW(69, "Write error : cannot write last zero"); 1466 } } 1467 } 1468 1469 1470 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode 1471 @return : 0 (no error) */ 1472 static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) 1473 { 1474 size_t const blockSize = MIN(64 KB, bufferSize); 1475 size_t readFromInput = 1; 1476 unsigned storedSkips = 0; 1477 1478 /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ 1479 { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); 1480 if (sizeCheck != alreadyLoaded) { 1481 DISPLAYLEVEL(1, "Pass-through write error \n"); 1482 return 1; 1483 } } 1484 1485 while (readFromInput) { 1486 readFromInput = fread(buffer, 1, blockSize, finput); 1487 storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); 1488 } 1489 1490 FIO_fwriteSparseEnd(foutput, storedSkips); 1491 return 0; 1492 } 1493 1494 /* FIO_highbit64() : 1495 * gives position of highest bit. 1496 * note : only works for v > 0 ! 1497 */ 1498 static unsigned FIO_highbit64(unsigned long long v) 1499 { 1500 unsigned count = 0; 1501 assert(v != 0); 1502 v >>= 1; 1503 while (v) { v >>= 1; count++; } 1504 return count; 1505 } 1506 1507 /* FIO_zstdErrorHelp() : 1508 * detailed error message when requested window size is too large */ 1509 static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName) 1510 { 1511 ZSTD_frameHeader header; 1512 1513 /* Help message only for one specific error */ 1514 if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) 1515 return; 1516 1517 /* Try to decode the frame header */ 1518 err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); 1519 if (err == 0) { 1520 unsigned long long const windowSize = header.windowSize; 1521 unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); 1522 assert(g_memLimit > 0); 1523 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", 1524 srcFileName, windowSize, g_memLimit); 1525 if (windowLog <= ZSTD_WINDOWLOG_MAX) { 1526 unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); 1527 assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ 1528 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n", 1529 srcFileName, windowLog, windowMB); 1530 return; 1531 } 1532 } 1533 DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported\n", 1534 srcFileName, ZSTD_WINDOWLOG_MAX); 1535 } 1536 1537 /** FIO_decompressFrame() : 1538 * @return : size of decoded zstd frame, or an error code 1539 */ 1540 #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) 1541 static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, 1542 FILE* finput, 1543 const char* srcFileName, 1544 U64 alreadyDecoded) 1545 { 1546 U64 frameSize = 0; 1547 U32 storedSkips = 0; 1548 1549 size_t const srcFileLength = strlen(srcFileName); 1550 if (srcFileLength>20) srcFileName += srcFileLength-20; /* display last 20 characters only */ 1551 1552 ZSTD_resetDStream(ress->dctx); 1553 1554 /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ 1555 { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; 1556 if (ress->srcBufferLoaded < toDecode) { 1557 size_t const toRead = toDecode - ress->srcBufferLoaded; 1558 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1559 ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); 1560 } } 1561 1562 /* Main decompression Loop */ 1563 while (1) { 1564 ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; 1565 ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; 1566 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); 1567 if (ZSTD_isError(readSizeHint)) { 1568 DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", 1569 srcFileName, ZSTD_getErrorName(readSizeHint)); 1570 FIO_zstdErrorHelp(ress, readSizeHint, srcFileName); 1571 return FIO_ERROR_FRAME_DECODING; 1572 } 1573 1574 /* Write block */ 1575 storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); 1576 frameSize += outBuff.pos; 1577 DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", 1578 srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); 1579 1580 if (inBuff.pos > 0) { 1581 memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); 1582 ress->srcBufferLoaded -= inBuff.pos; 1583 } 1584 1585 if (readSizeHint == 0) break; /* end of frame */ 1586 if (inBuff.size != inBuff.pos) { 1587 DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n", 1588 srcFileName); 1589 return FIO_ERROR_FRAME_DECODING; 1590 } 1591 1592 /* Fill input buffer */ 1593 { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ 1594 if (ress->srcBufferLoaded < toDecode) { 1595 size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ 1596 void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; 1597 size_t const readSize = fread(startPosition, 1, toRead, finput); 1598 if (readSize==0) { 1599 DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", 1600 srcFileName); 1601 return FIO_ERROR_FRAME_DECODING; 1602 } 1603 ress->srcBufferLoaded += readSize; 1604 } } } 1605 1606 FIO_fwriteSparseEnd(ress->dstFile, storedSkips); 1607 1608 return frameSize; 1609 } 1610 1611 1612 #ifdef ZSTD_GZDECOMPRESS 1613 static unsigned long long FIO_decompressGzFrame(dRess_t* ress, 1614 FILE* srcFile, const char* srcFileName) 1615 { 1616 unsigned long long outFileSize = 0; 1617 z_stream strm; 1618 int flush = Z_NO_FLUSH; 1619 int decodingError = 0; 1620 1621 strm.zalloc = Z_NULL; 1622 strm.zfree = Z_NULL; 1623 strm.opaque = Z_NULL; 1624 strm.next_in = 0; 1625 strm.avail_in = 0; 1626 /* see http://www.zlib.net/manual.html */ 1627 if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) 1628 return FIO_ERROR_FRAME_DECODING; 1629 1630 strm.next_out = (Bytef*)ress->dstBuffer; 1631 strm.avail_out = (uInt)ress->dstBufferSize; 1632 strm.avail_in = (uInt)ress->srcBufferLoaded; 1633 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1634 1635 for ( ; ; ) { 1636 int ret; 1637 if (strm.avail_in == 0) { 1638 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1639 if (ress->srcBufferLoaded == 0) flush = Z_FINISH; 1640 strm.next_in = (z_const unsigned char*)ress->srcBuffer; 1641 strm.avail_in = (uInt)ress->srcBufferLoaded; 1642 } 1643 ret = inflate(&strm, flush); 1644 if (ret == Z_BUF_ERROR) { 1645 DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); 1646 decodingError = 1; break; 1647 } 1648 if (ret != Z_OK && ret != Z_STREAM_END) { 1649 DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); 1650 decodingError = 1; break; 1651 } 1652 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1653 if (decompBytes) { 1654 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1655 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1656 decodingError = 1; break; 1657 } 1658 outFileSize += decompBytes; 1659 strm.next_out = (Bytef*)ress->dstBuffer; 1660 strm.avail_out = (uInt)ress->dstBufferSize; 1661 } 1662 } 1663 if (ret == Z_STREAM_END) break; 1664 } 1665 1666 if (strm.avail_in > 0) 1667 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1668 ress->srcBufferLoaded = strm.avail_in; 1669 if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ 1670 && (decodingError==0) ) { 1671 DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); 1672 decodingError = 1; 1673 } 1674 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1675 } 1676 #endif 1677 1678 1679 #ifdef ZSTD_LZMADECOMPRESS 1680 static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) 1681 { 1682 unsigned long long outFileSize = 0; 1683 lzma_stream strm = LZMA_STREAM_INIT; 1684 lzma_action action = LZMA_RUN; 1685 lzma_ret initRet; 1686 int decodingError = 0; 1687 1688 strm.next_in = 0; 1689 strm.avail_in = 0; 1690 if (plain_lzma) { 1691 initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ 1692 } else { 1693 initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ 1694 } 1695 1696 if (initRet != LZMA_OK) { 1697 DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", 1698 plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", 1699 srcFileName, initRet); 1700 return FIO_ERROR_FRAME_DECODING; 1701 } 1702 1703 strm.next_out = (BYTE*)ress->dstBuffer; 1704 strm.avail_out = ress->dstBufferSize; 1705 strm.next_in = (BYTE const*)ress->srcBuffer; 1706 strm.avail_in = ress->srcBufferLoaded; 1707 1708 for ( ; ; ) { 1709 lzma_ret ret; 1710 if (strm.avail_in == 0) { 1711 ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); 1712 if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; 1713 strm.next_in = (BYTE const*)ress->srcBuffer; 1714 strm.avail_in = ress->srcBufferLoaded; 1715 } 1716 ret = lzma_code(&strm, action); 1717 1718 if (ret == LZMA_BUF_ERROR) { 1719 DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); 1720 decodingError = 1; break; 1721 } 1722 if (ret != LZMA_OK && ret != LZMA_STREAM_END) { 1723 DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", 1724 srcFileName, ret); 1725 decodingError = 1; break; 1726 } 1727 { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; 1728 if (decompBytes) { 1729 if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) { 1730 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1731 decodingError = 1; break; 1732 } 1733 outFileSize += decompBytes; 1734 strm.next_out = (BYTE*)ress->dstBuffer; 1735 strm.avail_out = ress->dstBufferSize; 1736 } } 1737 if (ret == LZMA_STREAM_END) break; 1738 } 1739 1740 if (strm.avail_in > 0) 1741 memmove(ress->srcBuffer, strm.next_in, strm.avail_in); 1742 ress->srcBufferLoaded = strm.avail_in; 1743 lzma_end(&strm); 1744 return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; 1745 } 1746 #endif 1747 1748 #ifdef ZSTD_LZ4DECOMPRESS 1749 static unsigned long long FIO_decompressLz4Frame(dRess_t* ress, 1750 FILE* srcFile, const char* srcFileName) 1751 { 1752 unsigned long long filesize = 0; 1753 LZ4F_errorCode_t nextToLoad; 1754 LZ4F_decompressionContext_t dCtx; 1755 LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); 1756 int decodingError = 0; 1757 1758 if (LZ4F_isError(errorCode)) { 1759 DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); 1760 return FIO_ERROR_FRAME_DECODING; 1761 } 1762 1763 /* Init feed with magic number (already consumed from FILE* sFile) */ 1764 { size_t inSize = 4; 1765 size_t outSize= 0; 1766 MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); 1767 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); 1768 if (LZ4F_isError(nextToLoad)) { 1769 DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", 1770 srcFileName, LZ4F_getErrorName(nextToLoad)); 1771 LZ4F_freeDecompressionContext(dCtx); 1772 return FIO_ERROR_FRAME_DECODING; 1773 } } 1774 1775 /* Main Loop */ 1776 for (;nextToLoad;) { 1777 size_t readSize; 1778 size_t pos = 0; 1779 size_t decodedBytes = ress->dstBufferSize; 1780 1781 /* Read input */ 1782 if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; 1783 readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); 1784 if (!readSize) break; /* reached end of file or stream */ 1785 1786 while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ 1787 /* Decode Input (at least partially) */ 1788 size_t remaining = readSize - pos; 1789 decodedBytes = ress->dstBufferSize; 1790 nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); 1791 if (LZ4F_isError(nextToLoad)) { 1792 DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", 1793 srcFileName, LZ4F_getErrorName(nextToLoad)); 1794 decodingError = 1; nextToLoad = 0; break; 1795 } 1796 pos += remaining; 1797 1798 /* Write Block */ 1799 if (decodedBytes) { 1800 if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) { 1801 DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno)); 1802 decodingError = 1; nextToLoad = 0; break; 1803 } 1804 filesize += decodedBytes; 1805 DISPLAYUPDATE(2, "\rDecompressed : %u MB ", (unsigned)(filesize>>20)); 1806 } 1807 1808 if (!nextToLoad) break; 1809 } 1810 } 1811 /* can be out because readSize == 0, which could be an fread() error */ 1812 if (ferror(srcFile)) { 1813 DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); 1814 decodingError=1; 1815 } 1816 1817 if (nextToLoad!=0) { 1818 DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); 1819 decodingError=1; 1820 } 1821 1822 LZ4F_freeDecompressionContext(dCtx); 1823 ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ 1824 1825 return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; 1826 } 1827 #endif 1828 1829 1830 1831 /** FIO_decompressFrames() : 1832 * Find and decode frames inside srcFile 1833 * srcFile presumed opened and valid 1834 * @return : 0 : OK 1835 * 1 : error 1836 */ 1837 static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, 1838 const char* dstFileName, const char* srcFileName) 1839 { 1840 unsigned readSomething = 0; 1841 unsigned long long filesize = 0; 1842 assert(srcFile != NULL); 1843 1844 /* for each frame */ 1845 for ( ; ; ) { 1846 /* check magic number -> version */ 1847 size_t const toRead = 4; 1848 const BYTE* const buf = (const BYTE*)ress.srcBuffer; 1849 if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ 1850 ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, 1851 (size_t)1, toRead - ress.srcBufferLoaded, srcFile); 1852 if (ress.srcBufferLoaded==0) { 1853 if (readSomething==0) { /* srcFile is empty (which is invalid) */ 1854 DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); 1855 return 1; 1856 } /* else, just reached frame boundary */ 1857 break; /* no more input */ 1858 } 1859 readSomething = 1; /* there is at least 1 byte in srcFile */ 1860 if (ress.srcBufferLoaded < toRead) { 1861 DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); 1862 return 1; 1863 } 1864 if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { 1865 unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); 1866 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1867 filesize += frameSize; 1868 } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ 1869 #ifdef ZSTD_GZDECOMPRESS 1870 unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName); 1871 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1872 filesize += frameSize; 1873 #else 1874 DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); 1875 return 1; 1876 #endif 1877 } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ 1878 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ 1879 #ifdef ZSTD_LZMADECOMPRESS 1880 unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); 1881 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1882 filesize += frameSize; 1883 #else 1884 DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); 1885 return 1; 1886 #endif 1887 } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { 1888 #ifdef ZSTD_LZ4DECOMPRESS 1889 unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName); 1890 if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; 1891 filesize += frameSize; 1892 #else 1893 DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); 1894 return 1; 1895 #endif 1896 } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ 1897 return FIO_passThrough(ress.dstFile, srcFile, 1898 ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); 1899 } else { 1900 DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); 1901 return 1; 1902 } } /* for each frame */ 1903 1904 /* Final Status */ 1905 DISPLAYLEVEL(2, "\r%79s\r", ""); 1906 DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); 1907 1908 return 0; 1909 } 1910 1911 /** FIO_decompressDstFile() : 1912 open `dstFileName`, 1913 or path-through if ress.dstFile is already != 0, 1914 then start decompression process (FIO_decompressFrames()). 1915 @return : 0 : OK 1916 1 : operation aborted 1917 */ 1918 static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, 1919 const char* dstFileName, const char* srcFileName) 1920 { 1921 int result; 1922 stat_t statbuf; 1923 int transfer_permissions = 0; 1924 int releaseDstFile = 0; 1925 1926 if (ress.dstFile == NULL) { 1927 releaseDstFile = 1; 1928 1929 ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); 1930 if (ress.dstFile==0) return 1; 1931 1932 /* Must only be added after FIO_openDstFile() succeeds. 1933 * Otherwise we may delete the destination file if it already exists, 1934 * and the user presses Ctrl-C when asked if they wish to overwrite. 1935 */ 1936 addHandler(dstFileName); 1937 1938 if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ 1939 && UTIL_getFileStat(srcFileName, &statbuf) ) 1940 transfer_permissions = 1; 1941 } 1942 1943 1944 result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); 1945 1946 if (releaseDstFile) { 1947 FILE* const dstFile = ress.dstFile; 1948 clearHandler(); 1949 ress.dstFile = NULL; 1950 if (fclose(dstFile)) { 1951 DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); 1952 result = 1; 1953 } 1954 1955 if ( (result != 0) /* operation failure */ 1956 && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ 1957 && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ 1958 ) { 1959 FIO_remove(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ 1960 } else { /* operation success */ 1961 if ( strcmp(dstFileName, stdoutmark) /* special case : don't chmod stdout */ 1962 && strcmp(dstFileName, nulmark) /* special case : don't chmod /dev/null */ 1963 && transfer_permissions ) /* file permissions correctly extracted from src */ 1964 UTIL_setFileStat(dstFileName, &statbuf); /* transfer file permissions from src into dst */ 1965 } 1966 } 1967 1968 return result; 1969 } 1970 1971 1972 /** FIO_decompressSrcFile() : 1973 Open `srcFileName`, transfer control to decompressDstFile() 1974 @return : 0 : OK 1975 1 : error 1976 */ 1977 static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) 1978 { 1979 FILE* srcFile; 1980 int result; 1981 1982 if (UTIL_isDirectory(srcFileName)) { 1983 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); 1984 return 1; 1985 } 1986 1987 srcFile = FIO_openSrcFile(srcFileName); 1988 if (srcFile==NULL) return 1; 1989 ress.srcBufferLoaded = 0; 1990 1991 result = FIO_decompressDstFile(ress, srcFile, dstFileName, srcFileName); 1992 1993 /* Close file */ 1994 if (fclose(srcFile)) { 1995 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ 1996 return 1; 1997 } 1998 if ( g_removeSrcFile /* --rm */ 1999 && (result==0) /* decompression successful */ 2000 && strcmp(srcFileName, stdinmark) ) /* not stdin */ { 2001 /* We must clear the handler, since after this point calling it would 2002 * delete both the source and destination files. 2003 */ 2004 clearHandler(); 2005 if (FIO_remove(srcFileName)) { 2006 /* failed to remove src file */ 2007 DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); 2008 return 1; 2009 } } 2010 return result; 2011 } 2012 2013 2014 2015 int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, 2016 const char* dictFileName) 2017 { 2018 dRess_t const ress = FIO_createDResources(dictFileName); 2019 2020 int const decodingError = FIO_decompressSrcFile(ress, dstFileName, srcFileName); 2021 2022 FIO_freeDResources(ress); 2023 return decodingError; 2024 } 2025 2026 2027 /* FIO_determineDstName() : 2028 * create a destination filename from a srcFileName. 2029 * @return a pointer to it. 2030 * @return == NULL if there is an error */ 2031 static const char* 2032 FIO_determineDstName(const char* srcFileName) 2033 { 2034 static size_t dfnbCapacity = 0; 2035 static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ 2036 2037 size_t const sfnSize = strlen(srcFileName); 2038 size_t suffixSize; 2039 const char* const suffixPtr = strrchr(srcFileName, '.'); 2040 if (suffixPtr == NULL) { 2041 DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", 2042 srcFileName); 2043 return NULL; 2044 } 2045 suffixSize = strlen(suffixPtr); 2046 2047 /* check suffix is authorized */ 2048 if (sfnSize <= suffixSize 2049 || ( strcmp(suffixPtr, ZSTD_EXTENSION) 2050 #ifdef ZSTD_GZDECOMPRESS 2051 && strcmp(suffixPtr, GZ_EXTENSION) 2052 #endif 2053 #ifdef ZSTD_LZMADECOMPRESS 2054 && strcmp(suffixPtr, XZ_EXTENSION) 2055 && strcmp(suffixPtr, LZMA_EXTENSION) 2056 #endif 2057 #ifdef ZSTD_LZ4DECOMPRESS 2058 && strcmp(suffixPtr, LZ4_EXTENSION) 2059 #endif 2060 ) ) { 2061 const char* suffixlist = ZSTD_EXTENSION 2062 #ifdef ZSTD_GZDECOMPRESS 2063 "/" GZ_EXTENSION 2064 #endif 2065 #ifdef ZSTD_LZMADECOMPRESS 2066 "/" XZ_EXTENSION "/" LZMA_EXTENSION 2067 #endif 2068 #ifdef ZSTD_LZ4DECOMPRESS 2069 "/" LZ4_EXTENSION 2070 #endif 2071 ; 2072 DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s expected) -- ignored \n", 2073 srcFileName, suffixlist); 2074 return NULL; 2075 } 2076 2077 /* allocate enough space to write dstFilename into it */ 2078 if (dfnbCapacity+suffixSize <= sfnSize+1) { 2079 free(dstFileNameBuffer); 2080 dfnbCapacity = sfnSize + 20; 2081 dstFileNameBuffer = (char*)malloc(dfnbCapacity); 2082 if (dstFileNameBuffer==NULL) 2083 EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); 2084 } 2085 2086 /* return dst name == src name truncated from suffix */ 2087 assert(dstFileNameBuffer != NULL); 2088 memcpy(dstFileNameBuffer, srcFileName, sfnSize - suffixSize); 2089 dstFileNameBuffer[sfnSize-suffixSize] = '\0'; 2090 return dstFileNameBuffer; 2091 2092 /* note : dstFileNameBuffer memory is not going to be free */ 2093 } 2094 2095 2096 int 2097 FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles, 2098 const char* outFileName, 2099 const char* dictFileName) 2100 { 2101 int error = 0; 2102 dRess_t ress = FIO_createDResources(dictFileName); 2103 2104 if (outFileName) { 2105 unsigned u; 2106 ress.dstFile = FIO_openDstFile(NULL, outFileName); 2107 if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); 2108 for (u=0; u<nbFiles; u++) 2109 error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]); 2110 if (fclose(ress.dstFile)) 2111 EXM_THROW(72, "Write error : %s : cannot properly close output file", 2112 strerror(errno)); 2113 } else { 2114 unsigned u; 2115 for (u=0; u<nbFiles; u++) { /* create dstFileName */ 2116 const char* const srcFileName = srcNamesTable[u]; 2117 const char* const dstFileName = FIO_determineDstName(srcFileName); 2118 if (dstFileName == NULL) { error=1; continue; } 2119 2120 error |= FIO_decompressSrcFile(ress, dstFileName, srcFileName); 2121 } 2122 } 2123 2124 FIO_freeDResources(ress); 2125 return error; 2126 } 2127 2128 2129 2130 /* ************************************************************************** 2131 * .zst file info (--list command) 2132 ***************************************************************************/ 2133 2134 typedef struct { 2135 U64 decompressedSize; 2136 U64 compressedSize; 2137 U64 windowSize; 2138 int numActualFrames; 2139 int numSkippableFrames; 2140 int decompUnavailable; 2141 int usesCheck; 2142 U32 nbFiles; 2143 } fileInfo_t; 2144 2145 typedef enum { info_success=0, info_frame_error=1, info_not_zstd=2, info_file_error=3 } InfoError; 2146 2147 #define ERROR_IF(c,n,...) { \ 2148 if (c) { \ 2149 DISPLAYLEVEL(1, __VA_ARGS__); \ 2150 DISPLAYLEVEL(1, " \n"); \ 2151 return n; \ 2152 } \ 2153 } 2154 2155 static InfoError 2156 FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) 2157 { 2158 /* begin analyzing frame */ 2159 for ( ; ; ) { 2160 BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 2161 size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); 2162 if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN) { 2163 if ( feof(srcFile) 2164 && (numBytesRead == 0) 2165 && (info->compressedSize > 0) 2166 && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { 2167 break; /* correct end of file => success */ 2168 } 2169 ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); 2170 ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); 2171 } 2172 { U32 const magicNumber = MEM_readLE32(headerBuffer); 2173 /* Zstandard frame */ 2174 if (magicNumber == ZSTD_MAGICNUMBER) { 2175 ZSTD_frameHeader header; 2176 U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); 2177 if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR 2178 || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { 2179 info->decompUnavailable = 1; 2180 } else { 2181 info->decompressedSize += frameContentSize; 2182 } 2183 ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, 2184 info_frame_error, "Error: could not decode frame header"); 2185 info->windowSize = header.windowSize; 2186 /* move to the end of the frame header */ 2187 { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); 2188 ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); 2189 ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, 2190 info_frame_error, "Error: could not move to end of frame header"); 2191 } 2192 2193 /* skip all blocks in the frame */ 2194 { int lastBlock = 0; 2195 do { 2196 BYTE blockHeaderBuffer[3]; 2197 ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, 2198 info_frame_error, "Error while reading block header"); 2199 { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); 2200 U32 const blockTypeID = (blockHeader >> 1) & 3; 2201 U32 const isRLE = (blockTypeID == 1); 2202 U32 const isWrongBlock = (blockTypeID == 3); 2203 long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); 2204 ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); 2205 lastBlock = blockHeader & 1; 2206 ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, 2207 info_frame_error, "Error: could not skip to end of block"); 2208 } 2209 } while (lastBlock != 1); 2210 } 2211 2212 /* check if checksum is used */ 2213 { BYTE const frameHeaderDescriptor = headerBuffer[4]; 2214 int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; 2215 if (contentChecksumFlag) { 2216 info->usesCheck = 1; 2217 ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, 2218 info_frame_error, "Error: could not skip past checksum"); 2219 } } 2220 info->numActualFrames++; 2221 } 2222 /* Skippable frame */ 2223 else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { 2224 U32 const frameSize = MEM_readLE32(headerBuffer + 4); 2225 long const seek = (long)(8 + frameSize - numBytesRead); 2226 ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, 2227 info_frame_error, "Error: could not find end of skippable frame"); 2228 info->numSkippableFrames++; 2229 } 2230 /* unknown content */ 2231 else { 2232 return info_not_zstd; 2233 } 2234 } /* magic number analysis */ 2235 } /* end analyzing frames */ 2236 return info_success; 2237 } 2238 2239 2240 static InfoError 2241 getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) 2242 { 2243 InfoError status; 2244 FILE* const srcFile = FIO_openSrcFile(inFileName); 2245 ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); 2246 2247 info->compressedSize = UTIL_getFileSize(inFileName); 2248 status = FIO_analyzeFrames(info, srcFile); 2249 2250 fclose(srcFile); 2251 info->nbFiles = 1; 2252 return status; 2253 } 2254 2255 2256 /** getFileInfo() : 2257 * Reads information from file, stores in *info 2258 * @return : InfoError status 2259 */ 2260 static InfoError 2261 getFileInfo(fileInfo_t* info, const char* srcFileName) 2262 { 2263 ERROR_IF(!UTIL_isRegularFile(srcFileName), 2264 info_file_error, "Error : %s is not a file", srcFileName); 2265 return getFileInfo_fileConfirmed(info, srcFileName); 2266 } 2267 2268 2269 static void 2270 displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) 2271 { 2272 unsigned const unit = info->compressedSize < (1 MB) ? (1 KB) : (1 MB); 2273 const char* const unitStr = info->compressedSize < (1 MB) ? "KB" : "MB"; 2274 double const windowSizeUnit = (double)info->windowSize / unit; 2275 double const compressedSizeUnit = (double)info->compressedSize / unit; 2276 double const decompressedSizeUnit = (double)info->decompressedSize / unit; 2277 double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/info->compressedSize; 2278 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); 2279 if (displayLevel <= 2) { 2280 if (!info->decompUnavailable) { 2281 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n", 2282 info->numSkippableFrames + info->numActualFrames, 2283 info->numSkippableFrames, 2284 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 2285 ratio, checkString, inFileName); 2286 } else { 2287 DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n", 2288 info->numSkippableFrames + info->numActualFrames, 2289 info->numSkippableFrames, 2290 compressedSizeUnit, unitStr, 2291 checkString, inFileName); 2292 } 2293 } else { 2294 DISPLAYOUT("%s \n", inFileName); 2295 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); 2296 if (info->numSkippableFrames) 2297 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); 2298 DISPLAYOUT("Window Size: %.2f %2s (%llu B)\n", 2299 windowSizeUnit, unitStr, 2300 (unsigned long long)info->windowSize); 2301 DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n", 2302 compressedSizeUnit, unitStr, 2303 (unsigned long long)info->compressedSize); 2304 if (!info->decompUnavailable) { 2305 DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n", 2306 decompressedSizeUnit, unitStr, 2307 (unsigned long long)info->decompressedSize); 2308 DISPLAYOUT("Ratio: %.4f\n", ratio); 2309 } 2310 DISPLAYOUT("Check: %s\n", checkString); 2311 DISPLAYOUT("\n"); 2312 } 2313 } 2314 2315 static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) 2316 { 2317 fileInfo_t total; 2318 memset(&total, 0, sizeof(total)); 2319 total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; 2320 total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; 2321 total.compressedSize = fi1.compressedSize + fi2.compressedSize; 2322 total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; 2323 total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; 2324 total.usesCheck = fi1.usesCheck & fi2.usesCheck; 2325 total.nbFiles = fi1.nbFiles + fi2.nbFiles; 2326 return total; 2327 } 2328 2329 static int 2330 FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) 2331 { 2332 fileInfo_t info; 2333 memset(&info, 0, sizeof(info)); 2334 { InfoError const error = getFileInfo(&info, inFileName); 2335 if (error == info_frame_error) { 2336 /* display error, but provide output */ 2337 DISPLAYLEVEL(1, "Error while parsing %s \n", inFileName); 2338 } 2339 else if (error == info_not_zstd) { 2340 DISPLAYOUT("File %s not compressed by zstd \n", inFileName); 2341 if (displayLevel > 2) DISPLAYOUT("\n"); 2342 return 1; 2343 } 2344 else if (error == info_file_error) { 2345 /* error occurred while opening the file */ 2346 if (displayLevel > 2) DISPLAYOUT("\n"); 2347 return 1; 2348 } 2349 displayInfo(inFileName, &info, displayLevel); 2350 *total = FIO_addFInfo(*total, info); 2351 assert(error == info_success || error == info_frame_error); 2352 return error; 2353 } 2354 } 2355 2356 int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel) 2357 { 2358 /* ensure no specified input is stdin (needs fseek() capability) */ 2359 { unsigned u; 2360 for (u=0; u<numFiles;u++) { 2361 ERROR_IF(!strcmp (filenameTable[u], stdinmark), 2362 1, "zstd: --list does not support reading from standard input"); 2363 } } 2364 2365 if (numFiles == 0) { 2366 if (!IS_CONSOLE(stdin)) { 2367 DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); 2368 } 2369 DISPLAYLEVEL(1, "No files given \n"); 2370 return 1; 2371 } 2372 2373 if (displayLevel <= 2) { 2374 DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); 2375 } 2376 { int error = 0; 2377 fileInfo_t total; 2378 memset(&total, 0, sizeof(total)); 2379 total.usesCheck = 1; 2380 /* --list each file, and check for any error */ 2381 { unsigned u; 2382 for (u=0; u<numFiles;u++) { 2383 error |= FIO_listFile(&total, filenameTable[u], displayLevel); 2384 } } 2385 if (numFiles > 1 && displayLevel <= 2) { /* display total */ 2386 unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB); 2387 const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB"; 2388 double const compressedSizeUnit = (double)total.compressedSize / unit; 2389 double const decompressedSizeUnit = (double)total.decompressedSize / unit; 2390 double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize; 2391 const char* const checkString = (total.usesCheck ? "XXH64" : ""); 2392 DISPLAYOUT("----------------------------------------------------------------- \n"); 2393 if (total.decompUnavailable) { 2394 DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n", 2395 total.numSkippableFrames + total.numActualFrames, 2396 total.numSkippableFrames, 2397 compressedSizeUnit, unitStr, 2398 checkString, (unsigned)total.nbFiles); 2399 } else { 2400 DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n", 2401 total.numSkippableFrames + total.numActualFrames, 2402 total.numSkippableFrames, 2403 compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr, 2404 ratio, checkString, (unsigned)total.nbFiles); 2405 } } 2406 return error; 2407 } 2408 } 2409 2410 2411 #endif /* #ifndef ZSTD_NODECOMPRESS */ 2412