1 /*
2  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under both the BSD-style license (found in the
6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7  * in the COPYING file in the root directory of this source tree).
8  * You may select, at your option, one of the above-listed licenses.
9  */
10 
11 
12 /*-************************************
13 *  Tuning parameters
14 **************************************/
15 #ifndef ZSTDCLI_CLEVEL_DEFAULT
16 #  define ZSTDCLI_CLEVEL_DEFAULT 3
17 #endif
18 
19 #ifndef ZSTDCLI_CLEVEL_MAX
20 #  define ZSTDCLI_CLEVEL_MAX 19   /* without using --ultra */
21 #endif
22 
23 
24 
25 /*-************************************
26 *  Dependencies
27 **************************************/
28 #include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
29 #include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
30 #include <stdio.h>    /* fprintf(), stdin, stdout, stderr */
31 #include <string.h>   /* strcmp, strlen */
32 #include <errno.h>    /* errno */
33 #include "fileio.h"   /* stdinmark, stdoutmark, ZSTD_EXTENSION */
34 #ifndef ZSTD_NOBENCH
35 #  include "benchzstd.h"  /* BMK_benchFiles */
36 #endif
37 #ifndef ZSTD_NODICT
38 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
39 #endif
40 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_minCLevel */
41 #include "zstd.h"     /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */
42 
43 
44 /*-************************************
45 *  Constants
46 **************************************/
47 #define COMPRESSOR_NAME "zstd command line interface"
48 #ifndef ZSTD_VERSION
49 #  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
50 #endif
51 #define AUTHOR "Yann Collet"
52 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
53 
54 #define ZSTD_ZSTDMT "zstdmt"
55 #define ZSTD_UNZSTD "unzstd"
56 #define ZSTD_CAT "zstdcat"
57 #define ZSTD_ZCAT "zcat"
58 #define ZSTD_GZ "gzip"
59 #define ZSTD_GUNZIP "gunzip"
60 #define ZSTD_GZCAT "gzcat"
61 #define ZSTD_LZMA "lzma"
62 #define ZSTD_UNLZMA "unlzma"
63 #define ZSTD_XZ "xz"
64 #define ZSTD_UNXZ "unxz"
65 #define ZSTD_LZ4 "lz4"
66 #define ZSTD_UNLZ4 "unlz4"
67 
68 #define KB *(1 <<10)
69 #define MB *(1 <<20)
70 #define GB *(1U<<30)
71 
72 #define DISPLAY_LEVEL_DEFAULT 2
73 
74 static const char*    g_defaultDictName = "dictionary";
75 static const unsigned g_defaultMaxDictSize = 110 KB;
76 static const int      g_defaultDictCLevel = 3;
77 static const unsigned g_defaultSelectivityLevel = 9;
78 static const unsigned g_defaultMaxWindowLog = 27;
79 #define OVERLAP_LOG_DEFAULT 9999
80 #define LDM_PARAM_DEFAULT 9999  /* Default for parameters where 0 is valid */
81 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
82 static U32 g_ldmHashLog = 0;
83 static U32 g_ldmMinMatch = 0;
84 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
85 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
86 
87 
88 #define DEFAULT_ACCEL 1
89 
90 typedef enum { cover, fastCover, legacy } dictType;
91 
92 /*-************************************
93 *  Display Macros
94 **************************************/
95 #define DISPLAY(...)         fprintf(g_displayOut, __VA_ARGS__)
96 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
97 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
98 static FILE* g_displayOut;
99 
100 
101 /*-************************************
102 *  Command Line
103 **************************************/
usage(const char * programName)104 static int usage(const char* programName)
105 {
106     DISPLAY( "Usage : \n");
107     DISPLAY( "      %s [args] [FILE(s)] [-o file] \n", programName);
108     DISPLAY( "\n");
109     DISPLAY( "FILE    : a filename \n");
110     DISPLAY( "          with no FILE, or when FILE is - , read standard input\n");
111     DISPLAY( "Arguments : \n");
112 #ifndef ZSTD_NOCOMPRESS
113     DISPLAY( " -#     : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
114 #endif
115 #ifndef ZSTD_NODECOMPRESS
116     DISPLAY( " -d     : decompression \n");
117 #endif
118     DISPLAY( " -D file: use `file` as Dictionary \n");
119     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
120     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
121     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
122     DISPLAY( " -k     : preserve source file(s) (default) \n");
123     DISPLAY( " -h/-H  : display help/long help and exit \n");
124     return 0;
125 }
126 
usage_advanced(const char * programName)127 static int usage_advanced(const char* programName)
128 {
129     DISPLAY(WELCOME_MESSAGE);
130     usage(programName);
131     DISPLAY( "\n");
132     DISPLAY( "Advanced arguments : \n");
133     DISPLAY( " -V     : display Version number and exit \n");
134     DISPLAY( " -v     : verbose mode; specify multiple times to increase verbosity\n");
135     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
136     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
137     DISPLAY( " -l     : print information about zstd compressed files \n");
138 #ifndef ZSTD_NOCOMPRESS
139     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
140     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
141     DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
142     DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
143 #ifdef ZSTD_MULTITHREAD
144     DISPLAY( " -T#    : spawns # compression threads (default: 1, 0==# cores) \n");
145     DISPLAY( " -B#    : select size of each job (default: 0==automatic) \n");
146     DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
147 #endif
148     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
149     DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
150 #endif
151 #ifdef UTIL_HAS_CREATEFILELIST
152     DISPLAY( " -r     : operate recursively on directories \n");
153 #endif
154     DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
155 #ifdef ZSTD_GZCOMPRESS
156     DISPLAY( "--format=gzip : compress files to the .gz format \n");
157 #endif
158 #ifdef ZSTD_LZMACOMPRESS
159     DISPLAY( "--format=xz : compress files to the .xz format \n");
160     DISPLAY( "--format=lzma : compress files to the .lzma format \n");
161 #endif
162 #ifdef ZSTD_LZ4COMPRESS
163     DISPLAY( "--format=lz4 : compress files to the .lz4 format \n");
164 #endif
165 #ifndef ZSTD_NODECOMPRESS
166     DISPLAY( "--test  : test compressed file integrity \n");
167 #if ZSTD_SPARSE_DEFAULT
168     DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n");
169 #else
170     DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n");
171 #endif
172 #endif
173     DISPLAY( " -M#    : Set a memory usage limit for decompression \n");
174     DISPLAY( "--      : All arguments after \"--\" are treated as files \n");
175 #ifndef ZSTD_NODICT
176     DISPLAY( "\n");
177     DISPLAY( "Dictionary builder : \n");
178     DISPLAY( "--train ## : create a dictionary from a training set of files \n");
179     DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
180     DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n");
181     DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
182     DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
183     DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
184     DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n");
185 #endif
186 #ifndef ZSTD_NOBENCH
187     DISPLAY( "\n");
188     DISPLAY( "Benchmark arguments : \n");
189     DISPLAY( " -b#    : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
190     DISPLAY( " -e#    : test all compression levels from -bX to # (default: 1)\n");
191     DISPLAY( " -i#    : minimum evaluation time in seconds (default: 3s) \n");
192     DISPLAY( " -B#    : cut file into independent blocks of size # (default: no block)\n");
193     DISPLAY( "--priority=rt : set process priority to real-time \n");
194 #endif
195     return 0;
196 }
197 
badusage(const char * programName)198 static int badusage(const char* programName)
199 {
200     DISPLAYLEVEL(1, "Incorrect parameters\n");
201     if (g_displayLevel >= 2) usage(programName);
202     return 1;
203 }
204 
waitEnter(void)205 static void waitEnter(void)
206 {
207     int unused;
208     DISPLAY("Press enter to continue...\n");
209     unused = getchar();
210     (void)unused;
211 }
212 
lastNameFromPath(const char * path)213 static const char* lastNameFromPath(const char* path)
214 {
215     const char* name = path;
216     if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
217     if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
218     return name;
219 }
220 
221 /*! exeNameMatch() :
222     @return : a non-zero value if exeName matches test, excluding the extension
223    */
exeNameMatch(const char * exeName,const char * test)224 static int exeNameMatch(const char* exeName, const char* test)
225 {
226     return !strncmp(exeName, test, strlen(test)) &&
227         (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
228 }
229 
errorOut(const char * msg)230 static void errorOut(const char* msg)
231 {
232     DISPLAY("%s \n", msg); exit(1);
233 }
234 
235 /*! readU32FromChar() :
236  * @return : unsigned integer value read from input in `char` format.
237  *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
238  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
239  *  Note : function will exit() program if digit sequence overflows */
readU32FromChar(const char ** stringPtr)240 static unsigned readU32FromChar(const char** stringPtr)
241 {
242     const char errorMsg[] = "error: numeric value too large";
243     unsigned result = 0;
244     while ((**stringPtr >='0') && (**stringPtr <='9')) {
245         unsigned const max = (((unsigned)(-1)) / 10) - 1;
246         if (result > max) errorOut(errorMsg);
247         result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
248     }
249     if ((**stringPtr=='K') || (**stringPtr=='M')) {
250         unsigned const maxK = ((unsigned)(-1)) >> 10;
251         if (result > maxK) errorOut(errorMsg);
252         result <<= 10;
253         if (**stringPtr=='M') {
254             if (result > maxK) errorOut(errorMsg);
255             result <<= 10;
256         }
257         (*stringPtr)++;  /* skip `K` or `M` */
258         if (**stringPtr=='i') (*stringPtr)++;
259         if (**stringPtr=='B') (*stringPtr)++;
260     }
261     return result;
262 }
263 
264 /** longCommandWArg() :
265  *  check if *stringPtr is the same as longCommand.
266  *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
267  * @return 0 and doesn't modify *stringPtr otherwise.
268  */
longCommandWArg(const char ** stringPtr,const char * longCommand)269 static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
270 {
271     size_t const comSize = strlen(longCommand);
272     int const result = !strncmp(*stringPtr, longCommand, comSize);
273     if (result) *stringPtr += comSize;
274     return result;
275 }
276 
277 
278 #ifndef ZSTD_NODICT
279 /**
280  * parseCoverParameters() :
281  * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params
282  * @return 1 means that cover parameters were correct
283  * @return 0 in case of malformed parameters
284  */
parseCoverParameters(const char * stringPtr,ZDICT_cover_params_t * params)285 static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params)
286 {
287     memset(params, 0, sizeof(*params));
288     for (; ;) {
289         if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
290         if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
291         if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
292         if (longCommandWArg(&stringPtr, "split=")) {
293           unsigned splitPercentage = readU32FromChar(&stringPtr);
294           params->splitPoint = (double)splitPercentage / 100.0;
295           if (stringPtr[0]==',') { stringPtr++; continue; } else break;
296         }
297         return 0;
298     }
299     if (stringPtr[0] != 0) return 0;
300     DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100));
301     return 1;
302 }
303 
304 /**
305  * parseFastCoverParameters() :
306  * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
307  * @return 1 means that fastcover parameters were correct
308  * @return 0 in case of malformed parameters
309  */
parseFastCoverParameters(const char * stringPtr,ZDICT_fastCover_params_t * params)310 static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
311 {
312     memset(params, 0, sizeof(*params));
313     for (; ;) {
314         if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
315         if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
316         if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
317         if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
318         if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
319         if (longCommandWArg(&stringPtr, "split=")) {
320           unsigned splitPercentage = readU32FromChar(&stringPtr);
321           params->splitPoint = (double)splitPercentage / 100.0;
322           if (stringPtr[0]==',') { stringPtr++; continue; } else break;
323         }
324         return 0;
325     }
326     if (stringPtr[0] != 0) return 0;
327     DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel);
328     return 1;
329 }
330 
331 /**
332  * parseLegacyParameters() :
333  * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
334  * @return 1 means that legacy dictionary builder parameters were correct
335  * @return 0 in case of malformed parameters
336  */
parseLegacyParameters(const char * stringPtr,unsigned * selectivity)337 static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity)
338 {
339     if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; }
340     *selectivity = readU32FromChar(&stringPtr);
341     if (stringPtr[0] != 0) return 0;
342     DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity);
343     return 1;
344 }
345 
defaultCoverParams(void)346 static ZDICT_cover_params_t defaultCoverParams(void)
347 {
348     ZDICT_cover_params_t params;
349     memset(&params, 0, sizeof(params));
350     params.d = 8;
351     params.steps = 4;
352     params.splitPoint = 1.0;
353     return params;
354 }
355 
defaultFastCoverParams(void)356 static ZDICT_fastCover_params_t defaultFastCoverParams(void)
357 {
358     ZDICT_fastCover_params_t params;
359     memset(&params, 0, sizeof(params));
360     params.d = 8;
361     params.f = 20;
362     params.steps = 4;
363     params.splitPoint = 0.75; /* different from default splitPoint of cover */
364     params.accel = DEFAULT_ACCEL;
365     return params;
366 }
367 #endif
368 
369 
370 /** parseAdaptParameters() :
371  *  reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr.
372  *  Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized.
373  *  There is no guarantee that any of these values will be updated.
374  *  @return 1 means that parsing was successful,
375  *  @return 0 in case of malformed parameters
376  */
parseAdaptParameters(const char * stringPtr,int * adaptMinPtr,int * adaptMaxPtr)377 static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
378 {
379     for ( ; ;) {
380         if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
381         if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
382         DISPLAYLEVEL(4, "invalid compression parameter \n");
383         return 0;
384     }
385     if (stringPtr[0] != 0) return 0; /* check the end of string */
386     if (*adaptMinPtr > *adaptMaxPtr) {
387         DISPLAYLEVEL(4, "incoherent adaptation limits \n");
388         return 0;
389     }
390     return 1;
391 }
392 
393 
394 /** parseCompressionParameters() :
395  *  reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
396  *  @return 1 means that compression parameters were correct
397  *  @return 0 in case of malformed parameters
398  */
parseCompressionParameters(const char * stringPtr,ZSTD_compressionParameters * params)399 static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
400 {
401     for ( ; ;) {
402         if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
403         if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
404         if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
405         if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
406         if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
407         if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
408         if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
409         if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
410         if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
411         if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
412         if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
413         if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
414         DISPLAYLEVEL(4, "invalid compression parameter \n");
415         return 0;
416     }
417 
418     DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
419     DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
420     if (stringPtr[0] != 0) return 0; /* check the end of string */
421     return 1;
422 }
423 
printVersion(void)424 static void printVersion(void)
425 {
426     DISPLAY(WELCOME_MESSAGE);
427     /* format support */
428     DISPLAYLEVEL(3, "*** supports: zstd");
429 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
430     DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
431 #endif
432 #ifdef ZSTD_GZCOMPRESS
433     DISPLAYLEVEL(3, ", gzip");
434 #endif
435 #ifdef ZSTD_LZ4COMPRESS
436     DISPLAYLEVEL(3, ", lz4");
437 #endif
438 #ifdef ZSTD_LZMACOMPRESS
439     DISPLAYLEVEL(3, ", lzma, xz ");
440 #endif
441     DISPLAYLEVEL(3, "\n");
442     /* posix support */
443 #ifdef _POSIX_C_SOURCE
444     DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
445 #endif
446 #ifdef _POSIX_VERSION
447     DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
448 #endif
449 #ifdef PLATFORM_POSIX_VERSION
450     DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
451 #endif
452 }
453 
454 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
455 
456 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
457 
458 #ifdef ZSTD_NOCOMPRESS
459 /* symbols from compression library are not defined and should not be invoked */
460 # define MINCLEVEL  -50
461 # define MAXCLEVEL   22
462 #else
463 # define MINCLEVEL  ZSTD_minCLevel()
464 # define MAXCLEVEL  ZSTD_maxCLevel()
465 #endif
466 
main(int argCount,const char * argv[])467 int main(int argCount, const char* argv[])
468 {
469     int argNb,
470         followLinks = 0,
471         forceStdout = 0,
472         lastCommand = 0,
473         ldmFlag = 0,
474         main_pause = 0,
475         nbWorkers = 0,
476         adapt = 0,
477         adaptMin = MINCLEVEL,
478         adaptMax = MAXCLEVEL,
479         rsyncable = 0,
480         nextArgumentIsOutFileName = 0,
481         nextArgumentIsMaxDict = 0,
482         nextArgumentIsDictID = 0,
483         nextArgumentsAreFiles = 0,
484         nextEntryIsDictionary = 0,
485         operationResult = 0,
486         separateFiles = 0,
487         setRealTimePrio = 0,
488         singleThread = 0,
489         ultra=0;
490     double compressibility = 0.5;
491     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
492     size_t blockSize = 0;
493     zstd_operation_mode operation = zom_compress;
494     ZSTD_compressionParameters compressionParams;
495     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
496     int cLevelLast = -1000000000;
497     unsigned recursive = 0;
498     unsigned memLimit = 0;
499     const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
500     unsigned filenameIdx = 0;
501     const char* programName = argv[0];
502     const char* outFileName = NULL;
503     const char* dictFileName = NULL;
504     const char* suffix = ZSTD_EXTENSION;
505     unsigned maxDictSize = g_defaultMaxDictSize;
506     unsigned dictID = 0;
507     int dictCLevel = g_defaultDictCLevel;
508     unsigned dictSelect = g_defaultSelectivityLevel;
509 #ifdef UTIL_HAS_CREATEFILELIST
510     const char** extendedFileList = NULL;
511     char* fileNamesBuf = NULL;
512     unsigned fileNamesNb;
513 #endif
514 #ifndef ZSTD_NODICT
515     ZDICT_cover_params_t coverParams = defaultCoverParams();
516     ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
517     dictType dict = fastCover;
518 #endif
519 #ifndef ZSTD_NOBENCH
520     BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
521 #endif
522 
523 
524     /* init */
525     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
526     (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
527     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
528     filenameTable[0] = stdinmark;
529     g_displayOut = stderr;
530     programName = lastNameFromPath(programName);
531 #ifdef ZSTD_MULTITHREAD
532     nbWorkers = 1;
533 #endif
534 
535     /* preset behaviors */
536     if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
537     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
538     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }   /* supports multiple formats */
539     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }  /* behave like zcat, also supports multiple formats */
540     if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); }               /* behave like gzip */
541     if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); }                                                     /* behave like gunzip, also supports multiple formats */
542     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
543     if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }           /* behave like lzma */
544     if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }        /* behave like unlzma, also supports multiple formats */
545     if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }                 /* behave like xz */
546     if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }            /* behave like unxz, also supports multiple formats */
547     if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); }                                       /* behave like lz4 */
548     if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); }                                   /* behave like unlz4, also supports multiple formats */
549     memset(&compressionParams, 0, sizeof(compressionParams));
550 
551     /* init crash handler */
552     FIO_addAbortHandler();
553 
554     /* command switches */
555     for (argNb=1; argNb<argCount; argNb++) {
556         const char* argument = argv[argNb];
557         if(!argument) continue;   /* Protection if argument empty */
558 
559         if (nextArgumentsAreFiles==0) {
560             /* "-" means stdin/stdout */
561             if (!strcmp(argument, "-")){
562                 if (!filenameIdx) {
563                     filenameIdx=1, filenameTable[0]=stdinmark;
564                     outFileName=stdoutmark;
565                     g_displayLevel-=(g_displayLevel==2);
566                     continue;
567             }   }
568 
569             /* Decode commands (note : aggregated commands are allowed) */
570             if (argument[0]=='-') {
571 
572                 if (argument[1]=='-') {
573                     /* long commands (--long-word) */
574                     if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; }   /* only file names allowed from now on */
575                     if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
576                     if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
577                     if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
578                     if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
579                     if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; }
580                     if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
581                     if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
582                     if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
583                     if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
584                     if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
585                     if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
586                     if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
587                     if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
588                     if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
589                     if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
590                     if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
591                     if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
592                     if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }  /* kept available for compatibility with old syntax ; will be removed one day */
593                     if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }  /* kept available for compatibility with old syntax ; will be removed one day */
594                     if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
595                     if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
596                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
597                     if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
598                     if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
599                     if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
600                     if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
601                     if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; }
602 #ifdef ZSTD_GZCOMPRESS
603                     if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
604 #endif
605 #ifdef ZSTD_LZMACOMPRESS
606                     if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression);  continue; }
607                     if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression);  continue; }
608 #endif
609 #ifdef ZSTD_LZ4COMPRESS
610                     if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression);  continue; }
611 #endif
612                     if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
613 
614                     /* long commands with arguments */
615 #ifndef ZSTD_NODICT
616                     if (longCommandWArg(&argument, "--train-cover")) {
617                       operation = zom_train;
618                       if (outFileName == NULL)
619                           outFileName = g_defaultDictName;
620                       dict = cover;
621                       /* Allow optional arguments following an = */
622                       if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
623                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
624                       else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
625                       continue;
626                     }
627                     if (longCommandWArg(&argument, "--train-fastcover")) {
628                       operation = zom_train;
629                       if (outFileName == NULL)
630                           outFileName = g_defaultDictName;
631                       dict = fastCover;
632                       /* Allow optional arguments following an = */
633                       if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
634                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
635                       else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
636                       continue;
637                     }
638                     if (longCommandWArg(&argument, "--train-legacy")) {
639                       operation = zom_train;
640                       if (outFileName == NULL)
641                           outFileName = g_defaultDictName;
642                       dict = legacy;
643                       /* Allow optional arguments following an = */
644                       if (*argument == 0) { continue; }
645                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
646                       else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); }
647                       continue;
648                     }
649 #endif
650                     if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
651                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
652                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
653                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
654                     if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
655                     if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
656                     if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
657                     if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
658                     if (longCommandWArg(&argument, "--long")) {
659                         unsigned ldmWindowLog = 0;
660                         ldmFlag = 1;
661                         /* Parse optional window log */
662                         if (*argument == '=') {
663                             ++argument;
664                             ldmWindowLog = readU32FromChar(&argument);
665                         } else if (*argument != 0) {
666                             /* Invalid character following --long */
667                             CLEAN_RETURN(badusage(programName));
668                         }
669                         /* Only set windowLog if not already set by --zstd */
670                         if (compressionParams.windowLog == 0)
671                             compressionParams.windowLog = ldmWindowLog;
672                         continue;
673                     }
674 #ifndef ZSTD_NOCOMPRESS   /* linking ZSTD_minCLevel() requires compression support */
675                     if (longCommandWArg(&argument, "--fast")) {
676                         /* Parse optional acceleration factor */
677                         if (*argument == '=') {
678                             U32 const maxFast = (U32)-ZSTD_minCLevel();
679                             U32 fastLevel;
680                             ++argument;
681                             fastLevel = readU32FromChar(&argument);
682                             if (fastLevel > maxFast) fastLevel = maxFast;
683                             if (fastLevel) {
684                               dictCLevel = cLevel = -(int)fastLevel;
685                             } else {
686                               CLEAN_RETURN(badusage(programName));
687                             }
688                         } else if (*argument != 0) {
689                             /* Invalid character following --fast */
690                             CLEAN_RETURN(badusage(programName));
691                         } else {
692                             cLevel = -1;  /* default for --fast */
693                         }
694                         continue;
695                     }
696 #endif
697                     /* fall-through, will trigger bad_usage() later on */
698                 }
699 
700                 argument++;
701                 while (argument[0]!=0) {
702                     if (lastCommand) {
703                         DISPLAY("error : command must be followed by argument \n");
704                         CLEAN_RETURN(1);
705                     }
706 #ifndef ZSTD_NOCOMPRESS
707                     /* compression Level */
708                     if ((*argument>='0') && (*argument<='9')) {
709                         dictCLevel = cLevel = readU32FromChar(&argument);
710                         continue;
711                     }
712 #endif
713 
714                     switch(argument[0])
715                     {
716                         /* Display help */
717                     case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0);   /* Version Only */
718                     case 'H':
719                     case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
720 
721                          /* Compress */
722                     case 'z': operation=zom_compress; argument++; break;
723 
724                          /* Decoding */
725                     case 'd':
726 #ifndef ZSTD_NOBENCH
727                             benchParams.mode = BMK_decodeOnly;
728                             if (operation==zom_bench) { argument++; break; }  /* benchmark decode (hidden option) */
729 #endif
730                             operation=zom_decompress; argument++; break;
731 
732                         /* Force stdout, even if stdout==console */
733                     case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
734 
735                         /* Use file content as dictionary */
736                     case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
737 
738                         /* Overwrite */
739                     case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break;
740 
741                         /* Verbose mode */
742                     case 'v': g_displayLevel++; argument++; break;
743 
744                         /* Quiet mode */
745                     case 'q': g_displayLevel--; argument++; break;
746 
747                         /* keep source file (default) */
748                     case 'k': FIO_setRemoveSrcFile(0); argument++; break;
749 
750                         /* Checksum */
751                     case 'C': FIO_setChecksumFlag(2); argument++; break;
752 
753                         /* test compressed file */
754                     case 't': operation=zom_test; argument++; break;
755 
756                         /* destination file name */
757                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
758 
759                         /* limit decompression memory */
760                     case 'M':
761                         argument++;
762                         memLimit = readU32FromChar(&argument);
763                         break;
764                     case 'l': operation=zom_list; argument++; break;
765 #ifdef UTIL_HAS_CREATEFILELIST
766                         /* recursive */
767                     case 'r': recursive=1; argument++; break;
768 #endif
769 
770 #ifndef ZSTD_NOBENCH
771                         /* Benchmark */
772                     case 'b':
773                         operation=zom_bench;
774                         argument++;
775                         break;
776 
777                         /* range bench (benchmark only) */
778                     case 'e':
779                         /* compression Level */
780                         argument++;
781                         cLevelLast = readU32FromChar(&argument);
782                         break;
783 
784                         /* Modify Nb Iterations (benchmark only) */
785                     case 'i':
786                         argument++;
787                         bench_nbSeconds = readU32FromChar(&argument);
788                         break;
789 
790                         /* cut input into blocks (benchmark only) */
791                     case 'B':
792                         argument++;
793                         blockSize = readU32FromChar(&argument);
794                         break;
795 
796                         /* benchmark files separately (hidden option) */
797                     case 'S':
798                         argument++;
799                         separateFiles = 1;
800                         break;
801 
802 #endif   /* ZSTD_NOBENCH */
803 
804                         /* nb of threads (hidden option) */
805                     case 'T':
806                         argument++;
807                         nbWorkers = readU32FromChar(&argument);
808                         break;
809 
810                         /* Dictionary Selection level */
811                     case 's':
812                         argument++;
813                         dictSelect = readU32FromChar(&argument);
814                         break;
815 
816                         /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
817                     case 'p': argument++;
818 #ifndef ZSTD_NOBENCH
819                         if ((*argument>='0') && (*argument<='9')) {
820                             benchParams.additionalParam = (int)readU32FromChar(&argument);
821                         } else
822 #endif
823                             main_pause=1;
824                         break;
825 
826                         /* Select compressibility of synthetic sample */
827                     case 'P':
828                     {   argument++;
829                         compressibility = (double)readU32FromChar(&argument) / 100;
830                     }
831                     break;
832 
833                         /* unknown command */
834                     default : CLEAN_RETURN(badusage(programName));
835                     }
836                 }
837                 continue;
838             }   /* if (argument[0]=='-') */
839 
840             if (nextArgumentIsMaxDict) {  /* kept available for compatibility with old syntax ; will be removed one day */
841                 nextArgumentIsMaxDict = 0;
842                 lastCommand = 0;
843                 maxDictSize = readU32FromChar(&argument);
844                 continue;
845             }
846 
847             if (nextArgumentIsDictID) {  /* kept available for compatibility with old syntax ; will be removed one day */
848                 nextArgumentIsDictID = 0;
849                 lastCommand = 0;
850                 dictID = readU32FromChar(&argument);
851                 continue;
852             }
853 
854         }   /* if (nextArgumentIsAFile==0) */
855 
856         if (nextEntryIsDictionary) {
857             nextEntryIsDictionary = 0;
858             lastCommand = 0;
859             dictFileName = argument;
860             continue;
861         }
862 
863         if (nextArgumentIsOutFileName) {
864             nextArgumentIsOutFileName = 0;
865             lastCommand = 0;
866             outFileName = argument;
867             if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
868             continue;
869         }
870 
871         /* add filename to list */
872         filenameTable[filenameIdx++] = argument;
873     }
874 
875     if (lastCommand) { /* forgotten argument */
876         DISPLAY("error : command must be followed by argument \n");
877         CLEAN_RETURN(1);
878     }
879 
880     /* Welcome message (if verbose) */
881     DISPLAYLEVEL(3, WELCOME_MESSAGE);
882 
883 #ifdef ZSTD_MULTITHREAD
884     if ((nbWorkers==0) && (!singleThread)) {
885         /* automatically set # workers based on # of reported cpus */
886         nbWorkers = UTIL_countPhysicalCores();
887         DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
888     }
889 #else
890     (void)singleThread; (void)nbWorkers;
891 #endif
892 
893 #ifdef UTIL_HAS_CREATEFILELIST
894     g_utilDisplayLevel = g_displayLevel;
895     if (!followLinks) {
896         unsigned u;
897         for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
898             if (UTIL_isLink(filenameTable[u])) {
899                 DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
900             } else {
901                 filenameTable[fileNamesNb++] = filenameTable[u];
902             }
903         }
904         filenameIdx = fileNamesNb;
905     }
906     if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
907         extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
908         if (extendedFileList) {
909             unsigned u;
910             for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
911             free((void*)filenameTable);
912             filenameTable = extendedFileList;
913             filenameIdx = fileNamesNb;
914         }
915     }
916 #else
917     (void)followLinks;
918 #endif
919 
920     if (operation == zom_list) {
921 #ifndef ZSTD_NODECOMPRESS
922         int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
923         CLEAN_RETURN(ret);
924 #else
925         DISPLAY("file information is not supported \n");
926         CLEAN_RETURN(1);
927 #endif
928     }
929 
930     /* Check if benchmark is selected */
931     if (operation==zom_bench) {
932 #ifndef ZSTD_NOBENCH
933         benchParams.blockSize = blockSize;
934         benchParams.nbWorkers = nbWorkers;
935         benchParams.realTime = setRealTimePrio;
936         benchParams.nbSeconds = bench_nbSeconds;
937         benchParams.ldmFlag = ldmFlag;
938         benchParams.ldmMinMatch = g_ldmMinMatch;
939         benchParams.ldmHashLog = g_ldmHashLog;
940         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
941             benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
942         }
943         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
944             benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
945         }
946 
947         if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
948         if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
949         if (cLevelLast < cLevel) cLevelLast = cLevel;
950         if (cLevelLast > cLevel)
951             DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
952         if(filenameIdx) {
953             if(separateFiles) {
954                 unsigned i;
955                 for(i = 0; i < filenameIdx; i++) {
956                     int c;
957                     DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
958                     for(c = cLevel; c <= cLevelLast; c++) {
959                         BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
960                     }
961                 }
962             } else {
963                 for(; cLevel <= cLevelLast; cLevel++) {
964                     BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
965                 }
966             }
967         } else {
968             for(; cLevel <= cLevelLast; cLevel++) {
969                 BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
970             }
971         }
972 
973 #else
974         (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
975 #endif
976         goto _end;
977     }
978 
979     /* Check if dictionary builder is selected */
980     if (operation==zom_train) {
981 #ifndef ZSTD_NODICT
982         ZDICT_params_t zParams;
983         zParams.compressionLevel = dictCLevel;
984         zParams.notificationLevel = g_displayLevel;
985         zParams.dictID = dictID;
986         if (dict == cover) {
987             int const optimize = !coverParams.k || !coverParams.d;
988             coverParams.nbThreads = nbWorkers;
989             coverParams.zParams = zParams;
990             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
991         } else if (dict == fastCover) {
992             int const optimize = !fastCoverParams.k || !fastCoverParams.d;
993             fastCoverParams.nbThreads = nbWorkers;
994             fastCoverParams.zParams = zParams;
995             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
996         } else {
997             ZDICT_legacy_params_t dictParams;
998             memset(&dictParams, 0, sizeof(dictParams));
999             dictParams.selectivityLevel = dictSelect;
1000             dictParams.zParams = zParams;
1001             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
1002         }
1003 #else
1004         (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
1005         DISPLAYLEVEL(1, "training mode not available \n");
1006         operationResult = 1;
1007 #endif
1008         goto _end;
1009     }
1010 
1011 #ifndef ZSTD_NODECOMPRESS
1012     if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); }  /* test mode */
1013 #endif
1014 
1015     /* No input filename ==> use stdin and stdout */
1016     filenameIdx += !filenameIdx;   /* filenameTable[0] is stdin by default */
1017     if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
1018         outFileName = stdoutmark;  /* when input is stdin, default output is stdout */
1019 
1020     /* Check if input/output defined as console; trigger an error in this case */
1021     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
1022         CLEAN_RETURN(badusage(programName));
1023     if ( outFileName && !strcmp(outFileName, stdoutmark)
1024       && IS_CONSOLE(stdout)
1025       && !strcmp(filenameTable[0], stdinmark)
1026       && !forceStdout
1027       && operation!=zom_decompress )
1028         CLEAN_RETURN(badusage(programName));
1029 
1030 #ifndef ZSTD_NOCOMPRESS
1031     /* check compression level limits */
1032     {   int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
1033         if (cLevel > maxCLevel) {
1034             DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
1035             cLevel = maxCLevel;
1036     }   }
1037 #endif
1038 
1039     /* No status message in pipe mode (stdin - stdout) or multi-files mode */
1040     if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
1041     if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
1042 
1043     /* IO Stream/File */
1044     FIO_setNotificationLevel(g_displayLevel);
1045     if (operation==zom_compress) {
1046 #ifndef ZSTD_NOCOMPRESS
1047         FIO_setNbWorkers(nbWorkers);
1048         FIO_setBlockSize((U32)blockSize);
1049         if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
1050         FIO_setLdmFlag(ldmFlag);
1051         FIO_setLdmHashLog(g_ldmHashLog);
1052         FIO_setLdmMinMatch(g_ldmMinMatch);
1053         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
1054         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
1055         FIO_setAdaptiveMode(adapt);
1056         FIO_setAdaptMin(adaptMin);
1057         FIO_setAdaptMax(adaptMax);
1058         FIO_setRsyncable(rsyncable);
1059         if (adaptMin > cLevel) cLevel = adaptMin;
1060         if (adaptMax < cLevel) cLevel = adaptMax;
1061 
1062         if ((filenameIdx==1) && outFileName)
1063           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
1064         else
1065           operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
1066 #else
1067         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
1068         DISPLAY("Compression not supported \n");
1069 #endif
1070     } else {  /* decompression or test */
1071 #ifndef ZSTD_NODECOMPRESS
1072         if (memLimit == 0) {
1073             if (compressionParams.windowLog == 0)
1074                 memLimit = (U32)1 << g_defaultMaxWindowLog;
1075             else {
1076                 memLimit = (U32)1 << (compressionParams.windowLog & 31);
1077             }
1078         }
1079         FIO_setMemLimit(memLimit);
1080         if (filenameIdx==1 && outFileName)
1081             operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
1082         else
1083             operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
1084 #else
1085         DISPLAY("Decompression not supported \n");
1086 #endif
1087     }
1088 
1089 _end:
1090     if (main_pause) waitEnter();
1091 #ifdef UTIL_HAS_CREATEFILELIST
1092     if (extendedFileList)
1093         UTIL_freeFileList(extendedFileList, fileNamesBuf);
1094     else
1095 #endif
1096         free((void*)filenameTable);
1097     return operationResult;
1098 }
1099