1 /*
2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Tuning parameters
14 **************************************/
15 #ifndef ZSTDCLI_CLEVEL_DEFAULT
16 # define ZSTDCLI_CLEVEL_DEFAULT 3
17 #endif
18
19 #ifndef ZSTDCLI_CLEVEL_MAX
20 # define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
21 #endif
22
23
24
25 /*-************************************
26 * Dependencies
27 **************************************/
28 #include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
29 #include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
30 #include <stdio.h> /* fprintf(), stdin, stdout, stderr */
31 #include <string.h> /* strcmp, strlen */
32 #include <errno.h> /* errno */
33 #include "fileio.h" /* stdinmark, stdoutmark, ZSTD_EXTENSION */
34 #ifndef ZSTD_NOBENCH
35 # include "benchzstd.h" /* BMK_benchFiles */
36 #endif
37 #ifndef ZSTD_NODICT
38 # include "dibio.h" /* ZDICT_cover_params_t, DiB_trainFromFiles() */
39 #endif
40 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_minCLevel */
41 #include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */
42
43
44 /*-************************************
45 * Constants
46 **************************************/
47 #define COMPRESSOR_NAME "zstd command line interface"
48 #ifndef ZSTD_VERSION
49 # define ZSTD_VERSION "v" ZSTD_VERSION_STRING
50 #endif
51 #define AUTHOR "Yann Collet"
52 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
53
54 #define ZSTD_ZSTDMT "zstdmt"
55 #define ZSTD_UNZSTD "unzstd"
56 #define ZSTD_CAT "zstdcat"
57 #define ZSTD_ZCAT "zcat"
58 #define ZSTD_GZ "gzip"
59 #define ZSTD_GUNZIP "gunzip"
60 #define ZSTD_GZCAT "gzcat"
61 #define ZSTD_LZMA "lzma"
62 #define ZSTD_UNLZMA "unlzma"
63 #define ZSTD_XZ "xz"
64 #define ZSTD_UNXZ "unxz"
65 #define ZSTD_LZ4 "lz4"
66 #define ZSTD_UNLZ4 "unlz4"
67
68 #define KB *(1 <<10)
69 #define MB *(1 <<20)
70 #define GB *(1U<<30)
71
72 #define DISPLAY_LEVEL_DEFAULT 2
73
74 static const char* g_defaultDictName = "dictionary";
75 static const unsigned g_defaultMaxDictSize = 110 KB;
76 static const int g_defaultDictCLevel = 3;
77 static const unsigned g_defaultSelectivityLevel = 9;
78 static const unsigned g_defaultMaxWindowLog = 27;
79 #define OVERLAP_LOG_DEFAULT 9999
80 #define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
81 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
82 static U32 g_ldmHashLog = 0;
83 static U32 g_ldmMinMatch = 0;
84 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
85 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
86
87
88 #define DEFAULT_ACCEL 1
89
90 typedef enum { cover, fastCover, legacy } dictType;
91
92 /*-************************************
93 * Display Macros
94 **************************************/
95 #define DISPLAY(...) fprintf(g_displayOut, __VA_ARGS__)
96 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
97 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
98 static FILE* g_displayOut;
99
100
101 /*-************************************
102 * Command Line
103 **************************************/
usage(const char * programName)104 static int usage(const char* programName)
105 {
106 DISPLAY( "Usage : \n");
107 DISPLAY( " %s [args] [FILE(s)] [-o file] \n", programName);
108 DISPLAY( "\n");
109 DISPLAY( "FILE : a filename \n");
110 DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
111 DISPLAY( "Arguments : \n");
112 #ifndef ZSTD_NOCOMPRESS
113 DISPLAY( " -# : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
114 #endif
115 #ifndef ZSTD_NODECOMPRESS
116 DISPLAY( " -d : decompression \n");
117 #endif
118 DISPLAY( " -D file: use `file` as Dictionary \n");
119 DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
120 DISPLAY( " -f : overwrite output without prompting and (de)compress links \n");
121 DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
122 DISPLAY( " -k : preserve source file(s) (default) \n");
123 DISPLAY( " -h/-H : display help/long help and exit \n");
124 return 0;
125 }
126
usage_advanced(const char * programName)127 static int usage_advanced(const char* programName)
128 {
129 DISPLAY(WELCOME_MESSAGE);
130 usage(programName);
131 DISPLAY( "\n");
132 DISPLAY( "Advanced arguments : \n");
133 DISPLAY( " -V : display Version number and exit \n");
134 DISPLAY( " -v : verbose mode; specify multiple times to increase verbosity\n");
135 DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
136 DISPLAY( " -c : force write to standard output, even if it is the console\n");
137 DISPLAY( " -l : print information about zstd compressed files \n");
138 #ifndef ZSTD_NOCOMPRESS
139 DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
140 DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
141 DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
142 DISPLAY( "--adapt : dynamically adapt compression level to I/O conditions \n");
143 #ifdef ZSTD_MULTITHREAD
144 DISPLAY( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
145 DISPLAY( " -B# : select size of each job (default: 0==automatic) \n");
146 DISPLAY( " --rsyncable : compress using a rsync-friendly method (-B sets block size) \n");
147 #endif
148 DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
149 DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
150 #endif
151 #ifdef UTIL_HAS_CREATEFILELIST
152 DISPLAY( " -r : operate recursively on directories \n");
153 #endif
154 DISPLAY( "--format=zstd : compress files to the .zst format (default) \n");
155 #ifdef ZSTD_GZCOMPRESS
156 DISPLAY( "--format=gzip : compress files to the .gz format \n");
157 #endif
158 #ifdef ZSTD_LZMACOMPRESS
159 DISPLAY( "--format=xz : compress files to the .xz format \n");
160 DISPLAY( "--format=lzma : compress files to the .lzma format \n");
161 #endif
162 #ifdef ZSTD_LZ4COMPRESS
163 DISPLAY( "--format=lz4 : compress files to the .lz4 format \n");
164 #endif
165 #ifndef ZSTD_NODECOMPRESS
166 DISPLAY( "--test : test compressed file integrity \n");
167 #if ZSTD_SPARSE_DEFAULT
168 DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n");
169 #else
170 DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n");
171 #endif
172 #endif
173 DISPLAY( " -M# : Set a memory usage limit for decompression \n");
174 DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
175 #ifndef ZSTD_NODICT
176 DISPLAY( "\n");
177 DISPLAY( "Dictionary builder : \n");
178 DISPLAY( "--train ## : create a dictionary from a training set of files \n");
179 DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n");
180 DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n");
181 DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
182 DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
183 DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
184 DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n");
185 #endif
186 #ifndef ZSTD_NOBENCH
187 DISPLAY( "\n");
188 DISPLAY( "Benchmark arguments : \n");
189 DISPLAY( " -b# : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
190 DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
191 DISPLAY( " -i# : minimum evaluation time in seconds (default: 3s) \n");
192 DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
193 DISPLAY( "--priority=rt : set process priority to real-time \n");
194 #endif
195 return 0;
196 }
197
badusage(const char * programName)198 static int badusage(const char* programName)
199 {
200 DISPLAYLEVEL(1, "Incorrect parameters\n");
201 if (g_displayLevel >= 2) usage(programName);
202 return 1;
203 }
204
waitEnter(void)205 static void waitEnter(void)
206 {
207 int unused;
208 DISPLAY("Press enter to continue...\n");
209 unused = getchar();
210 (void)unused;
211 }
212
lastNameFromPath(const char * path)213 static const char* lastNameFromPath(const char* path)
214 {
215 const char* name = path;
216 if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
217 if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
218 return name;
219 }
220
221 /*! exeNameMatch() :
222 @return : a non-zero value if exeName matches test, excluding the extension
223 */
exeNameMatch(const char * exeName,const char * test)224 static int exeNameMatch(const char* exeName, const char* test)
225 {
226 return !strncmp(exeName, test, strlen(test)) &&
227 (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
228 }
229
errorOut(const char * msg)230 static void errorOut(const char* msg)
231 {
232 DISPLAY("%s \n", msg); exit(1);
233 }
234
235 /*! readU32FromChar() :
236 * @return : unsigned integer value read from input in `char` format.
237 * allows and interprets K, KB, KiB, M, MB and MiB suffix.
238 * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
239 * Note : function will exit() program if digit sequence overflows */
readU32FromChar(const char ** stringPtr)240 static unsigned readU32FromChar(const char** stringPtr)
241 {
242 const char errorMsg[] = "error: numeric value too large";
243 unsigned result = 0;
244 while ((**stringPtr >='0') && (**stringPtr <='9')) {
245 unsigned const max = (((unsigned)(-1)) / 10) - 1;
246 if (result > max) errorOut(errorMsg);
247 result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
248 }
249 if ((**stringPtr=='K') || (**stringPtr=='M')) {
250 unsigned const maxK = ((unsigned)(-1)) >> 10;
251 if (result > maxK) errorOut(errorMsg);
252 result <<= 10;
253 if (**stringPtr=='M') {
254 if (result > maxK) errorOut(errorMsg);
255 result <<= 10;
256 }
257 (*stringPtr)++; /* skip `K` or `M` */
258 if (**stringPtr=='i') (*stringPtr)++;
259 if (**stringPtr=='B') (*stringPtr)++;
260 }
261 return result;
262 }
263
264 /** longCommandWArg() :
265 * check if *stringPtr is the same as longCommand.
266 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
267 * @return 0 and doesn't modify *stringPtr otherwise.
268 */
longCommandWArg(const char ** stringPtr,const char * longCommand)269 static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
270 {
271 size_t const comSize = strlen(longCommand);
272 int const result = !strncmp(*stringPtr, longCommand, comSize);
273 if (result) *stringPtr += comSize;
274 return result;
275 }
276
277
278 #ifndef ZSTD_NODICT
279 /**
280 * parseCoverParameters() :
281 * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params
282 * @return 1 means that cover parameters were correct
283 * @return 0 in case of malformed parameters
284 */
parseCoverParameters(const char * stringPtr,ZDICT_cover_params_t * params)285 static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params)
286 {
287 memset(params, 0, sizeof(*params));
288 for (; ;) {
289 if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
290 if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
291 if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
292 if (longCommandWArg(&stringPtr, "split=")) {
293 unsigned splitPercentage = readU32FromChar(&stringPtr);
294 params->splitPoint = (double)splitPercentage / 100.0;
295 if (stringPtr[0]==',') { stringPtr++; continue; } else break;
296 }
297 return 0;
298 }
299 if (stringPtr[0] != 0) return 0;
300 DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->steps, (unsigned)(params->splitPoint * 100));
301 return 1;
302 }
303
304 /**
305 * parseFastCoverParameters() :
306 * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params
307 * @return 1 means that fastcover parameters were correct
308 * @return 0 in case of malformed parameters
309 */
parseFastCoverParameters(const char * stringPtr,ZDICT_fastCover_params_t * params)310 static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params)
311 {
312 memset(params, 0, sizeof(*params));
313 for (; ;) {
314 if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
315 if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
316 if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
317 if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
318 if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
319 if (longCommandWArg(&stringPtr, "split=")) {
320 unsigned splitPercentage = readU32FromChar(&stringPtr);
321 params->splitPoint = (double)splitPercentage / 100.0;
322 if (stringPtr[0]==',') { stringPtr++; continue; } else break;
323 }
324 return 0;
325 }
326 if (stringPtr[0] != 0) return 0;
327 DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel);
328 return 1;
329 }
330
331 /**
332 * parseLegacyParameters() :
333 * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
334 * @return 1 means that legacy dictionary builder parameters were correct
335 * @return 0 in case of malformed parameters
336 */
parseLegacyParameters(const char * stringPtr,unsigned * selectivity)337 static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity)
338 {
339 if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; }
340 *selectivity = readU32FromChar(&stringPtr);
341 if (stringPtr[0] != 0) return 0;
342 DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity);
343 return 1;
344 }
345
defaultCoverParams(void)346 static ZDICT_cover_params_t defaultCoverParams(void)
347 {
348 ZDICT_cover_params_t params;
349 memset(¶ms, 0, sizeof(params));
350 params.d = 8;
351 params.steps = 4;
352 params.splitPoint = 1.0;
353 return params;
354 }
355
defaultFastCoverParams(void)356 static ZDICT_fastCover_params_t defaultFastCoverParams(void)
357 {
358 ZDICT_fastCover_params_t params;
359 memset(¶ms, 0, sizeof(params));
360 params.d = 8;
361 params.f = 20;
362 params.steps = 4;
363 params.splitPoint = 0.75; /* different from default splitPoint of cover */
364 params.accel = DEFAULT_ACCEL;
365 return params;
366 }
367 #endif
368
369
370 /** parseAdaptParameters() :
371 * reads adapt parameters from *stringPtr (e.g. "--zstd=min=1,max=19) and store them into adaptMinPtr and adaptMaxPtr.
372 * Both adaptMinPtr and adaptMaxPtr must be already allocated and correctly initialized.
373 * There is no guarantee that any of these values will be updated.
374 * @return 1 means that parsing was successful,
375 * @return 0 in case of malformed parameters
376 */
parseAdaptParameters(const char * stringPtr,int * adaptMinPtr,int * adaptMaxPtr)377 static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
378 {
379 for ( ; ;) {
380 if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
381 if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
382 DISPLAYLEVEL(4, "invalid compression parameter \n");
383 return 0;
384 }
385 if (stringPtr[0] != 0) return 0; /* check the end of string */
386 if (*adaptMinPtr > *adaptMaxPtr) {
387 DISPLAYLEVEL(4, "incoherent adaptation limits \n");
388 return 0;
389 }
390 return 1;
391 }
392
393
394 /** parseCompressionParameters() :
395 * reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
396 * @return 1 means that compression parameters were correct
397 * @return 0 in case of malformed parameters
398 */
parseCompressionParameters(const char * stringPtr,ZSTD_compressionParameters * params)399 static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
400 {
401 for ( ; ;) {
402 if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
403 if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
404 if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
405 if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
406 if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
407 if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
408 if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
409 if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
410 if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
411 if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
412 if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
413 if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
414 DISPLAYLEVEL(4, "invalid compression parameter \n");
415 return 0;
416 }
417
418 DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
419 DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
420 if (stringPtr[0] != 0) return 0; /* check the end of string */
421 return 1;
422 }
423
printVersion(void)424 static void printVersion(void)
425 {
426 DISPLAY(WELCOME_MESSAGE);
427 /* format support */
428 DISPLAYLEVEL(3, "*** supports: zstd");
429 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
430 DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
431 #endif
432 #ifdef ZSTD_GZCOMPRESS
433 DISPLAYLEVEL(3, ", gzip");
434 #endif
435 #ifdef ZSTD_LZ4COMPRESS
436 DISPLAYLEVEL(3, ", lz4");
437 #endif
438 #ifdef ZSTD_LZMACOMPRESS
439 DISPLAYLEVEL(3, ", lzma, xz ");
440 #endif
441 DISPLAYLEVEL(3, "\n");
442 /* posix support */
443 #ifdef _POSIX_C_SOURCE
444 DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
445 #endif
446 #ifdef _POSIX_VERSION
447 DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
448 #endif
449 #ifdef PLATFORM_POSIX_VERSION
450 DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
451 #endif
452 }
453
454 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
455
456 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
457
458 #ifdef ZSTD_NOCOMPRESS
459 /* symbols from compression library are not defined and should not be invoked */
460 # define MINCLEVEL -50
461 # define MAXCLEVEL 22
462 #else
463 # define MINCLEVEL ZSTD_minCLevel()
464 # define MAXCLEVEL ZSTD_maxCLevel()
465 #endif
466
main(int argCount,const char * argv[])467 int main(int argCount, const char* argv[])
468 {
469 int argNb,
470 followLinks = 0,
471 forceStdout = 0,
472 lastCommand = 0,
473 ldmFlag = 0,
474 main_pause = 0,
475 nbWorkers = 0,
476 adapt = 0,
477 adaptMin = MINCLEVEL,
478 adaptMax = MAXCLEVEL,
479 rsyncable = 0,
480 nextArgumentIsOutFileName = 0,
481 nextArgumentIsMaxDict = 0,
482 nextArgumentIsDictID = 0,
483 nextArgumentsAreFiles = 0,
484 nextEntryIsDictionary = 0,
485 operationResult = 0,
486 separateFiles = 0,
487 setRealTimePrio = 0,
488 singleThread = 0,
489 ultra=0;
490 double compressibility = 0.5;
491 unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
492 size_t blockSize = 0;
493 zstd_operation_mode operation = zom_compress;
494 ZSTD_compressionParameters compressionParams;
495 int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
496 int cLevelLast = -1000000000;
497 unsigned recursive = 0;
498 unsigned memLimit = 0;
499 const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
500 unsigned filenameIdx = 0;
501 const char* programName = argv[0];
502 const char* outFileName = NULL;
503 const char* dictFileName = NULL;
504 const char* suffix = ZSTD_EXTENSION;
505 unsigned maxDictSize = g_defaultMaxDictSize;
506 unsigned dictID = 0;
507 int dictCLevel = g_defaultDictCLevel;
508 unsigned dictSelect = g_defaultSelectivityLevel;
509 #ifdef UTIL_HAS_CREATEFILELIST
510 const char** extendedFileList = NULL;
511 char* fileNamesBuf = NULL;
512 unsigned fileNamesNb;
513 #endif
514 #ifndef ZSTD_NODICT
515 ZDICT_cover_params_t coverParams = defaultCoverParams();
516 ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams();
517 dictType dict = fastCover;
518 #endif
519 #ifndef ZSTD_NOBENCH
520 BMK_advancedParams_t benchParams = BMK_initAdvancedParams();
521 #endif
522
523
524 /* init */
525 (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
526 (void)memLimit; /* not used when ZSTD_NODECOMPRESS set */
527 if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
528 filenameTable[0] = stdinmark;
529 g_displayOut = stderr;
530 programName = lastNameFromPath(programName);
531 #ifdef ZSTD_MULTITHREAD
532 nbWorkers = 1;
533 #endif
534
535 /* preset behaviors */
536 if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
537 if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
538 if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
539 if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
540 if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */
541 if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */
542 if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
543 if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */
544 if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */
545 if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */
546 if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */
547 if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */
548 if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */
549 memset(&compressionParams, 0, sizeof(compressionParams));
550
551 /* init crash handler */
552 FIO_addAbortHandler();
553
554 /* command switches */
555 for (argNb=1; argNb<argCount; argNb++) {
556 const char* argument = argv[argNb];
557 if(!argument) continue; /* Protection if argument empty */
558
559 if (nextArgumentsAreFiles==0) {
560 /* "-" means stdin/stdout */
561 if (!strcmp(argument, "-")){
562 if (!filenameIdx) {
563 filenameIdx=1, filenameTable[0]=stdinmark;
564 outFileName=stdoutmark;
565 g_displayLevel-=(g_displayLevel==2);
566 continue;
567 } }
568
569 /* Decode commands (note : aggregated commands are allowed) */
570 if (argument[0]=='-') {
571
572 if (argument[1]=='-') {
573 /* long commands (--long-word) */
574 if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; } /* only file names allowed from now on */
575 if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
576 if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
577 if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
578 if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
579 if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; }
580 if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
581 if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
582 if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
583 if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
584 if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
585 if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
586 if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
587 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
588 if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
589 if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
590 if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
591 if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
592 if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
593 if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
594 if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
595 if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
596 if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
597 if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
598 if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
599 if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
600 if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
601 if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; }
602 #ifdef ZSTD_GZCOMPRESS
603 if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
604 #endif
605 #ifdef ZSTD_LZMACOMPRESS
606 if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; }
607 if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; }
608 #endif
609 #ifdef ZSTD_LZ4COMPRESS
610 if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; }
611 #endif
612 if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
613
614 /* long commands with arguments */
615 #ifndef ZSTD_NODICT
616 if (longCommandWArg(&argument, "--train-cover")) {
617 operation = zom_train;
618 if (outFileName == NULL)
619 outFileName = g_defaultDictName;
620 dict = cover;
621 /* Allow optional arguments following an = */
622 if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
623 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
624 else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
625 continue;
626 }
627 if (longCommandWArg(&argument, "--train-fastcover")) {
628 operation = zom_train;
629 if (outFileName == NULL)
630 outFileName = g_defaultDictName;
631 dict = fastCover;
632 /* Allow optional arguments following an = */
633 if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); }
634 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
635 else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); }
636 continue;
637 }
638 if (longCommandWArg(&argument, "--train-legacy")) {
639 operation = zom_train;
640 if (outFileName == NULL)
641 outFileName = g_defaultDictName;
642 dict = legacy;
643 /* Allow optional arguments following an = */
644 if (*argument == 0) { continue; }
645 else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
646 else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); }
647 continue;
648 }
649 #endif
650 if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
651 if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
652 if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
653 if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
654 if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
655 if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
656 if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
657 if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
658 if (longCommandWArg(&argument, "--long")) {
659 unsigned ldmWindowLog = 0;
660 ldmFlag = 1;
661 /* Parse optional window log */
662 if (*argument == '=') {
663 ++argument;
664 ldmWindowLog = readU32FromChar(&argument);
665 } else if (*argument != 0) {
666 /* Invalid character following --long */
667 CLEAN_RETURN(badusage(programName));
668 }
669 /* Only set windowLog if not already set by --zstd */
670 if (compressionParams.windowLog == 0)
671 compressionParams.windowLog = ldmWindowLog;
672 continue;
673 }
674 #ifndef ZSTD_NOCOMPRESS /* linking ZSTD_minCLevel() requires compression support */
675 if (longCommandWArg(&argument, "--fast")) {
676 /* Parse optional acceleration factor */
677 if (*argument == '=') {
678 U32 const maxFast = (U32)-ZSTD_minCLevel();
679 U32 fastLevel;
680 ++argument;
681 fastLevel = readU32FromChar(&argument);
682 if (fastLevel > maxFast) fastLevel = maxFast;
683 if (fastLevel) {
684 dictCLevel = cLevel = -(int)fastLevel;
685 } else {
686 CLEAN_RETURN(badusage(programName));
687 }
688 } else if (*argument != 0) {
689 /* Invalid character following --fast */
690 CLEAN_RETURN(badusage(programName));
691 } else {
692 cLevel = -1; /* default for --fast */
693 }
694 continue;
695 }
696 #endif
697 /* fall-through, will trigger bad_usage() later on */
698 }
699
700 argument++;
701 while (argument[0]!=0) {
702 if (lastCommand) {
703 DISPLAY("error : command must be followed by argument \n");
704 CLEAN_RETURN(1);
705 }
706 #ifndef ZSTD_NOCOMPRESS
707 /* compression Level */
708 if ((*argument>='0') && (*argument<='9')) {
709 dictCLevel = cLevel = readU32FromChar(&argument);
710 continue;
711 }
712 #endif
713
714 switch(argument[0])
715 {
716 /* Display help */
717 case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0); /* Version Only */
718 case 'H':
719 case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
720
721 /* Compress */
722 case 'z': operation=zom_compress; argument++; break;
723
724 /* Decoding */
725 case 'd':
726 #ifndef ZSTD_NOBENCH
727 benchParams.mode = BMK_decodeOnly;
728 if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */
729 #endif
730 operation=zom_decompress; argument++; break;
731
732 /* Force stdout, even if stdout==console */
733 case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
734
735 /* Use file content as dictionary */
736 case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
737
738 /* Overwrite */
739 case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break;
740
741 /* Verbose mode */
742 case 'v': g_displayLevel++; argument++; break;
743
744 /* Quiet mode */
745 case 'q': g_displayLevel--; argument++; break;
746
747 /* keep source file (default) */
748 case 'k': FIO_setRemoveSrcFile(0); argument++; break;
749
750 /* Checksum */
751 case 'C': FIO_setChecksumFlag(2); argument++; break;
752
753 /* test compressed file */
754 case 't': operation=zom_test; argument++; break;
755
756 /* destination file name */
757 case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
758
759 /* limit decompression memory */
760 case 'M':
761 argument++;
762 memLimit = readU32FromChar(&argument);
763 break;
764 case 'l': operation=zom_list; argument++; break;
765 #ifdef UTIL_HAS_CREATEFILELIST
766 /* recursive */
767 case 'r': recursive=1; argument++; break;
768 #endif
769
770 #ifndef ZSTD_NOBENCH
771 /* Benchmark */
772 case 'b':
773 operation=zom_bench;
774 argument++;
775 break;
776
777 /* range bench (benchmark only) */
778 case 'e':
779 /* compression Level */
780 argument++;
781 cLevelLast = readU32FromChar(&argument);
782 break;
783
784 /* Modify Nb Iterations (benchmark only) */
785 case 'i':
786 argument++;
787 bench_nbSeconds = readU32FromChar(&argument);
788 break;
789
790 /* cut input into blocks (benchmark only) */
791 case 'B':
792 argument++;
793 blockSize = readU32FromChar(&argument);
794 break;
795
796 /* benchmark files separately (hidden option) */
797 case 'S':
798 argument++;
799 separateFiles = 1;
800 break;
801
802 #endif /* ZSTD_NOBENCH */
803
804 /* nb of threads (hidden option) */
805 case 'T':
806 argument++;
807 nbWorkers = readU32FromChar(&argument);
808 break;
809
810 /* Dictionary Selection level */
811 case 's':
812 argument++;
813 dictSelect = readU32FromChar(&argument);
814 break;
815
816 /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
817 case 'p': argument++;
818 #ifndef ZSTD_NOBENCH
819 if ((*argument>='0') && (*argument<='9')) {
820 benchParams.additionalParam = (int)readU32FromChar(&argument);
821 } else
822 #endif
823 main_pause=1;
824 break;
825
826 /* Select compressibility of synthetic sample */
827 case 'P':
828 { argument++;
829 compressibility = (double)readU32FromChar(&argument) / 100;
830 }
831 break;
832
833 /* unknown command */
834 default : CLEAN_RETURN(badusage(programName));
835 }
836 }
837 continue;
838 } /* if (argument[0]=='-') */
839
840 if (nextArgumentIsMaxDict) { /* kept available for compatibility with old syntax ; will be removed one day */
841 nextArgumentIsMaxDict = 0;
842 lastCommand = 0;
843 maxDictSize = readU32FromChar(&argument);
844 continue;
845 }
846
847 if (nextArgumentIsDictID) { /* kept available for compatibility with old syntax ; will be removed one day */
848 nextArgumentIsDictID = 0;
849 lastCommand = 0;
850 dictID = readU32FromChar(&argument);
851 continue;
852 }
853
854 } /* if (nextArgumentIsAFile==0) */
855
856 if (nextEntryIsDictionary) {
857 nextEntryIsDictionary = 0;
858 lastCommand = 0;
859 dictFileName = argument;
860 continue;
861 }
862
863 if (nextArgumentIsOutFileName) {
864 nextArgumentIsOutFileName = 0;
865 lastCommand = 0;
866 outFileName = argument;
867 if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
868 continue;
869 }
870
871 /* add filename to list */
872 filenameTable[filenameIdx++] = argument;
873 }
874
875 if (lastCommand) { /* forgotten argument */
876 DISPLAY("error : command must be followed by argument \n");
877 CLEAN_RETURN(1);
878 }
879
880 /* Welcome message (if verbose) */
881 DISPLAYLEVEL(3, WELCOME_MESSAGE);
882
883 #ifdef ZSTD_MULTITHREAD
884 if ((nbWorkers==0) && (!singleThread)) {
885 /* automatically set # workers based on # of reported cpus */
886 nbWorkers = UTIL_countPhysicalCores();
887 DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
888 }
889 #else
890 (void)singleThread; (void)nbWorkers;
891 #endif
892
893 #ifdef UTIL_HAS_CREATEFILELIST
894 g_utilDisplayLevel = g_displayLevel;
895 if (!followLinks) {
896 unsigned u;
897 for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
898 if (UTIL_isLink(filenameTable[u])) {
899 DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
900 } else {
901 filenameTable[fileNamesNb++] = filenameTable[u];
902 }
903 }
904 filenameIdx = fileNamesNb;
905 }
906 if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
907 extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
908 if (extendedFileList) {
909 unsigned u;
910 for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
911 free((void*)filenameTable);
912 filenameTable = extendedFileList;
913 filenameIdx = fileNamesNb;
914 }
915 }
916 #else
917 (void)followLinks;
918 #endif
919
920 if (operation == zom_list) {
921 #ifndef ZSTD_NODECOMPRESS
922 int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
923 CLEAN_RETURN(ret);
924 #else
925 DISPLAY("file information is not supported \n");
926 CLEAN_RETURN(1);
927 #endif
928 }
929
930 /* Check if benchmark is selected */
931 if (operation==zom_bench) {
932 #ifndef ZSTD_NOBENCH
933 benchParams.blockSize = blockSize;
934 benchParams.nbWorkers = nbWorkers;
935 benchParams.realTime = setRealTimePrio;
936 benchParams.nbSeconds = bench_nbSeconds;
937 benchParams.ldmFlag = ldmFlag;
938 benchParams.ldmMinMatch = g_ldmMinMatch;
939 benchParams.ldmHashLog = g_ldmHashLog;
940 if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
941 benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
942 }
943 if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
944 benchParams.ldmHashEveryLog = g_ldmHashEveryLog;
945 }
946
947 if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
948 if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
949 if (cLevelLast < cLevel) cLevelLast = cLevel;
950 if (cLevelLast > cLevel)
951 DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
952 if(filenameIdx) {
953 if(separateFiles) {
954 unsigned i;
955 for(i = 0; i < filenameIdx; i++) {
956 int c;
957 DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
958 for(c = cLevel; c <= cLevelLast; c++) {
959 BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
960 }
961 }
962 } else {
963 for(; cLevel <= cLevelLast; cLevel++) {
964 BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
965 }
966 }
967 } else {
968 for(; cLevel <= cLevelLast; cLevel++) {
969 BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
970 }
971 }
972
973 #else
974 (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
975 #endif
976 goto _end;
977 }
978
979 /* Check if dictionary builder is selected */
980 if (operation==zom_train) {
981 #ifndef ZSTD_NODICT
982 ZDICT_params_t zParams;
983 zParams.compressionLevel = dictCLevel;
984 zParams.notificationLevel = g_displayLevel;
985 zParams.dictID = dictID;
986 if (dict == cover) {
987 int const optimize = !coverParams.k || !coverParams.d;
988 coverParams.nbThreads = nbWorkers;
989 coverParams.zParams = zParams;
990 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
991 } else if (dict == fastCover) {
992 int const optimize = !fastCoverParams.k || !fastCoverParams.d;
993 fastCoverParams.nbThreads = nbWorkers;
994 fastCoverParams.zParams = zParams;
995 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
996 } else {
997 ZDICT_legacy_params_t dictParams;
998 memset(&dictParams, 0, sizeof(dictParams));
999 dictParams.selectivityLevel = dictSelect;
1000 dictParams.zParams = zParams;
1001 operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0);
1002 }
1003 #else
1004 (void)dictCLevel; (void)dictSelect; (void)dictID; (void)maxDictSize; /* not used when ZSTD_NODICT set */
1005 DISPLAYLEVEL(1, "training mode not available \n");
1006 operationResult = 1;
1007 #endif
1008 goto _end;
1009 }
1010
1011 #ifndef ZSTD_NODECOMPRESS
1012 if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */
1013 #endif
1014
1015 /* No input filename ==> use stdin and stdout */
1016 filenameIdx += !filenameIdx; /* filenameTable[0] is stdin by default */
1017 if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
1018 outFileName = stdoutmark; /* when input is stdin, default output is stdout */
1019
1020 /* Check if input/output defined as console; trigger an error in this case */
1021 if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
1022 CLEAN_RETURN(badusage(programName));
1023 if ( outFileName && !strcmp(outFileName, stdoutmark)
1024 && IS_CONSOLE(stdout)
1025 && !strcmp(filenameTable[0], stdinmark)
1026 && !forceStdout
1027 && operation!=zom_decompress )
1028 CLEAN_RETURN(badusage(programName));
1029
1030 #ifndef ZSTD_NOCOMPRESS
1031 /* check compression level limits */
1032 { int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
1033 if (cLevel > maxCLevel) {
1034 DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
1035 cLevel = maxCLevel;
1036 } }
1037 #endif
1038
1039 /* No status message in pipe mode (stdin - stdout) or multi-files mode */
1040 if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
1041 if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
1042
1043 /* IO Stream/File */
1044 FIO_setNotificationLevel(g_displayLevel);
1045 if (operation==zom_compress) {
1046 #ifndef ZSTD_NOCOMPRESS
1047 FIO_setNbWorkers(nbWorkers);
1048 FIO_setBlockSize((U32)blockSize);
1049 if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
1050 FIO_setLdmFlag(ldmFlag);
1051 FIO_setLdmHashLog(g_ldmHashLog);
1052 FIO_setLdmMinMatch(g_ldmMinMatch);
1053 if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
1054 if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
1055 FIO_setAdaptiveMode(adapt);
1056 FIO_setAdaptMin(adaptMin);
1057 FIO_setAdaptMax(adaptMax);
1058 FIO_setRsyncable(rsyncable);
1059 if (adaptMin > cLevel) cLevel = adaptMin;
1060 if (adaptMax < cLevel) cLevel = adaptMax;
1061
1062 if ((filenameIdx==1) && outFileName)
1063 operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
1064 else
1065 operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
1066 #else
1067 (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
1068 DISPLAY("Compression not supported \n");
1069 #endif
1070 } else { /* decompression or test */
1071 #ifndef ZSTD_NODECOMPRESS
1072 if (memLimit == 0) {
1073 if (compressionParams.windowLog == 0)
1074 memLimit = (U32)1 << g_defaultMaxWindowLog;
1075 else {
1076 memLimit = (U32)1 << (compressionParams.windowLog & 31);
1077 }
1078 }
1079 FIO_setMemLimit(memLimit);
1080 if (filenameIdx==1 && outFileName)
1081 operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
1082 else
1083 operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
1084 #else
1085 DISPLAY("Decompression not supported \n");
1086 #endif
1087 }
1088
1089 _end:
1090 if (main_pause) waitEnter();
1091 #ifdef UTIL_HAS_CREATEFILELIST
1092 if (extendedFileList)
1093 UTIL_freeFileList(extendedFileList, fileNamesBuf);
1094 else
1095 #endif
1096 free((void*)filenameTable);
1097 return operationResult;
1098 }
1099