1#!/bin/sh 2 3set -e 4 5die() { 6 println "$@" 1>&2 7 exit 1 8} 9 10datagen() { 11 "$DATAGEN_BIN" "$@" 12} 13 14zstd() { 15 if [ -z "$EXEC_PREFIX" ]; then 16 "$ZSTD_BIN" "$@" 17 else 18 "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 19 fi 20} 21 22sudoZstd() { 23 if [ -z "$EXEC_PREFIX" ]; then 24 sudo "$ZSTD_BIN" "$@" 25 else 26 sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 27 fi 28} 29 30roundTripTest() { 31 if [ -n "$3" ]; then 32 cLevel="$3" 33 proba="$2" 34 else 35 cLevel="$2" 36 proba="" 37 fi 38 if [ -n "$4" ]; then 39 dLevel="$4" 40 else 41 dLevel="$cLevel" 42 fi 43 44 rm -f tmp1 tmp2 45 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" 46 datagen $1 $proba | $MD5SUM > tmp1 47 datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 48 $DIFF -q tmp1 tmp2 49} 50 51fileRoundTripTest() { 52 if [ -n "$3" ]; then 53 local_c="$3" 54 local_p="$2" 55 else 56 local_c="$2" 57 local_p="" 58 fi 59 if [ -n "$4" ]; then 60 local_d="$4" 61 else 62 local_d="$local_c" 63 fi 64 65 rm -f tmp.zst tmp.md5.1 tmp.md5.2 66 println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" 67 datagen $1 $local_p > tmp 68 < tmp $MD5SUM > tmp.md5.1 69 zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 70 $DIFF -q tmp.md5.1 tmp.md5.2 71} 72 73truncateLastByte() { 74 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" 75} 76 77println() { 78 printf '%b\n' "${*}" 79} 80 81if [ -z "${size}" ]; then 82 size= 83else 84 size=${size} 85fi 86 87SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 88PRGDIR="$SCRIPT_DIR/../programs" 89TESTDIR="$SCRIPT_DIR/../tests" 90UNAME=$(uname) 91ZSTDGREP="$PRGDIR/zstdgrep" 92 93detectedTerminal=false 94if [ -t 0 ] && [ -t 1 ] 95then 96 detectedTerminal=true 97fi 98isTerminal=${isTerminal:-$detectedTerminal} 99 100isWindows=false 101INTOVOID="/dev/null" 102case "$UNAME" in 103 GNU) DEVDEVICE="/dev/random" ;; 104 *) DEVDEVICE="/dev/zero" ;; 105esac 106case "$OS" in 107 Windows*) 108 isWindows=true 109 INTOVOID="NUL" 110 DEVDEVICE="NUL" 111 ;; 112esac 113 114case "$UNAME" in 115 Darwin) MD5SUM="md5 -r" ;; 116 FreeBSD) MD5SUM="gmd5sum" ;; 117 NetBSD) MD5SUM="md5 -n" ;; 118 OpenBSD) MD5SUM="md5" ;; 119 *) MD5SUM="md5sum" ;; 120esac 121 122MTIME="stat -c %Y" 123case "$UNAME" in 124 Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; 125esac 126 127DIFF="diff" 128case "$UNAME" in 129 SunOS) DIFF="gdiff" ;; 130esac 131 132 133# check if ZSTD_BIN is defined. if not, use the default value 134if [ -z "${ZSTD_BIN}" ]; then 135 println "\nZSTD_BIN is not set. Using the default value..." 136 ZSTD_BIN="$PRGDIR/zstd" 137fi 138 139# check if DATAGEN_BIN is defined. if not, use the default value 140if [ -z "${DATAGEN_BIN}" ]; then 141 println "\nDATAGEN_BIN is not set. Using the default value..." 142 DATAGEN_BIN="$TESTDIR/datagen" 143fi 144 145ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" 146 147# assertions 148[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md" 149[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " 150println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" 151 152if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' 153then 154 hasMT="" 155else 156 hasMT="true" 157fi 158 159 160 161println "\n===> simple tests " 162 163datagen > tmp 164println "test : basic compression " 165zstd -f tmp # trivial compression case, creates tmp.zst 166println "test : basic decompression" 167zstd -df tmp.zst # trivial decompression case (overwrites tmp) 168println "test : too large compression level => auto-fix" 169zstd -99 -f tmp # too large compression level, automatic sized down 170zstd -5000000000 -f tmp && die "too large numeric value : must fail" 171println "test : --fast aka negative compression levels" 172zstd --fast -f tmp # == -1 173zstd --fast=3 -f tmp # == -3 174zstd --fast=200000 -f tmp # too low compression level, automatic fixed 175zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" 176zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" 177println "test : too large numeric argument" 178zstd --fast=9999999999 -f tmp && die "should have refused numeric value" 179println "test : set compression level with environment variable ZSTD_CLEVEL" 180ZSTD_CLEVEL=12 zstd -f tmp # positive compression level 181ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level 182ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign 183ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting 184ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting 185ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting 186ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting 187ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting 188ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting 189println "test : override ZSTD_CLEVEL with command line option" 190ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option 191println "test : compress to stdout" 192zstd tmp -c > tmpCompressed 193zstd tmp --stdout > tmpCompressed # long command format 194println "test : compress to named file" 195rm tmpCompressed 196zstd tmp -o tmpCompressed 197test -f tmpCompressed # file must be created 198println "test : force write, correct order" 199zstd tmp -fo tmpCompressed 200println "test : forgotten argument" 201cp tmp tmp2 202zstd tmp2 -fo && die "-o must be followed by filename " 203println "test : implied stdout when input is stdin" 204println bob | zstd | zstd -d 205if [ "$isTerminal" = true ]; then 206println "test : compressed data to terminal" 207println bob | zstd && die "should have refused : compressed data to terminal" 208println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" 209zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" 210fi 211println "test : null-length file roundtrip" 212println -n '' | zstd - --stdout | zstd -d --stdout 213println "test : ensure small file doesn't add 3-bytes null block" 214datagen -g1 > tmp1 215zstd tmp1 -c | wc -c | grep "14" 216zstd < tmp1 | wc -c | grep "14" 217println "test : decompress file with wrong suffix (must fail)" 218zstd -d tmpCompressed && die "wrong suffix error not detected!" 219zstd -df tmp && die "should have refused : wrong extension" 220println "test : decompress into stdout" 221zstd -d tmpCompressed -c > tmpResult # decompression using stdout 222zstd --decompress tmpCompressed -c > tmpResult 223zstd --decompress tmpCompressed --stdout > tmpResult 224println "test : decompress from stdin into stdout" 225zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout 226zstd -dc - < tmp.zst > $INTOVOID 227zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used 228zstd -d - < tmp.zst > $INTOVOID 229println "test : impose memory limitation (must fail)" 230zstd -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" 231zstd -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 232zstd -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 233zstd -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 234println "test : overwrite protection" 235zstd -q tmp && die "overwrite check failed!" 236println "test : force overwrite" 237zstd -q -f tmp 238zstd -q --force tmp 239println "test : overwrite readonly file" 240rm -f tmpro tmpro.zst 241println foo > tmpro.zst 242println foo > tmpro 243chmod 400 tmpro.zst 244zstd -q tmpro && die "should have refused to overwrite read-only file" 245zstd -q -f tmpro 246println "test: --no-progress flag" 247zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress 248zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress 249rm -f tmpro tmpro.zst 250println "test: overwrite input file (must fail)" 251zstd tmp -fo tmp && die "zstd compression overwrote the input file" 252zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" 253println "test: detect that input file does not exist" 254zstd nothere && die "zstd hasn't detected that input file does not exist" 255println "test: --[no-]compress-literals" 256zstd tmp -c --no-compress-literals -1 | zstd -t 257zstd tmp -c --no-compress-literals --fast=1 | zstd -t 258zstd tmp -c --no-compress-literals -19 | zstd -t 259zstd tmp -c --compress-literals -1 | zstd -t 260zstd tmp -c --compress-literals --fast=1 | zstd -t 261zstd tmp -c --compress-literals -19 | zstd -t 262zstd -b --fast=1 -i0e1 tmp --compress-literals 263zstd -b --fast=1 -i0e1 tmp --no-compress-literals 264println "test: --no-check for decompression" 265zstd -f tmp -o tmp_corrupt.zst --check 266zstd -f tmp -o tmp.zst --no-check 267printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp 268zstd -d -f tmp_corrupt.zst --no-check 269zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides 270zstd -d -f tmp.zst --no-check 271 272println "\n===> zstdgrep tests" 273ln -sf "$ZSTD_BIN" zstdcat 274rm -f tmp_grep 275echo "1234" > tmp_grep 276zstd -f tmp_grep 277lines=$(ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) 278test 2 -eq $lines 279ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" 280ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true 281rm -f tmp_grep* 282 283println "\n===> --exclude-compressed flag" 284rm -rf precompressedFilterTestDir 285mkdir -p precompressedFilterTestDir 286datagen $size > precompressedFilterTestDir/input.5 287datagen $size > precompressedFilterTestDir/input.6 288zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 289datagen $size > precompressedFilterTestDir/input.7 290datagen $size > precompressedFilterTestDir/input.8 291zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 292test ! -f precompressedFilterTestDir/input.5.zst.zst 293test ! -f precompressedFilterTestDir/input.6.zst.zst 294file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` 295file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` 296if [ $file2timestamp -ge $file1timestamp ]; then 297 println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." 298else 299 println "Test is not successful" 300fi 301# File Extension check. 302datagen $size > precompressedFilterTestDir/input.zstbar 303zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 304# zstd should compress input.zstbar 305test -f precompressedFilterTestDir/input.zstbar.zst 306# Check without the --exclude-compressed flag 307zstd --long --rm -r precompressedFilterTestDir 308# Files should get compressed again without the --exclude-compressed flag. 309test -f precompressedFilterTestDir/input.5.zst.zst 310test -f precompressedFilterTestDir/input.6.zst.zst 311println "Test completed" 312 313 314 315println "\n===> warning prompts should not occur if stdin is an input" 316println "y" > tmpPrompt 317println "hello world" >> tmpPrompt 318zstd tmpPrompt -f 319zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 320zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f 321zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated 322$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed 323 324echo 'yes' | zstd tmpPrompt -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files 325echo 'yes' | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 326zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" 327 328println "Test completed" 329 330 331println "\n===> recursive mode test " 332# combination of -r with empty list of input file 333zstd -c -r < tmp > tmp.zst 334 335 336println "\n===> file removal" 337zstd -f --rm tmp 338test ! -f tmp # tmp should no longer be present 339zstd -f -d --rm tmp.zst 340test ! -f tmp.zst # tmp.zst should no longer be present 341println "test : should quietly not remove non-regular file" 342println hello > tmp 343zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" 344grep -v "Refusing to remove non-regular file" tmplog 345rm -f tmplog 346zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" 347println "test : --rm on stdin" 348println a | zstd --rm > $INTOVOID # --rm should remain silent 349rm tmp 350zstd -f tmp && die "tmp not present : should have failed" 351test ! -f tmp.zst # tmp.zst should not be created 352println "test : -d -f do not delete destination when source is not present" 353touch tmp # create destination file 354zstd -d -f tmp.zst && die "attempt to decompress a non existing file" 355test -f tmp # destination file should still be present 356println "test : -f do not delete destination when source is not present" 357rm tmp # erase source file 358touch tmp.zst # create destination file 359zstd -f tmp && die "attempt to compress a non existing file" 360test -f tmp.zst # destination file should still be present 361rm -rf tmp* # may also erase tmp* directory from previous failed run 362 363 364println "\n===> decompression only tests " 365# the following test verifies that the decoder is compatible with RLE as first block 366# older versions of zstd cli are not able to decode such corner case. 367# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. 368dd bs=1048576 count=1 if=/dev/zero of=tmp 369zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" 370$DIFF -s tmp1 tmp 371rm tmp* 372 373 374println "\n===> compress multiple files" 375println hello > tmp1 376println world > tmp2 377zstd tmp1 tmp2 -o "$INTOVOID" -f 378zstd tmp1 tmp2 -c | zstd -t 379zstd tmp1 tmp2 -o tmp.zst 380test ! -f tmp1.zst 381test ! -f tmp2.zst 382zstd tmp1 tmp2 383zstd -t tmp1.zst tmp2.zst 384zstd -dc tmp1.zst tmp2.zst 385zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f 386zstd -d tmp1.zst tmp2.zst -o tmp 387touch tmpexists 388zstd tmp1 tmp2 -f -o tmpexists 389zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" 390println gooder > tmp_rm1 391println boi > tmp_rm2 392println worldly > tmp_rm3 393echo 'y' | zstd tmp_rm1 tmp_rm2 -o tmp_rm3.zst --rm # tests the warning prompt for --rm with multiple inputs into once source 394test ! -f tmp_rm1 395test ! -f tmp_rm2 396cp tmp_rm3.zst tmp_rm4.zst 397echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -o tmp_rm_out --rm 398test ! -f tmp_rm3.zst 399test ! -f tmp_rm4.zst 400echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 401echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 402println gooder > tmpexists1 403zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID 404 405# Bug: PR #972 406if [ "$?" -eq 139 ]; then 407 die "should not have segfaulted" 408fi 409println "\n===> multiple files and shell completion " 410datagen -s1 > tmp1 2> $INTOVOID 411datagen -s2 -g100K > tmp2 2> $INTOVOID 412datagen -s3 -g1M > tmp3 2> $INTOVOID 413println "compress tmp* : " 414zstd -f tmp* 415test -f tmp1.zst 416test -f tmp2.zst 417test -f tmp3.zst 418rm tmp1 tmp2 tmp3 419println "decompress tmp* : " 420zstd -df ./*.zst 421test -f tmp1 422test -f tmp2 423test -f tmp3 424println "compress tmp* into stdout > tmpall : " 425zstd -c tmp1 tmp2 tmp3 > tmpall 426test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) 427println "decompress tmpall* into stdout > tmpdec : " 428cp tmpall tmpall2 429zstd -dc tmpall* > tmpdec 430test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) 431println "compress multiple files including a missing one (notHere) : " 432zstd -f tmp1 notHere tmp2 && die "missing file not detected!" 433rm tmp* 434 435 436if [ "$isWindows" = false ] ; then 437 println "\n===> zstd fifo named pipe test " 438 echo "Hello World!" > tmp_original 439 mkfifo tmp_named_pipe 440 # note : fifo test doesn't work in combination with `dd` or `cat` 441 echo "Hello World!" > tmp_named_pipe & 442 zstd tmp_named_pipe -o tmp_compressed 443 zstd -d -o tmp_decompressed tmp_compressed 444 $DIFF -s tmp_original tmp_decompressed 445 rm -rf tmp* 446fi 447 448 449if [ -n "$DEVNULLRIGHTS" ] ; then 450 # these tests requires sudo rights, which is uncommon. 451 # they are only triggered if DEVNULLRIGHTS macro is defined. 452 println "\n===> checking /dev/null permissions are unaltered " 453 datagen > tmp 454 sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions 455 sudoZstd tmp -c > $INTOVOID 456 zstd tmp -f -o tmp.zst 457 sudoZstd -d tmp.zst -c > $INTOVOID 458 sudoZstd -d tmp.zst -o $INTOVOID 459 ls -las $INTOVOID | grep "rw-rw-rw-" 460fi 461 462 463println "\n===> compress multiple files into an output directory, --output-dir-flat" 464println henlo > tmp1 465mkdir tmpInputTestDir 466mkdir tmpInputTestDir/we 467mkdir tmpInputTestDir/we/must 468mkdir tmpInputTestDir/we/must/go 469mkdir tmpInputTestDir/we/must/go/deeper 470println cool > tmpInputTestDir/we/must/go/deeper/tmp2 471mkdir tmpOutDir 472zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir 473test -f tmpOutDir/tmp1.zst 474test -f tmpOutDir/tmp2.zst 475println "test : decompress multiple files into an output directory, --output-dir-flat" 476mkdir tmpOutDirDecomp 477zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp 478test -f tmpOutDirDecomp/tmp2 479test -f tmpOutDirDecomp/tmp1 480rm -f tmpOutDirDecomp/* 481zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp 482test -f tmpOutDirDecomp/tmp2 483test -f tmpOutDirDecomp/tmp1 484rm -rf tmp* 485 486if [ "$isWindows" = false ] ; then 487 println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" 488 println "test --output-dir-mirror" > tmp1 489 mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. 490 println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 491 zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir 492 test -f tmpOutDir/tmp1.zst 493 test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst 494 495 println "test: compress input dir will be ignored if it has '..'" 496 zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 497 zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" 498 zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 499 test ! -d non-exist 500 501 println "test: compress input dir should succeed with benign uses of '..'" 502 zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout 503 test -d tmpout 504 505 println "test : decompress multiple files into an output directory, --output-dir-mirror" 506 zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp 507 test -f tmpOutDirDecomp/tmpOutDir/tmp1 508 test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 509 510 println "test: decompress input dir will be ignored if it has '..'" 511 zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" 512 test ! -d non-exist 513 514 rm -rf tmp* 515fi 516 517 518println "test : compress multiple files reading them from a file, --filelist=FILE" 519println "Hello world!, file1" > tmp1 520println "Hello world!, file2" > tmp2 521println tmp1 > tmp_fileList 522println tmp2 >> tmp_fileList 523zstd -f --filelist=tmp_fileList 524test -f tmp2.zst 525test -f tmp1.zst 526 527println "test : alternate syntax: --filelist FILE" 528zstd -f --filelist tmp_fileList 529test -f tmp2.zst 530test -f tmp1.zst 531 532println "test : reading file list from a symlink, --filelist=FILE" 533rm -f *.zst 534ln -s tmp_fileList tmp_symLink 535zstd -f --filelist=tmp_symLink 536test -f tmp2.zst 537test -f tmp1.zst 538 539println "test : compress multiple files reading them from multiple files, --filelist=FILE" 540rm -f *.zst 541println "Hello world!, file3" > tmp3 542println "Hello world!, file4" > tmp4 543println tmp3 > tmp_fileList2 544println tmp4 >> tmp_fileList2 545zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 546test -f tmp1.zst 547test -f tmp2.zst 548test -f tmp3.zst 549test -f tmp4.zst 550 551println "test : decompress multiple files reading them from a file, --filelist=FILE" 552rm -f tmp1 tmp2 553println tmp1.zst > tmpZst 554println tmp2.zst >> tmpZst 555zstd -d -f --filelist=tmpZst 556test -f tmp1 557test -f tmp2 558 559println "test : decompress multiple files reading them from multiple files, --filelist=FILE" 560rm -f tmp1 tmp2 tmp3 tmp4 561println tmp3.zst > tmpZst2 562println tmp4.zst >> tmpZst2 563zstd -d -f --filelist=tmpZst --filelist=tmpZst2 564test -f tmp1 565test -f tmp2 566test -f tmp3 567test -f tmp4 568 569println "test : survive a list of files which is text garbage (--filelist=FILE)" 570datagen > tmp_badList 571zstd -f --filelist=tmp_badList && die "should have failed : list is text garbage" 572 573println "test : survive a list of files which is binary garbage (--filelist=FILE)" 574datagen -P0 -g1M > tmp_badList 575zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console 576 577println "test : try to overflow internal list of files (--filelist=FILE)" 578touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 579ls tmp* > tmpList 580zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list 581rm -rf tmp* 582 583println "\n===> --[no-]content-size tests" 584 585datagen > tmp_contentsize 586zstd -f tmp_contentsize 587zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 588zstd -f --no-content-size tmp_contentsize 589zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 590zstd -f --content-size tmp_contentsize 591zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 592zstd -f --content-size --no-content-size tmp_contentsize 593zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 594rm -rf tmp* 595 596println "test : show-default-cparams regular" 597datagen > tmp 598zstd --show-default-cparams -f tmp 599rm -rf tmp* 600 601println "test : show-default-cparams recursive" 602mkdir tmp_files 603datagen -g15000 > tmp_files/tmp1 604datagen -g129000 > tmp_files/tmp2 605datagen -g257000 > tmp_files/tmp3 606zstd --show-default-cparams -f -r tmp_files 607rm -rf tmp* 608 609println "\n===> Advanced compression parameters " 610println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" 611println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" 612println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" 613println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist 614test ! -f tmp.zst # tmp.zst should not be created 615roundTripTest -g512K 616roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" 617roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" 618roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" 619roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" 620roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" 621roundTripTest -g64K "19 --zstd=strat=9" # btultra2 622 623 624println "\n===> Pass-Through mode " 625println "Hello world 1!" | zstd -df 626println "Hello world 2!" | zstd -dcf 627println "Hello world 3!" > tmp1 628zstd -dcf tmp1 629 630 631println "\n===> frame concatenation " 632println "hello " > hello.tmp 633println "world!" > world.tmp 634cat hello.tmp world.tmp > helloworld.tmp 635zstd -c hello.tmp > hello.zst 636zstd -c world.tmp > world.zst 637cat hello.zst world.zst > helloworld.zst 638zstd -dc helloworld.zst > result.tmp 639cat result.tmp 640$DIFF helloworld.tmp result.tmp 641println "frame concatenation without checksum" 642zstd -c hello.tmp > hello.zst --no-check 643zstd -c world.tmp > world.zst --no-check 644cat hello.zst world.zst > helloworld.zstd 645zstd -dc helloworld.zst > result.tmp 646$DIFF helloworld.tmp result.tmp 647println "testing zstdcat symlink" 648ln -sf "$ZSTD_BIN" zstdcat 649$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp 650$DIFF helloworld.tmp result.tmp 651ln -s helloworld.zst helloworld.link.zst 652$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp 653$DIFF helloworld.tmp result.tmp 654rm zstdcat 655rm result.tmp 656println "testing zcat symlink" 657ln -sf "$ZSTD_BIN" zcat 658$EXE_PREFIX ./zcat helloworld.zst > result.tmp 659$DIFF helloworld.tmp result.tmp 660$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp 661$DIFF helloworld.tmp result.tmp 662rm zcat 663rm ./*.tmp ./*.zstd 664println "frame concatenation tests completed" 665 666 667if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then 668println "\n**** flush write error test **** " 669 670println "println foo | zstd > /dev/full" 671println foo | zstd > /dev/full && die "write error not detected!" 672println "println foo | zstd | zstd -d > /dev/full" 673println foo | zstd | zstd -d > /dev/full && die "write error not detected!" 674 675fi 676 677 678if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then 679 680println "\n===> symbolic link test " 681 682rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 683println "hello world" > hello.tmp 684ln -s hello.tmp world.tmp 685ln -s hello.tmp world2.tmp 686zstd world.tmp hello.tmp || true 687test -f hello.tmp.zst # regular file should have been compressed! 688test ! -f world.tmp.zst # symbolic link should not have been compressed! 689zstd world.tmp || true 690test ! -f world.tmp.zst # symbolic link should not have been compressed! 691zstd world.tmp world2.tmp || true 692test ! -f world.tmp.zst # symbolic link should not have been compressed! 693test ! -f world2.tmp.zst # symbolic link should not have been compressed! 694zstd world.tmp hello.tmp -f 695test -f world.tmp.zst # symbolic link should have been compressed with --force 696rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 697 698fi 699 700 701println "\n===> test sparse file support " 702 703datagen -g5M -P100 > tmpSparse 704zstd tmpSparse -c | zstd -dv -o tmpSparseRegen 705$DIFF -s tmpSparse tmpSparseRegen 706zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse 707$DIFF -s tmpSparse tmpOutSparse 708zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse 709$DIFF -s tmpSparse tmpOutNoSparse 710ls -ls tmpSparse* # look at file size and block size on disk 711datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) 712datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd 713ls -ls tmpSparseOdd # look at file size and block size on disk 714println "\n Sparse Compatibility with Console :" 715println "Hello World 1 !" | zstd | zstd -d -c 716println "Hello World 2 !" | zstd | zstd -d | cat 717println "\n Sparse Compatibility with Append :" 718datagen -P100 -g1M > tmpSparse1M 719cat tmpSparse1M tmpSparse1M > tmpSparse2M 720zstd -v -f tmpSparse1M -o tmpSparseCompressed 721zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated 722zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated 723ls -ls tmpSparse* # look at file size and block size on disk 724$DIFF tmpSparse2M tmpSparseRegenerated 725rm tmpSparse* 726 727 728println "\n===> stream-size mode" 729 730datagen -g11000 > tmp 731println "test : basic file compression vs sized streaming compression" 732file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 733stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) 734if [ "$stream_size" -gt "$file_size" ]; then 735 die "hinted compression larger than expected" 736fi 737println "test : sized streaming compression and decompression" 738cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 739zstd -df tmp.zst -o tmp_decompress 740cmp tmp tmp_decompress || die "difference between original and decompressed file" 741println "test : incorrect stream size" 742cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" 743 744println "\n===> zstd zero weight dict test " 745rm -f tmp* 746cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input 747zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 748zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 749$DIFF tmp_decomp tmp_input 750rm -rf tmp* 751 752println "\n===> zstd (valid) zero weight dict test " 753rm -f tmp* 754# 0 has a non-zero weight in the dictionary 755echo "0000000000000000000000000" > tmp_input 756zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 757zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 758$DIFF tmp_decomp tmp_input 759rm -rf tmp* 760 761println "\n===> size-hint mode" 762 763datagen -g11000 > tmp 764datagen -g11000 > tmp2 765datagen > tmpDict 766println "test : basic file compression vs hinted streaming compression" 767file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 768stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) 769if [ "$stream_size" -ge "$file_size" ]; then 770 die "hinted compression larger than expected" 771fi 772println "test : hinted streaming compression and decompression" 773cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 774zstd -df tmp.zst -o tmp_decompress 775cmp tmp tmp_decompress || die "difference between original and decompressed file" 776println "test : hinted streaming compression with dictionary" 777cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict 778println "test : multiple file compression with hints and dictionary" 779zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 780zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp 781zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 782cmp tmp.zst tmp1_.zst || die "first file's output differs" 783cmp tmp2.zst tmp2_.zst || die "second file's output differs" 784println "test : incorrect hinted stream sizes" 785cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high 786cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low 787cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high 788cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low 789 790 791println "\n===> dictionary tests " 792 793println "- test with raw dict (content only) " 794datagen > tmpDict 795datagen -g1M | $MD5SUM > tmp1 796datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 797$DIFF -q tmp1 tmp2 798println "- Create first dictionary " 799TESTFILE="$PRGDIR"/zstdcli.c 800zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 801cp "$TESTFILE" tmp 802println "- Test dictionary compression with tmpDict as an input file and dictionary" 803zstd -f tmpDict -D tmpDict && die "compression error not detected!" 804println "- Dictionary compression roundtrip" 805zstd -f tmp -D tmpDict 806zstd -d tmp.zst -D tmpDict -fo result 807$DIFF "$TESTFILE" result 808println "- Dictionary compression with btlazy2 strategy" 809zstd -f tmp -D tmpDict --zstd=strategy=6 810zstd -d tmp.zst -D tmpDict -fo result 811$DIFF "$TESTFILE" result 812if [ -n "$hasMT" ] 813then 814 println "- Test dictionary compression with multithreading " 815 datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 816fi 817println "- Create second (different) dictionary " 818zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 819zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 820println "- Create dictionary with short dictID" 821zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 822cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 823println "- Create dictionary with wrong dictID parameter order (must fail)" 824zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " 825println "- Create dictionary with size limit" 826zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v 827println "- Create dictionary with small size limit" 828zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v 829println "- Create dictionary with wrong parameter order (must fail)" 830zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " 831println "- Compress without dictID" 832zstd -f tmp -D tmpDict1 --no-dictID 833zstd -d tmp.zst -D tmpDict -fo result 834$DIFF "$TESTFILE" result 835println "- Compress multiple files with dictionary" 836rm -rf dirTestDict 837mkdir dirTestDict 838cp "$TESTDIR"/*.c dirTestDict 839cp "$PRGDIR"/*.c dirTestDict 840cp "$PRGDIR"/*.h dirTestDict 841$MD5SUM dirTestDict/* > tmph1 842zstd -f --rm dirTestDict/* -D tmpDictC 843zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default 844case "$UNAME" in 845 Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 846 *) $MD5SUM -c tmph1 ;; 847esac 848rm -rf dirTestDict 849println "- dictionary builder on bogus input" 850println "Hello World" > tmp 851zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" 852datagen -P0 -g10M > tmp 853zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" 854println "- Test -o before --train" 855rm -f tmpDict dictionary 856zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c 857test -f tmpDict 858zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c 859test -f dictionary 860println "- Test dictionary training fails" 861echo "000000000000000000000000000000000" > tmpz 862zstd --train tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 863if [ -n "$hasMT" ] 864then 865 zstd --train -T0 tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 866 println "- Create dictionary with multithreading enabled" 867 zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 868fi 869rm tmp* dictionary 870 871 872println "\n===> fastCover dictionary builder : advanced options " 873TESTFILE="$PRGDIR"/zstdcli.c 874datagen > tmpDict 875println "- Create first dictionary" 876zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 877cp "$TESTFILE" tmp 878zstd -f tmp -D tmpDict 879zstd -d tmp.zst -D tmpDict -fo result 880$DIFF "$TESTFILE" result 881println "- Create second (different) dictionary" 882zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 883zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 884zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" 885println "- Create dictionary with short dictID" 886zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 887cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 888println "- Create dictionaries with shrink-dict flag enabled" 889zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict 890zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 891zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 892println "- Create dictionary with size limit" 893zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 894println "- Create dictionary using all samples for both training and testing" 895zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 896println "- Create dictionary using f=16" 897zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 898zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" 899println "- Create dictionary using accel=2" 900zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 901println "- Create dictionary using accel=10" 902zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 903println "- Create dictionary with multithreading" 904zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 905println "- Test -o before --train-fastcover" 906rm -f tmpDict dictionary 907zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 908test -f tmpDict 909zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 910test -f dictionary 911rm tmp* dictionary 912 913 914println "\n===> legacy dictionary builder " 915 916TESTFILE="$PRGDIR"/zstdcli.c 917datagen > tmpDict 918println "- Create first dictionary" 919zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 920cp "$TESTFILE" tmp 921zstd -f tmp -D tmpDict 922zstd -d tmp.zst -D tmpDict -fo result 923$DIFF "$TESTFILE" result 924zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" 925println "- Create second (different) dictionary" 926zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 927zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 928println "- Create dictionary with short dictID" 929zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 930cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 931println "- Create dictionary with size limit" 932zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 933println "- Test -o before --train-legacy" 934rm -f tmpDict dictionary 935zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 936test -f tmpDict 937zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 938test -f dictionary 939rm tmp* dictionary 940 941 942println "\n===> integrity tests " 943 944println "test one file (tmp1.zst) " 945datagen > tmp1 946zstd tmp1 947zstd -t tmp1.zst 948zstd --test tmp1.zst 949println "test multiple files (*.zst) " 950zstd -t ./*.zst 951println "test bad files (*) " 952zstd -t ./* && die "bad files not detected !" 953zstd -t tmp1 && die "bad file not detected !" 954cp tmp1 tmp2.zst 955zstd -t tmp2.zst && die "bad file not detected !" 956datagen -g0 > tmp3 957zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad 958println "test --rm and --test combined " 959zstd -t --rm tmp1.zst 960test -f tmp1.zst # check file is still present 961split -b16384 tmp1.zst tmpSplit. 962zstd -t tmpSplit.* && die "bad file not detected !" 963datagen | zstd -c | zstd -t 964 965 966println "\n===> golden files tests " 967 968zstd -t -r "$TESTDIR/golden-decompression" 969zstd -c -r "$TESTDIR/golden-compression" | zstd -t 970zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t 971 972 973println "\n===> benchmark mode tests " 974 975println "bench one file" 976datagen > tmp1 977zstd -bi0 tmp1 978println "bench multiple levels" 979zstd -i0b0e3 tmp1 980println "bench negative level" 981zstd -bi0 --fast tmp1 982println "with recursive and quiet modes" 983zstd -rqi0b1e2 tmp1 984println "benchmark decompression only" 985zstd -f tmp1 986zstd -b -d -i0 tmp1.zst 987 988 989println "\n===> zstd compatibility tests " 990 991datagen > tmp 992rm -f tmp.zst 993zstd --format=zstd -f tmp 994test -f tmp.zst 995 996 997println "\n===> gzip compatibility tests " 998 999GZIPMODE=1 1000zstd --format=gzip -V || GZIPMODE=0 1001if [ $GZIPMODE -eq 1 ]; then 1002 println "gzip support detected" 1003 GZIPEXE=1 1004 gzip -V || GZIPEXE=0 1005 if [ $GZIPEXE -eq 1 ]; then 1006 datagen > tmp 1007 zstd --format=gzip -f tmp 1008 gzip -t -v tmp.gz 1009 gzip -f tmp 1010 zstd -d -f -v tmp.gz 1011 rm tmp* 1012 else 1013 println "gzip binary not detected" 1014 fi 1015else 1016 println "gzip mode not supported" 1017fi 1018 1019 1020println "\n===> gzip frame tests " 1021 1022if [ $GZIPMODE -eq 1 ]; then 1023 datagen > tmp 1024 zstd -f --format=gzip tmp 1025 zstd -f tmp 1026 cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp 1027 truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1028 rm tmp* 1029else 1030 println "gzip mode not supported" 1031fi 1032 1033if [ $GZIPMODE -eq 1 ]; then 1034 datagen > tmp 1035 rm -f tmp.zst 1036 zstd --format=gzip --format=zstd -f tmp 1037 test -f tmp.zst 1038fi 1039 1040println "\n===> xz compatibility tests " 1041 1042LZMAMODE=1 1043zstd --format=xz -V || LZMAMODE=0 1044if [ $LZMAMODE -eq 1 ]; then 1045 println "xz support detected" 1046 XZEXE=1 1047 xz -Q -V && lzma -Q -V || XZEXE=0 1048 if [ $XZEXE -eq 1 ]; then 1049 println "Testing zstd xz and lzma support" 1050 datagen > tmp 1051 zstd --format=lzma -f tmp 1052 zstd --format=xz -f tmp 1053 xz -Q -t -v tmp.xz 1054 xz -Q -t -v tmp.lzma 1055 xz -Q -f -k tmp 1056 lzma -Q -f -k --lzma1 tmp 1057 zstd -d -f -v tmp.xz 1058 zstd -d -f -v tmp.lzma 1059 rm tmp* 1060 println "Creating symlinks" 1061 ln -s "$ZSTD_BIN" ./xz 1062 ln -s "$ZSTD_BIN" ./unxz 1063 ln -s "$ZSTD_BIN" ./lzma 1064 ln -s "$ZSTD_BIN" ./unlzma 1065 println "Testing xz and lzma symlinks" 1066 datagen > tmp 1067 ./xz tmp 1068 xz -Q -d tmp.xz 1069 ./lzma tmp 1070 lzma -Q -d tmp.lzma 1071 println "Testing unxz and unlzma symlinks" 1072 xz -Q tmp 1073 ./xz -d tmp.xz 1074 lzma -Q tmp 1075 ./lzma -d tmp.lzma 1076 rm xz unxz lzma unlzma 1077 rm tmp* 1078 else 1079 println "xz binary not detected" 1080 fi 1081else 1082 println "xz mode not supported" 1083fi 1084 1085 1086println "\n===> xz frame tests " 1087 1088if [ $LZMAMODE -eq 1 ]; then 1089 datagen > tmp 1090 zstd -f --format=xz tmp 1091 zstd -f --format=lzma tmp 1092 zstd -f tmp 1093 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp 1094 truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1095 truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1096 rm tmp* 1097else 1098 println "xz mode not supported" 1099fi 1100 1101println "\n===> lz4 compatibility tests " 1102 1103LZ4MODE=1 1104zstd --format=lz4 -V || LZ4MODE=0 1105if [ $LZ4MODE -eq 1 ]; then 1106 println "lz4 support detected" 1107 LZ4EXE=1 1108 lz4 -V || LZ4EXE=0 1109 if [ $LZ4EXE -eq 1 ]; then 1110 datagen > tmp 1111 zstd --format=lz4 -f tmp 1112 lz4 -t -v tmp.lz4 1113 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout 1114 zstd -d -f -v tmp.lz4 1115 rm tmp* 1116 else 1117 println "lz4 binary not detected" 1118 fi 1119else 1120 println "lz4 mode not supported" 1121fi 1122 1123 1124if [ $LZ4MODE -eq 1 ]; then 1125 println "\n===> lz4 frame tests " 1126 datagen > tmp 1127 zstd -f --format=lz4 tmp 1128 zstd -f tmp 1129 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp 1130 truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1131 rm tmp* 1132else 1133 println "\nlz4 mode not supported" 1134fi 1135 1136 1137println "\n===> suffix list test" 1138 1139! zstd -d tmp.abc 2> tmplg 1140 1141if [ $GZIPMODE -ne 1 ]; then 1142 grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1143fi 1144 1145if [ $LZMAMODE -ne 1 ]; then 1146 grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" 1147 grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1148fi 1149 1150if [ $LZ4MODE -ne 1 ]; then 1151 grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" 1152fi 1153 1154touch tmp1 1155zstd tmp1 -o tmp1.zstd 1156zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix 1157 1158println "\n===> tar extension tests " 1159 1160rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd 1161 1162datagen > tmp 1163tar cf tmp.tar tmp 1164zstd tmp.tar -o tmp.tzst 1165rm tmp.tar 1166zstd -d tmp.tzst 1167[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" 1168rm -f tmp.tar tmp.tzst 1169 1170if [ $GZIPMODE -eq 1 ]; then 1171 tar czf tmp.tgz tmp 1172 zstd -d tmp.tgz 1173 [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" 1174 rm -f tmp.tar tmp.tgz 1175fi 1176 1177if [ $LZMAMODE -eq 1 ]; then 1178 tar c tmp | zstd --format=xz > tmp.txz 1179 zstd -d tmp.txz 1180 [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" 1181 rm -f tmp.tar tmp.txz 1182fi 1183 1184if [ $LZ4MODE -eq 1 ]; then 1185 tar c tmp | zstd --format=lz4 > tmp.tlz4 1186 zstd -d tmp.tlz4 1187 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" 1188 rm -f tmp.tar tmp.tlz4 1189fi 1190 1191touch tmp.t tmp.tz tmp.tzs 1192! zstd -d tmp.t 1193! zstd -d tmp.tz 1194! zstd -d tmp.tzs 1195 1196 1197println "\n===> zstd round-trip tests " 1198 1199roundTripTest 1200roundTripTest -g15K # TableID==3 1201roundTripTest -g127K # TableID==2 1202roundTripTest -g255K # TableID==1 1203roundTripTest -g522K # TableID==0 1204roundTripTest -g519K 6 # greedy, hash chain 1205roundTripTest -g517K 16 # btlazy2 1206roundTripTest -g516K 19 # btopt 1207 1208fileRoundTripTest -g500K 1209 1210println "\n===> zstd long distance matching round-trip tests " 1211roundTripTest -g0 "2 --single-thread --long" 1212roundTripTest -g1000K "1 --single-thread --long" 1213roundTripTest -g517K "6 --single-thread --long" 1214roundTripTest -g516K "16 --single-thread --long" 1215roundTripTest -g518K "19 --single-thread --long" 1216roundTripTest -g2M "22 --single-thread --ultra --long" 1217fileRoundTripTest -g5M "3 --single-thread --long" 1218 1219 1220roundTripTest -g96K "5 --single-thread" 1221if [ -n "$hasMT" ] 1222then 1223 println "\n===> zstdmt round-trip tests " 1224 roundTripTest -g4M "1 -T0" 1225 roundTripTest -g8M "3 -T2" 1226 roundTripTest -g8M "19 -T0 --long" 1227 roundTripTest -g8000K "2 --threads=2" 1228 fileRoundTripTest -g4M "19 -T2 -B1M" 1229 1230 println "\n===> zstdmt long distance matching round-trip tests " 1231 roundTripTest -g8M "3 --long=24 -T2" 1232 1233 println "\n===> zstdmt environment variable tests " 1234 echo "multifoo" >> mt_tmp 1235 ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting 1236 ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting 1237 ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting 1238 ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1239 ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1240 ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting 1241 ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= 1242 ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage 1243 ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread 1244 rm mt_tmp* 1245 1246 println "\n===> ovLog tests " 1247 datagen -g2MB > tmp 1248 refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) 1249 ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) 1250 ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) 1251 if [ "$refSize" -eq "$ov9Size" ]; then 1252 echo ov9Size should be different from refSize 1253 exit 1 1254 fi 1255 if [ "$refSize" -eq "$ov1Size" ]; then 1256 echo ov1Size should be different from refSize 1257 exit 1 1258 fi 1259 if [ "$ov9Size" -ge "$ov1Size" ]; then 1260 echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" 1261 exit 1 1262 fi 1263 1264else 1265 println "\n===> no multithreading, skipping zstdmt tests " 1266fi 1267 1268rm tmp* 1269 1270println "\n===> zstd --list/-l single frame tests " 1271datagen > tmp1 1272datagen > tmp2 1273datagen > tmp3 1274zstd tmp* 1275zstd -l ./*.zst 1276zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header 1277zstd --list ./*.zst 1278zstd --list -v ./*.zst 1279 1280println "\n===> zstd --list/-l multiple frame tests " 1281cat tmp1.zst tmp2.zst > tmp12.zst 1282cat tmp12.zst tmp3.zst > tmp123.zst 1283zstd -l ./*.zst 1284zstd -lv ./*.zst 1285 1286println "\n===> zstd --list/-l error detection tests " 1287zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" 1288zstd --list tmp* && die "-l must fail on non-zstd file" 1289zstd -lv tmp1* && die "-l must fail on non-zstd file" 1290zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" 1291 1292println "test : detect truncated compressed file " 1293TEST_DATA_FILE=truncatable-input.txt 1294FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst 1295TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst 1296datagen -g50000 > $TEST_DATA_FILE 1297zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE 1298dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE 1299zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" 1300 1301rm $TEST_DATA_FILE 1302rm $FULL_COMPRESSED_FILE 1303rm $TRUNCATED_COMPRESSED_FILE 1304 1305println "\n===> zstd --list/-l errors when presented with stdin / no files" 1306zstd -l && die "-l must fail on empty list of files" 1307zstd -l - && die "-l does not work on stdin" 1308zstd -l < tmp1.zst && die "-l does not work on stdin" 1309zstd -l - < tmp1.zst && die "-l does not work on stdin" 1310zstd -l - tmp1.zst && die "-l does not work on stdin" 1311zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" 1312zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty 1313 1314println "\n===> zstd --list/-l test with null files " 1315datagen -g0 > tmp5 1316zstd tmp5 1317zstd -l tmp5.zst 1318zstd -l tmp5* && die "-l must fail on non-zstd file" 1319zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header 1320zstd -lv tmp5* && die "-l must fail on non-zstd file" 1321 1322println "\n===> zstd --list/-l test with no content size field " 1323datagen -g513K | zstd > tmp6.zst 1324zstd -l tmp6.zst 1325zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" 1326 1327println "\n===> zstd --list/-l test with no checksum " 1328zstd -f --no-check tmp1 1329zstd -l tmp1.zst 1330zstd -lv tmp1.zst 1331 1332println "\n===> zstd trace tests " 1333zstd -f --trace tmp.trace tmp1 1334zstd -f --trace tmp.trace tmp1 tmp2 tmp3 1335zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null 1336zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread 1337zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null 1338zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread 1339zstd --trace tmp.trace -t tmp1.zst 1340zstd --trace tmp.trace -t tmp1.zst tmp2.zst 1341zstd -f --trace tmp.trace -d tmp1.zst 1342zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst 1343zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 1344zstd -b1e10i0 --trace tmp.trace tmp1 1345zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 1346 1347rm tmp* 1348 1349 1350println "\n===> zstd long distance matching tests " 1351roundTripTest -g0 " --single-thread --long" 1352roundTripTest -g9M "2 --single-thread --long" 1353# Test parameter parsing 1354roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" 1355roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" 1356roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" 1357roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" 1358 1359 1360println "\n===> zstd long distance matching with optimal parser compressed size tests " 1361optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) 1362longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) 1363optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) 1364longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) 1365optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) 1366longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) 1367if [ "$longCSize16" -gt "$optCSize16" ]; then 1368 echo using --long on compression level 16 should not cause compressed size regression 1369 exit 1 1370elif [ "$longCSize19" -gt "$optCSize19" ]; then 1371 echo using --long on compression level 19 should not cause compressed size regression 1372 exit 1 1373elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then 1374 echo using --long on compression level 19 with wLog=23 should not cause compressed size regression 1375 exit 1 1376fi 1377 1378 1379if [ "$1" != "--test-large-data" ]; then 1380 println "Skipping large data tests" 1381 exit 0 1382fi 1383 1384 1385############################################################################# 1386 1387 1388if [ -n "$hasMT" ] 1389then 1390 println "\n===> adaptive mode " 1391 roundTripTest -g270000000 " --adapt" 1392 roundTripTest -g27000000 " --adapt=min=1,max=4" 1393 println "===> test: --adapt must fail on incoherent bounds " 1394 datagen > tmp 1395 zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" 1396 1397 println "\n===> rsyncable mode " 1398 roundTripTest -g10M " --rsyncable" 1399 roundTripTest -g10M " --rsyncable -B100K" 1400 println "===> test: --rsyncable must fail with --single-thread" 1401 zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" 1402fi 1403 1404println "\n===> patch-from=origin tests" 1405datagen -g1000 -P50 > tmp_dict 1406datagen -g1000 -P10 > tmp_patch 1407zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff 1408zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon 1409$DIFF -s tmp_patch_recon tmp_patch 1410 1411println "\n===> alternate syntax: patch-from origin" 1412zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff 1413zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon 1414$DIFF -s tmp_patch_recon tmp_patch 1415rm -rf tmp_* 1416 1417println "\n===> patch-from recursive tests" 1418mkdir tmp_dir 1419datagen > tmp_dir/tmp1 1420datagen > tmp_dir/tmp2 1421datagen > tmp_dict 1422zstd --patch-from=tmp_dict -r tmp_dir && die 1423rm -rf tmp* 1424 1425println "\n===> patch-from long mode trigger larger file test" 1426datagen -g5000000 > tmp_dict 1427datagen -g5000000 > tmp_patch 1428zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" 1429rm -rf tmp* 1430 1431println "\n===> patch-from --stream-size test" 1432datagen -g1000 -P50 > tmp_dict 1433datagen -g1000 -P10 > tmp_patch 1434cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die 1435cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff 1436rm -rf tmp* 1437 1438println "\n===> large files tests " 1439 1440roundTripTest -g270000000 1 1441roundTripTest -g250000000 2 1442roundTripTest -g230000000 3 1443 1444roundTripTest -g140000000 -P60 4 1445roundTripTest -g130000000 -P62 5 1446roundTripTest -g120000000 -P65 6 1447 1448roundTripTest -g70000000 -P70 7 1449roundTripTest -g60000000 -P71 8 1450roundTripTest -g50000000 -P73 9 1451 1452roundTripTest -g35000000 -P75 10 1453roundTripTest -g30000000 -P76 11 1454roundTripTest -g25000000 -P78 12 1455 1456roundTripTest -g18000013 -P80 13 1457roundTripTest -g18000014 -P80 14 1458roundTripTest -g18000015 -P81 15 1459roundTripTest -g18000016 -P84 16 1460roundTripTest -g18000017 -P88 17 1461roundTripTest -g18000018 -P94 18 1462roundTripTest -g18000019 -P96 19 1463 1464roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" 1465roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale 1466 1467fileRoundTripTest -g4193M -P99 1 1468 1469 1470println "\n===> zstd long, long distance matching round-trip tests " 1471roundTripTest -g270000000 "1 --single-thread --long" 1472roundTripTest -g130000000 -P60 "5 --single-thread --long" 1473roundTripTest -g35000000 -P70 "8 --single-thread --long" 1474roundTripTest -g18000001 -P80 "18 --single-thread --long" 1475# Test large window logs 1476roundTripTest -g700M -P50 "1 --single-thread --long=29" 1477roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" 1478 1479 1480if [ -n "$hasMT" ] 1481then 1482 println "\n===> zstdmt long round-trip tests " 1483 roundTripTest -g80000000 -P99 "19 -T2" " " 1484 roundTripTest -g5000000000 -P99 "1 -T2" " " 1485 roundTripTest -g500000000 -P97 "1 -T999" " " 1486 fileRoundTripTest -g4103M -P98 " -T0" " " 1487 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " 1488 # Exposes the bug in https://github.com/facebook/zstd/pull/1678 1489 # This test fails on 4 different travis builds at the time of writing 1490 # because it needs to allocate 8 GB of memory. 1491 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" 1492else 1493 println "\n**** no multithreading, skipping zstdmt tests **** " 1494fi 1495 1496 1497println "\n===> cover dictionary builder : advanced options " 1498 1499TESTFILE="$PRGDIR"/zstdcli.c 1500datagen > tmpDict 1501println "- Create first dictionary" 1502zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1503cp "$TESTFILE" tmp 1504zstd -f tmp -D tmpDict 1505zstd -d tmp.zst -D tmpDict -fo result 1506$DIFF "$TESTFILE" result 1507zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" 1508println "- Create second (different) dictionary" 1509zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1510zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1511println "- Create dictionary using shrink-dict flag" 1512zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict 1513zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 1514zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 1515println "- Create dictionary with short dictID" 1516zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1517cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1518println "- Create dictionary with size limit" 1519zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1520println "- Compare size of dictionary from 90% training samples with 80% training samples" 1521zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1522zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1523println "- Create dictionary using all samples for both training and testing" 1524zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1525println "- Test -o before --train-cover" 1526rm -f tmpDict dictionary 1527zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1528test -f tmpDict 1529zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1530test -f dictionary 1531rm -f tmp* dictionary 1532 1533rm -f tmp* 1534