1#!/bin/sh 2 3set -e 4 5die() { 6 println "$@" 1>&2 7 exit 1 8} 9 10datagen() { 11 "$DATAGEN_BIN" "$@" 12} 13 14zstd() { 15 if [ -z "$EXEC_PREFIX" ]; then 16 "$ZSTD_BIN" "$@" 17 else 18 "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 19 fi 20} 21 22sudoZstd() { 23 if [ -z "$EXEC_PREFIX" ]; then 24 sudo "$ZSTD_BIN" "$@" 25 else 26 sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 27 fi 28} 29 30roundTripTest() { 31 if [ -n "$3" ]; then 32 cLevel="$3" 33 proba="$2" 34 else 35 cLevel="$2" 36 proba="" 37 fi 38 if [ -n "$4" ]; then 39 dLevel="$4" 40 else 41 dLevel="$cLevel" 42 fi 43 44 rm -f tmp1 tmp2 45 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" 46 datagen $1 $proba | $MD5SUM > tmp1 47 datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 48 $DIFF -q tmp1 tmp2 49} 50 51fileRoundTripTest() { 52 if [ -n "$3" ]; then 53 local_c="$3" 54 local_p="$2" 55 else 56 local_c="$2" 57 local_p="" 58 fi 59 if [ -n "$4" ]; then 60 local_d="$4" 61 else 62 local_d="$local_c" 63 fi 64 65 rm -f tmp.zst tmp.md5.1 tmp.md5.2 66 println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" 67 datagen $1 $local_p > tmp 68 < tmp $MD5SUM > tmp.md5.1 69 zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 70 $DIFF -q tmp.md5.1 tmp.md5.2 71} 72 73truncateLastByte() { 74 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" 75} 76 77println() { 78 printf '%b\n' "${*}" 79} 80 81if [ -z "${size}" ]; then 82 size= 83else 84 size=${size} 85fi 86 87SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 88PRGDIR="$SCRIPT_DIR/../programs" 89TESTDIR="$SCRIPT_DIR/../tests" 90UNAME=$(uname) 91ZSTDGREP="$PRGDIR/zstdgrep" 92 93detectedTerminal=false 94if [ -t 0 ] && [ -t 1 ] 95then 96 detectedTerminal=true 97fi 98isTerminal=${isTerminal:-$detectedTerminal} 99 100isWindows=false 101INTOVOID="/dev/null" 102case "$UNAME" in 103 GNU) DEVDEVICE="/dev/random" ;; 104 *) DEVDEVICE="/dev/zero" ;; 105esac 106case "$OS" in 107 Windows*) 108 isWindows=true 109 INTOVOID="NUL" 110 DEVDEVICE="NUL" 111 ;; 112esac 113 114case "$UNAME" in 115 Darwin) MD5SUM="md5 -r" ;; 116 FreeBSD) MD5SUM="gmd5sum" ;; 117 OpenBSD) MD5SUM="md5" ;; 118 *) MD5SUM="md5sum" ;; 119esac 120 121MTIME="stat -c %Y" 122case "$UNAME" in 123 Darwin | FreeBSD | OpenBSD) MTIME="stat -f %m" ;; 124esac 125 126DIFF="diff" 127case "$UNAME" in 128 SunOS) DIFF="gdiff" ;; 129esac 130 131 132# check if ZSTD_BIN is defined. if not, use the default value 133if [ -z "${ZSTD_BIN}" ]; then 134 println "\nZSTD_BIN is not set. Using the default value..." 135 ZSTD_BIN="$PRGDIR/zstd" 136fi 137 138# check if DATAGEN_BIN is defined. if not, use the default value 139if [ -z "${DATAGEN_BIN}" ]; then 140 println "\nDATAGEN_BIN is not set. Using the default value..." 141 DATAGEN_BIN="$TESTDIR/datagen" 142fi 143 144ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" 145 146# assertions 147[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md" 148[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " 149println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" 150 151if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' 152then 153 hasMT="" 154else 155 hasMT="true" 156fi 157 158 159 160println "\n===> simple tests " 161 162datagen > tmp 163println "test : basic compression " 164zstd -f tmp # trivial compression case, creates tmp.zst 165println "test : basic decompression" 166zstd -df tmp.zst # trivial decompression case (overwrites tmp) 167println "test : too large compression level => auto-fix" 168zstd -99 -f tmp # too large compression level, automatic sized down 169zstd -5000000000 -f tmp && die "too large numeric value : must fail" 170println "test : --fast aka negative compression levels" 171zstd --fast -f tmp # == -1 172zstd --fast=3 -f tmp # == -3 173zstd --fast=200000 -f tmp # too low compression level, automatic fixed 174zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" 175zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" 176println "test : too large numeric argument" 177zstd --fast=9999999999 -f tmp && die "should have refused numeric value" 178println "test : set compression level with environment variable ZSTD_CLEVEL" 179ZSTD_CLEVEL=12 zstd -f tmp # positive compression level 180ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level 181ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign 182ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting 183ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting 184ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting 185ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting 186ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting 187ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting 188println "test : override ZSTD_CLEVEL with command line option" 189ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option 190println "test : compress to stdout" 191zstd tmp -c > tmpCompressed 192zstd tmp --stdout > tmpCompressed # long command format 193println "test : compress to named file" 194rm tmpCompressed 195zstd tmp -o tmpCompressed 196test -f tmpCompressed # file must be created 197println "test : force write, correct order" 198zstd tmp -fo tmpCompressed 199println "test : forgotten argument" 200cp tmp tmp2 201zstd tmp2 -fo && die "-o must be followed by filename " 202println "test : implied stdout when input is stdin" 203println bob | zstd | zstd -d 204if [ "$isTerminal" = true ]; then 205println "test : compressed data to terminal" 206println bob | zstd && die "should have refused : compressed data to terminal" 207println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" 208zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" 209fi 210println "test : null-length file roundtrip" 211println -n '' | zstd - --stdout | zstd -d --stdout 212println "test : ensure small file doesn't add 3-bytes null block" 213datagen -g1 > tmp1 214zstd tmp1 -c | wc -c | grep "14" 215zstd < tmp1 | wc -c | grep "14" 216println "test : decompress file with wrong suffix (must fail)" 217zstd -d tmpCompressed && die "wrong suffix error not detected!" 218zstd -df tmp && die "should have refused : wrong extension" 219println "test : decompress into stdout" 220zstd -d tmpCompressed -c > tmpResult # decompression using stdout 221zstd --decompress tmpCompressed -c > tmpResult 222zstd --decompress tmpCompressed --stdout > tmpResult 223println "test : decompress from stdin into stdout" 224zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout 225zstd -dc - < tmp.zst > $INTOVOID 226zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used 227zstd -d - < tmp.zst > $INTOVOID 228println "test : impose memory limitation (must fail)" 229zstd -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" 230zstd -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 231zstd -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 232zstd -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 233println "test : overwrite protection" 234zstd -q tmp && die "overwrite check failed!" 235println "test : force overwrite" 236zstd -q -f tmp 237zstd -q --force tmp 238println "test : overwrite readonly file" 239rm -f tmpro tmpro.zst 240println foo > tmpro.zst 241println foo > tmpro 242chmod 400 tmpro.zst 243zstd -q tmpro && die "should have refused to overwrite read-only file" 244zstd -q -f tmpro 245println "test: --no-progress flag" 246zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress 247zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress 248rm -f tmpro tmpro.zst 249println "test: overwrite input file (must fail)" 250zstd tmp -fo tmp && die "zstd compression overwrote the input file" 251zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" 252println "test: detect that input file does not exist" 253zstd nothere && die "zstd hasn't detected that input file does not exist" 254println "test: --[no-]compress-literals" 255zstd tmp -c --no-compress-literals -1 | zstd -t 256zstd tmp -c --no-compress-literals --fast=1 | zstd -t 257zstd tmp -c --no-compress-literals -19 | zstd -t 258zstd tmp -c --compress-literals -1 | zstd -t 259zstd tmp -c --compress-literals --fast=1 | zstd -t 260zstd tmp -c --compress-literals -19 | zstd -t 261zstd -b --fast=1 -i0e1 tmp --compress-literals 262zstd -b --fast=1 -i0e1 tmp --no-compress-literals 263println "test: --no-check for decompression" 264zstd -f tmp -o tmp_corrupt.zst --check 265zstd -f tmp -o tmp.zst --no-check 266printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp 267zstd -d -f tmp_corrupt.zst --no-check 268zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides 269zstd -d -f tmp.zst --no-check 270 271println "\n===> zstdgrep tests" 272ln -sf "$ZSTD_BIN" zstdcat 273rm -f tmp_grep 274echo "1234" > tmp_grep 275zstd -f tmp_grep 276lines=$(ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) 277test 2 -eq $lines 278ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" 279ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true 280rm -f tmp_grep* 281 282println "\n===> --exclude-compressed flag" 283rm -rf precompressedFilterTestDir 284mkdir -p precompressedFilterTestDir 285datagen $size > precompressedFilterTestDir/input.5 286datagen $size > precompressedFilterTestDir/input.6 287zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 288datagen $size > precompressedFilterTestDir/input.7 289datagen $size > precompressedFilterTestDir/input.8 290zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 291test ! -f precompressedFilterTestDir/input.5.zst.zst 292test ! -f precompressedFilterTestDir/input.6.zst.zst 293file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` 294file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` 295if [ $file2timestamp -ge $file1timestamp ]; then 296 println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." 297else 298 println "Test is not successful" 299fi 300# File Extension check. 301datagen $size > precompressedFilterTestDir/input.zstbar 302zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 303# zstd should compress input.zstbar 304test -f precompressedFilterTestDir/input.zstbar.zst 305# Check without the --exclude-compressed flag 306zstd --long --rm -r precompressedFilterTestDir 307# Files should get compressed again without the --exclude-compressed flag. 308test -f precompressedFilterTestDir/input.5.zst.zst 309test -f precompressedFilterTestDir/input.6.zst.zst 310println "Test completed" 311 312 313 314println "\n===> warning prompts should not occur if stdin is an input" 315println "y" > tmpPrompt 316println "hello world" >> tmpPrompt 317zstd tmpPrompt -f 318zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 319zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f 320zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated 321$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed 322 323echo 'yes' | zstd tmpPrompt -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files 324echo 'yes' | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 325zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" 326 327println "Test completed" 328 329 330println "\n===> recursive mode test " 331# combination of -r with empty list of input file 332zstd -c -r < tmp > tmp.zst 333 334 335println "\n===> file removal" 336zstd -f --rm tmp 337test ! -f tmp # tmp should no longer be present 338zstd -f -d --rm tmp.zst 339test ! -f tmp.zst # tmp.zst should no longer be present 340println "test : should quietly not remove non-regular file" 341println hello > tmp 342zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" 343grep -v "Refusing to remove non-regular file" tmplog 344rm -f tmplog 345zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" 346println "test : --rm on stdin" 347println a | zstd --rm > $INTOVOID # --rm should remain silent 348rm tmp 349zstd -f tmp && die "tmp not present : should have failed" 350test ! -f tmp.zst # tmp.zst should not be created 351println "test : -d -f do not delete destination when source is not present" 352touch tmp # create destination file 353zstd -d -f tmp.zst && die "attempt to decompress a non existing file" 354test -f tmp # destination file should still be present 355println "test : -f do not delete destination when source is not present" 356rm tmp # erase source file 357touch tmp.zst # create destination file 358zstd -f tmp && die "attempt to compress a non existing file" 359test -f tmp.zst # destination file should still be present 360rm -rf tmp* # may also erase tmp* directory from previous failed run 361 362 363println "\n===> decompression only tests " 364# the following test verifies that the decoder is compatible with RLE as first block 365# older versions of zstd cli are not able to decode such corner case. 366# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. 367dd bs=1048576 count=1 if=/dev/zero of=tmp 368zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" 369$DIFF -s tmp1 tmp 370rm tmp* 371 372 373println "\n===> compress multiple files" 374println hello > tmp1 375println world > tmp2 376zstd tmp1 tmp2 -o "$INTOVOID" -f 377zstd tmp1 tmp2 -c | zstd -t 378zstd tmp1 tmp2 -o tmp.zst 379test ! -f tmp1.zst 380test ! -f tmp2.zst 381zstd tmp1 tmp2 382zstd -t tmp1.zst tmp2.zst 383zstd -dc tmp1.zst tmp2.zst 384zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f 385zstd -d tmp1.zst tmp2.zst -o tmp 386touch tmpexists 387zstd tmp1 tmp2 -f -o tmpexists 388zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" 389println gooder > tmp_rm1 390println boi > tmp_rm2 391println worldly > tmp_rm3 392echo 'y' | zstd tmp_rm1 tmp_rm2 -o tmp_rm3.zst --rm # tests the warning prompt for --rm with multiple inputs into once source 393test ! -f tmp_rm1 394test ! -f tmp_rm2 395cp tmp_rm3.zst tmp_rm4.zst 396echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -o tmp_rm_out --rm 397test ! -f tmp_rm3.zst 398test ! -f tmp_rm4.zst 399echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 400echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 401println gooder > tmpexists1 402zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID 403 404# Bug: PR #972 405if [ "$?" -eq 139 ]; then 406 die "should not have segfaulted" 407fi 408println "\n===> multiple files and shell completion " 409datagen -s1 > tmp1 2> $INTOVOID 410datagen -s2 -g100K > tmp2 2> $INTOVOID 411datagen -s3 -g1M > tmp3 2> $INTOVOID 412println "compress tmp* : " 413zstd -f tmp* 414test -f tmp1.zst 415test -f tmp2.zst 416test -f tmp3.zst 417rm tmp1 tmp2 tmp3 418println "decompress tmp* : " 419zstd -df ./*.zst 420test -f tmp1 421test -f tmp2 422test -f tmp3 423println "compress tmp* into stdout > tmpall : " 424zstd -c tmp1 tmp2 tmp3 > tmpall 425test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) 426println "decompress tmpall* into stdout > tmpdec : " 427cp tmpall tmpall2 428zstd -dc tmpall* > tmpdec 429test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) 430println "compress multiple files including a missing one (notHere) : " 431zstd -f tmp1 notHere tmp2 && die "missing file not detected!" 432rm tmp* 433 434 435if [ "$isWindows" = false ] ; then 436 println "\n===> zstd fifo named pipe test " 437 echo "Hello World!" > tmp_original 438 mkfifo tmp_named_pipe 439 # note : fifo test doesn't work in combination with `dd` or `cat` 440 echo "Hello World!" > tmp_named_pipe & 441 zstd tmp_named_pipe -o tmp_compressed 442 zstd -d -o tmp_decompressed tmp_compressed 443 $DIFF -s tmp_original tmp_decompressed 444 rm -rf tmp* 445fi 446 447 448if [ -n "$DEVNULLRIGHTS" ] ; then 449 # these tests requires sudo rights, which is uncommon. 450 # they are only triggered if DEVNULLRIGHTS macro is defined. 451 println "\n===> checking /dev/null permissions are unaltered " 452 datagen > tmp 453 sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions 454 sudoZstd tmp -c > $INTOVOID 455 zstd tmp -f -o tmp.zst 456 sudoZstd -d tmp.zst -c > $INTOVOID 457 sudoZstd -d tmp.zst -o $INTOVOID 458 ls -las $INTOVOID | grep "rw-rw-rw-" 459fi 460 461 462println "\n===> compress multiple files into an output directory, --output-dir-flat" 463println henlo > tmp1 464mkdir tmpInputTestDir 465mkdir tmpInputTestDir/we 466mkdir tmpInputTestDir/we/must 467mkdir tmpInputTestDir/we/must/go 468mkdir tmpInputTestDir/we/must/go/deeper 469println cool > tmpInputTestDir/we/must/go/deeper/tmp2 470mkdir tmpOutDir 471zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir 472test -f tmpOutDir/tmp1.zst 473test -f tmpOutDir/tmp2.zst 474println "test : decompress multiple files into an output directory, --output-dir-flat" 475mkdir tmpOutDirDecomp 476zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp 477test -f tmpOutDirDecomp/tmp2 478test -f tmpOutDirDecomp/tmp1 479rm -f tmpOutDirDecomp/* 480zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp 481test -f tmpOutDirDecomp/tmp2 482test -f tmpOutDirDecomp/tmp1 483rm -rf tmp* 484 485if [ "$isWindows" = false ] ; then 486 println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" 487 println "test --output-dir-mirror" > tmp1 488 mkdir -p tmpInputTestDir/we/must/go/deeper 489 println cool > tmpInputTestDir/we/must/go/deeper/tmp2 490 zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir 491 test -f tmpOutDir/tmp1.zst 492 test -f tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2.zst 493 494 println "test: compress input dir will be ignored if it has '..'" 495 zstd -r tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'" 496 test ! -d non-exist 497 498 println "test : decompress multiple files into an output directory, --output-dir-mirror" 499 zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp 500 test -f tmpOutDirDecomp/tmpOutDir/tmp1 501 test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2 502 503 println "test: decompress input dir will be ignored if it has '..'" 504 zstd -r tmpOutDir/tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'" 505 test ! -d non-exist 506 507 rm -rf tmp* 508fi 509 510 511println "test : compress multiple files reading them from a file, --filelist=FILE" 512println "Hello world!, file1" > tmp1 513println "Hello world!, file2" > tmp2 514println tmp1 > tmp_fileList 515println tmp2 >> tmp_fileList 516zstd -f --filelist=tmp_fileList 517test -f tmp2.zst 518test -f tmp1.zst 519 520println "test : alternate syntax: --filelist FILE" 521zstd -f --filelist tmp_fileList 522test -f tmp2.zst 523test -f tmp1.zst 524 525println "test : reading file list from a symlink, --filelist=FILE" 526rm -f *.zst 527ln -s tmp_fileList tmp_symLink 528zstd -f --filelist=tmp_symLink 529test -f tmp2.zst 530test -f tmp1.zst 531 532println "test : compress multiple files reading them from multiple files, --filelist=FILE" 533rm -f *.zst 534println "Hello world!, file3" > tmp3 535println "Hello world!, file4" > tmp4 536println tmp3 > tmp_fileList2 537println tmp4 >> tmp_fileList2 538zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 539test -f tmp1.zst 540test -f tmp2.zst 541test -f tmp3.zst 542test -f tmp4.zst 543 544println "test : decompress multiple files reading them from a file, --filelist=FILE" 545rm -f tmp1 tmp2 546println tmp1.zst > tmpZst 547println tmp2.zst >> tmpZst 548zstd -d -f --filelist=tmpZst 549test -f tmp1 550test -f tmp2 551 552println "test : decompress multiple files reading them from multiple files, --filelist=FILE" 553rm -f tmp1 tmp2 tmp3 tmp4 554println tmp3.zst > tmpZst2 555println tmp4.zst >> tmpZst2 556zstd -d -f --filelist=tmpZst --filelist=tmpZst2 557test -f tmp1 558test -f tmp2 559test -f tmp3 560test -f tmp4 561 562println "test : survive a list of files which is text garbage (--filelist=FILE)" 563datagen > tmp_badList 564zstd -f --filelist=tmp_badList && die "should have failed : list is text garbage" 565 566println "test : survive a list of files which is binary garbage (--filelist=FILE)" 567datagen -P0 -g1M > tmp_badList 568zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console 569 570println "test : try to overflow internal list of files (--filelist=FILE)" 571touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 572ls tmp* > tmpList 573zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list 574rm -rf tmp* 575 576println "\n===> --[no-]content-size tests" 577 578datagen > tmp_contentsize 579zstd -f tmp_contentsize 580zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 581zstd -f --no-content-size tmp_contentsize 582zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 583zstd -f --content-size tmp_contentsize 584zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 585zstd -f --content-size --no-content-size tmp_contentsize 586zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 587rm -rf tmp* 588 589println "test : show-default-cparams regular" 590datagen > tmp 591zstd --show-default-cparams -f tmp 592rm -rf tmp* 593 594println "test : show-default-cparams recursive" 595mkdir tmp_files 596datagen -g15000 > tmp_files/tmp1 597datagen -g129000 > tmp_files/tmp2 598datagen -g257000 > tmp_files/tmp3 599zstd --show-default-cparams -f -r tmp_files 600rm -rf tmp* 601 602println "\n===> Advanced compression parameters " 603println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" 604println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" 605println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" 606println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist 607test ! -f tmp.zst # tmp.zst should not be created 608roundTripTest -g512K 609roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" 610roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" 611roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" 612roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" 613roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" 614roundTripTest -g64K "19 --zstd=strat=9" # btultra2 615 616 617println "\n===> Pass-Through mode " 618println "Hello world 1!" | zstd -df 619println "Hello world 2!" | zstd -dcf 620println "Hello world 3!" > tmp1 621zstd -dcf tmp1 622 623 624println "\n===> frame concatenation " 625println "hello " > hello.tmp 626println "world!" > world.tmp 627cat hello.tmp world.tmp > helloworld.tmp 628zstd -c hello.tmp > hello.zst 629zstd -c world.tmp > world.zst 630cat hello.zst world.zst > helloworld.zst 631zstd -dc helloworld.zst > result.tmp 632cat result.tmp 633$DIFF helloworld.tmp result.tmp 634println "frame concatenation without checksum" 635zstd -c hello.tmp > hello.zst --no-check 636zstd -c world.tmp > world.zst --no-check 637cat hello.zst world.zst > helloworld.zstd 638zstd -dc helloworld.zst > result.tmp 639$DIFF helloworld.tmp result.tmp 640println "testing zstdcat symlink" 641ln -sf "$ZSTD_BIN" zstdcat 642$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp 643$DIFF helloworld.tmp result.tmp 644ln -s helloworld.zst helloworld.link.zst 645$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp 646$DIFF helloworld.tmp result.tmp 647rm zstdcat 648rm result.tmp 649println "testing zcat symlink" 650ln -sf "$ZSTD_BIN" zcat 651$EXE_PREFIX ./zcat helloworld.zst > result.tmp 652$DIFF helloworld.tmp result.tmp 653$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp 654$DIFF helloworld.tmp result.tmp 655rm zcat 656rm ./*.tmp ./*.zstd 657println "frame concatenation tests completed" 658 659 660if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then 661println "\n**** flush write error test **** " 662 663println "println foo | zstd > /dev/full" 664println foo | zstd > /dev/full && die "write error not detected!" 665println "println foo | zstd | zstd -d > /dev/full" 666println foo | zstd | zstd -d > /dev/full && die "write error not detected!" 667 668fi 669 670 671if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then 672 673println "\n===> symbolic link test " 674 675rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 676println "hello world" > hello.tmp 677ln -s hello.tmp world.tmp 678ln -s hello.tmp world2.tmp 679zstd world.tmp hello.tmp || true 680test -f hello.tmp.zst # regular file should have been compressed! 681test ! -f world.tmp.zst # symbolic link should not have been compressed! 682zstd world.tmp || true 683test ! -f world.tmp.zst # symbolic link should not have been compressed! 684zstd world.tmp world2.tmp || true 685test ! -f world.tmp.zst # symbolic link should not have been compressed! 686test ! -f world2.tmp.zst # symbolic link should not have been compressed! 687zstd world.tmp hello.tmp -f 688test -f world.tmp.zst # symbolic link should have been compressed with --force 689rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 690 691fi 692 693 694println "\n===> test sparse file support " 695 696datagen -g5M -P100 > tmpSparse 697zstd tmpSparse -c | zstd -dv -o tmpSparseRegen 698$DIFF -s tmpSparse tmpSparseRegen 699zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse 700$DIFF -s tmpSparse tmpOutSparse 701zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse 702$DIFF -s tmpSparse tmpOutNoSparse 703ls -ls tmpSparse* # look at file size and block size on disk 704datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) 705datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd 706ls -ls tmpSparseOdd # look at file size and block size on disk 707println "\n Sparse Compatibility with Console :" 708println "Hello World 1 !" | zstd | zstd -d -c 709println "Hello World 2 !" | zstd | zstd -d | cat 710println "\n Sparse Compatibility with Append :" 711datagen -P100 -g1M > tmpSparse1M 712cat tmpSparse1M tmpSparse1M > tmpSparse2M 713zstd -v -f tmpSparse1M -o tmpSparseCompressed 714zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated 715zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated 716ls -ls tmpSparse* # look at file size and block size on disk 717$DIFF tmpSparse2M tmpSparseRegenerated 718rm tmpSparse* 719 720 721println "\n===> stream-size mode" 722 723datagen -g11000 > tmp 724println "test : basic file compression vs sized streaming compression" 725file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 726stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) 727if [ "$stream_size" -gt "$file_size" ]; then 728 die "hinted compression larger than expected" 729fi 730println "test : sized streaming compression and decompression" 731cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 732zstd -df tmp.zst -o tmp_decompress 733cmp tmp tmp_decompress || die "difference between original and decompressed file" 734println "test : incorrect stream size" 735cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" 736 737println "\n===> zstd zero weight dict test " 738rm -f tmp* 739cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input 740zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 741zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 742$DIFF tmp_decomp tmp_input 743rm -rf tmp* 744 745println "\n===> zstd (valid) zero weight dict test " 746rm -f tmp* 747# 0 has a non-zero weight in the dictionary 748echo "0000000000000000000000000" > tmp_input 749zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 750zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 751$DIFF tmp_decomp tmp_input 752rm -rf tmp* 753 754println "\n===> size-hint mode" 755 756datagen -g11000 > tmp 757datagen -g11000 > tmp2 758datagen > tmpDict 759println "test : basic file compression vs hinted streaming compression" 760file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 761stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) 762if [ "$stream_size" -ge "$file_size" ]; then 763 die "hinted compression larger than expected" 764fi 765println "test : hinted streaming compression and decompression" 766cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 767zstd -df tmp.zst -o tmp_decompress 768cmp tmp tmp_decompress || die "difference between original and decompressed file" 769println "test : hinted streaming compression with dictionary" 770cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict 771println "test : multiple file compression with hints and dictionary" 772zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 773zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp 774zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 775cmp tmp.zst tmp1_.zst || die "first file's output differs" 776cmp tmp2.zst tmp2_.zst || die "second file's output differs" 777println "test : incorrect hinted stream sizes" 778cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high 779cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low 780cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high 781cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low 782 783 784println "\n===> dictionary tests " 785 786println "- test with raw dict (content only) " 787datagen > tmpDict 788datagen -g1M | $MD5SUM > tmp1 789datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 790$DIFF -q tmp1 tmp2 791println "- Create first dictionary " 792TESTFILE="$PRGDIR"/zstdcli.c 793zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 794cp "$TESTFILE" tmp 795println "- Test dictionary compression with tmpDict as an input file and dictionary" 796zstd -f tmpDict -D tmpDict && die "compression error not detected!" 797println "- Dictionary compression roundtrip" 798zstd -f tmp -D tmpDict 799zstd -d tmp.zst -D tmpDict -fo result 800$DIFF "$TESTFILE" result 801println "- Dictionary compression with btlazy2 strategy" 802zstd -f tmp -D tmpDict --zstd=strategy=6 803zstd -d tmp.zst -D tmpDict -fo result 804$DIFF "$TESTFILE" result 805if [ -n "$hasMT" ] 806then 807 println "- Test dictionary compression with multithreading " 808 datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 809fi 810println "- Create second (different) dictionary " 811zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 812zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 813println "- Create dictionary with short dictID" 814zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 815cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 816println "- Create dictionary with wrong dictID parameter order (must fail)" 817zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " 818println "- Create dictionary with size limit" 819zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v 820println "- Create dictionary with small size limit" 821zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v 822println "- Create dictionary with wrong parameter order (must fail)" 823zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " 824println "- Compress without dictID" 825zstd -f tmp -D tmpDict1 --no-dictID 826zstd -d tmp.zst -D tmpDict -fo result 827$DIFF "$TESTFILE" result 828println "- Compress multiple files with dictionary" 829rm -rf dirTestDict 830mkdir dirTestDict 831cp "$TESTDIR"/*.c dirTestDict 832cp "$PRGDIR"/*.c dirTestDict 833cp "$PRGDIR"/*.h dirTestDict 834$MD5SUM dirTestDict/* > tmph1 835zstd -f --rm dirTestDict/* -D tmpDictC 836zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default 837case "$UNAME" in 838 Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 839 *) $MD5SUM -c tmph1 ;; 840esac 841rm -rf dirTestDict 842println "- dictionary builder on bogus input" 843println "Hello World" > tmp 844zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" 845datagen -P0 -g10M > tmp 846zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" 847println "- Test -o before --train" 848rm -f tmpDict dictionary 849zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c 850test -f tmpDict 851zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c 852test -f dictionary 853println "- Test dictionary training fails" 854echo "000000000000000000000000000000000" > tmpz 855zstd --train tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 856if [ -n "$hasMT" ] 857then 858 zstd --train -T0 tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 859 println "- Create dictionary with multithreading enabled" 860 zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 861fi 862rm tmp* dictionary 863 864 865println "\n===> fastCover dictionary builder : advanced options " 866TESTFILE="$PRGDIR"/zstdcli.c 867datagen > tmpDict 868println "- Create first dictionary" 869zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 870cp "$TESTFILE" tmp 871zstd -f tmp -D tmpDict 872zstd -d tmp.zst -D tmpDict -fo result 873$DIFF "$TESTFILE" result 874println "- Create second (different) dictionary" 875zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 876zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 877zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" 878println "- Create dictionary with short dictID" 879zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 880cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 881println "- Create dictionaries with shrink-dict flag enabled" 882zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict 883zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 884zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 885println "- Create dictionary with size limit" 886zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 887println "- Create dictionary using all samples for both training and testing" 888zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 889println "- Create dictionary using f=16" 890zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 891zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" 892println "- Create dictionary using accel=2" 893zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 894println "- Create dictionary using accel=10" 895zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 896println "- Create dictionary with multithreading" 897zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 898println "- Test -o before --train-fastcover" 899rm -f tmpDict dictionary 900zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 901test -f tmpDict 902zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 903test -f dictionary 904rm tmp* dictionary 905 906 907println "\n===> legacy dictionary builder " 908 909TESTFILE="$PRGDIR"/zstdcli.c 910datagen > tmpDict 911println "- Create first dictionary" 912zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 913cp "$TESTFILE" tmp 914zstd -f tmp -D tmpDict 915zstd -d tmp.zst -D tmpDict -fo result 916$DIFF "$TESTFILE" result 917zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" 918println "- Create second (different) dictionary" 919zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 920zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 921println "- Create dictionary with short dictID" 922zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 923cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 924println "- Create dictionary with size limit" 925zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 926println "- Test -o before --train-legacy" 927rm -f tmpDict dictionary 928zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 929test -f tmpDict 930zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 931test -f dictionary 932rm tmp* dictionary 933 934 935println "\n===> integrity tests " 936 937println "test one file (tmp1.zst) " 938datagen > tmp1 939zstd tmp1 940zstd -t tmp1.zst 941zstd --test tmp1.zst 942println "test multiple files (*.zst) " 943zstd -t ./*.zst 944println "test bad files (*) " 945zstd -t ./* && die "bad files not detected !" 946zstd -t tmp1 && die "bad file not detected !" 947cp tmp1 tmp2.zst 948zstd -t tmp2.zst && die "bad file not detected !" 949datagen -g0 > tmp3 950zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad 951println "test --rm and --test combined " 952zstd -t --rm tmp1.zst 953test -f tmp1.zst # check file is still present 954split -b16384 tmp1.zst tmpSplit. 955zstd -t tmpSplit.* && die "bad file not detected !" 956datagen | zstd -c | zstd -t 957 958 959println "\n===> golden files tests " 960 961zstd -t -r "$TESTDIR/golden-decompression" 962zstd -c -r "$TESTDIR/golden-compression" | zstd -t 963zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t 964 965 966println "\n===> benchmark mode tests " 967 968println "bench one file" 969datagen > tmp1 970zstd -bi0 tmp1 971println "bench multiple levels" 972zstd -i0b0e3 tmp1 973println "bench negative level" 974zstd -bi0 --fast tmp1 975println "with recursive and quiet modes" 976zstd -rqi0b1e2 tmp1 977println "benchmark decompression only" 978zstd -f tmp1 979zstd -b -d -i0 tmp1.zst 980 981 982println "\n===> zstd compatibility tests " 983 984datagen > tmp 985rm -f tmp.zst 986zstd --format=zstd -f tmp 987test -f tmp.zst 988 989 990println "\n===> gzip compatibility tests " 991 992GZIPMODE=1 993zstd --format=gzip -V || GZIPMODE=0 994if [ $GZIPMODE -eq 1 ]; then 995 println "gzip support detected" 996 GZIPEXE=1 997 gzip -V || GZIPEXE=0 998 if [ $GZIPEXE -eq 1 ]; then 999 datagen > tmp 1000 zstd --format=gzip -f tmp 1001 gzip -t -v tmp.gz 1002 gzip -f tmp 1003 zstd -d -f -v tmp.gz 1004 rm tmp* 1005 else 1006 println "gzip binary not detected" 1007 fi 1008else 1009 println "gzip mode not supported" 1010fi 1011 1012 1013println "\n===> gzip frame tests " 1014 1015if [ $GZIPMODE -eq 1 ]; then 1016 datagen > tmp 1017 zstd -f --format=gzip tmp 1018 zstd -f tmp 1019 cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp 1020 truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1021 rm tmp* 1022else 1023 println "gzip mode not supported" 1024fi 1025 1026if [ $GZIPMODE -eq 1 ]; then 1027 datagen > tmp 1028 rm -f tmp.zst 1029 zstd --format=gzip --format=zstd -f tmp 1030 test -f tmp.zst 1031fi 1032 1033println "\n===> xz compatibility tests " 1034 1035LZMAMODE=1 1036zstd --format=xz -V || LZMAMODE=0 1037if [ $LZMAMODE -eq 1 ]; then 1038 println "xz support detected" 1039 XZEXE=1 1040 xz -Q -V && lzma -Q -V || XZEXE=0 1041 if [ $XZEXE -eq 1 ]; then 1042 println "Testing zstd xz and lzma support" 1043 datagen > tmp 1044 zstd --format=lzma -f tmp 1045 zstd --format=xz -f tmp 1046 xz -Q -t -v tmp.xz 1047 xz -Q -t -v tmp.lzma 1048 xz -Q -f -k tmp 1049 lzma -Q -f -k --lzma1 tmp 1050 zstd -d -f -v tmp.xz 1051 zstd -d -f -v tmp.lzma 1052 rm tmp* 1053 println "Creating symlinks" 1054 ln -s "$ZSTD_BIN" ./xz 1055 ln -s "$ZSTD_BIN" ./unxz 1056 ln -s "$ZSTD_BIN" ./lzma 1057 ln -s "$ZSTD_BIN" ./unlzma 1058 println "Testing xz and lzma symlinks" 1059 datagen > tmp 1060 ./xz tmp 1061 xz -Q -d tmp.xz 1062 ./lzma tmp 1063 lzma -Q -d tmp.lzma 1064 println "Testing unxz and unlzma symlinks" 1065 xz -Q tmp 1066 ./xz -d tmp.xz 1067 lzma -Q tmp 1068 ./lzma -d tmp.lzma 1069 rm xz unxz lzma unlzma 1070 rm tmp* 1071 else 1072 println "xz binary not detected" 1073 fi 1074else 1075 println "xz mode not supported" 1076fi 1077 1078 1079println "\n===> xz frame tests " 1080 1081if [ $LZMAMODE -eq 1 ]; then 1082 datagen > tmp 1083 zstd -f --format=xz tmp 1084 zstd -f --format=lzma tmp 1085 zstd -f tmp 1086 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp 1087 truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1088 truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1089 rm tmp* 1090else 1091 println "xz mode not supported" 1092fi 1093 1094println "\n===> lz4 compatibility tests " 1095 1096LZ4MODE=1 1097zstd --format=lz4 -V || LZ4MODE=0 1098if [ $LZ4MODE -eq 1 ]; then 1099 println "lz4 support detected" 1100 LZ4EXE=1 1101 lz4 -V || LZ4EXE=0 1102 if [ $LZ4EXE -eq 1 ]; then 1103 datagen > tmp 1104 zstd --format=lz4 -f tmp 1105 lz4 -t -v tmp.lz4 1106 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout 1107 zstd -d -f -v tmp.lz4 1108 rm tmp* 1109 else 1110 println "lz4 binary not detected" 1111 fi 1112else 1113 println "lz4 mode not supported" 1114fi 1115 1116 1117if [ $LZ4MODE -eq 1 ]; then 1118 println "\n===> lz4 frame tests " 1119 datagen > tmp 1120 zstd -f --format=lz4 tmp 1121 zstd -f tmp 1122 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp 1123 truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1124 rm tmp* 1125else 1126 println "\nlz4 mode not supported" 1127fi 1128 1129 1130println "\n===> suffix list test" 1131 1132! zstd -d tmp.abc 2> tmplg 1133 1134if [ $GZIPMODE -ne 1 ]; then 1135 grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1136fi 1137 1138if [ $LZMAMODE -ne 1 ]; then 1139 grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" 1140 grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1141fi 1142 1143if [ $LZ4MODE -ne 1 ]; then 1144 grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" 1145fi 1146 1147touch tmp1 1148zstd tmp1 -o tmp1.zstd 1149zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix 1150 1151println "\n===> tar extension tests " 1152 1153rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd 1154 1155datagen > tmp 1156tar cf tmp.tar tmp 1157zstd tmp.tar -o tmp.tzst 1158rm tmp.tar 1159zstd -d tmp.tzst 1160[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" 1161rm -f tmp.tar tmp.tzst 1162 1163if [ $GZIPMODE -eq 1 ]; then 1164 tar czf tmp.tgz tmp 1165 zstd -d tmp.tgz 1166 [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" 1167 rm -f tmp.tar tmp.tgz 1168fi 1169 1170if [ $LZMAMODE -eq 1 ]; then 1171 tar c tmp | zstd --format=xz > tmp.txz 1172 zstd -d tmp.txz 1173 [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" 1174 rm -f tmp.tar tmp.txz 1175fi 1176 1177if [ $LZ4MODE -eq 1 ]; then 1178 tar c tmp | zstd --format=lz4 > tmp.tlz4 1179 zstd -d tmp.tlz4 1180 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" 1181 rm -f tmp.tar tmp.tlz4 1182fi 1183 1184touch tmp.t tmp.tz tmp.tzs 1185! zstd -d tmp.t 1186! zstd -d tmp.tz 1187! zstd -d tmp.tzs 1188 1189 1190println "\n===> zstd round-trip tests " 1191 1192roundTripTest 1193roundTripTest -g15K # TableID==3 1194roundTripTest -g127K # TableID==2 1195roundTripTest -g255K # TableID==1 1196roundTripTest -g522K # TableID==0 1197roundTripTest -g519K 6 # greedy, hash chain 1198roundTripTest -g517K 16 # btlazy2 1199roundTripTest -g516K 19 # btopt 1200 1201fileRoundTripTest -g500K 1202 1203println "\n===> zstd long distance matching round-trip tests " 1204roundTripTest -g0 "2 --single-thread --long" 1205roundTripTest -g1000K "1 --single-thread --long" 1206roundTripTest -g517K "6 --single-thread --long" 1207roundTripTest -g516K "16 --single-thread --long" 1208roundTripTest -g518K "19 --single-thread --long" 1209roundTripTest -g2M "22 --single-thread --ultra --long" 1210fileRoundTripTest -g5M "3 --single-thread --long" 1211 1212 1213roundTripTest -g96K "5 --single-thread" 1214if [ -n "$hasMT" ] 1215then 1216 println "\n===> zstdmt round-trip tests " 1217 roundTripTest -g4M "1 -T0" 1218 roundTripTest -g8M "3 -T2" 1219 roundTripTest -g8M "19 -T0 --long" 1220 roundTripTest -g8000K "2 --threads=2" 1221 fileRoundTripTest -g4M "19 -T2 -B1M" 1222 1223 println "\n===> zstdmt long distance matching round-trip tests " 1224 roundTripTest -g8M "3 --long=24 -T2" 1225 1226 println "\n===> zstdmt environment variable tests " 1227 echo "multifoo" >> mt_tmp 1228 ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting 1229 ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting 1230 ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting 1231 ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1232 ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1233 ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting 1234 ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= 1235 ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage 1236 ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread 1237 rm mt_tmp* 1238 1239 println "\n===> ovLog tests " 1240 datagen -g2MB > tmp 1241 refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) 1242 ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) 1243 ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) 1244 if [ "$refSize" -eq "$ov9Size" ]; then 1245 echo ov9Size should be different from refSize 1246 exit 1 1247 fi 1248 if [ "$refSize" -eq "$ov1Size" ]; then 1249 echo ov1Size should be different from refSize 1250 exit 1 1251 fi 1252 if [ "$ov9Size" -ge "$ov1Size" ]; then 1253 echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" 1254 exit 1 1255 fi 1256 1257else 1258 println "\n===> no multithreading, skipping zstdmt tests " 1259fi 1260 1261rm tmp* 1262 1263println "\n===> zstd --list/-l single frame tests " 1264datagen > tmp1 1265datagen > tmp2 1266datagen > tmp3 1267zstd tmp* 1268zstd -l ./*.zst 1269zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header 1270zstd --list ./*.zst 1271zstd --list -v ./*.zst 1272 1273println "\n===> zstd --list/-l multiple frame tests " 1274cat tmp1.zst tmp2.zst > tmp12.zst 1275cat tmp12.zst tmp3.zst > tmp123.zst 1276zstd -l ./*.zst 1277zstd -lv ./*.zst 1278 1279println "\n===> zstd --list/-l error detection tests " 1280zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" 1281zstd --list tmp* && die "-l must fail on non-zstd file" 1282zstd -lv tmp1* && die "-l must fail on non-zstd file" 1283zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" 1284 1285println "test : detect truncated compressed file " 1286TEST_DATA_FILE=truncatable-input.txt 1287FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst 1288TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst 1289datagen -g50000 > $TEST_DATA_FILE 1290zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE 1291dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE 1292zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" 1293 1294rm $TEST_DATA_FILE 1295rm $FULL_COMPRESSED_FILE 1296rm $TRUNCATED_COMPRESSED_FILE 1297 1298println "\n===> zstd --list/-l errors when presented with stdin / no files" 1299zstd -l && die "-l must fail on empty list of files" 1300zstd -l - && die "-l does not work on stdin" 1301zstd -l < tmp1.zst && die "-l does not work on stdin" 1302zstd -l - < tmp1.zst && die "-l does not work on stdin" 1303zstd -l - tmp1.zst && die "-l does not work on stdin" 1304zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" 1305zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty 1306 1307println "\n===> zstd --list/-l test with null files " 1308datagen -g0 > tmp5 1309zstd tmp5 1310zstd -l tmp5.zst 1311zstd -l tmp5* && die "-l must fail on non-zstd file" 1312zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header 1313zstd -lv tmp5* && die "-l must fail on non-zstd file" 1314 1315println "\n===> zstd --list/-l test with no content size field " 1316datagen -g513K | zstd > tmp6.zst 1317zstd -l tmp6.zst 1318zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" 1319 1320println "\n===> zstd --list/-l test with no checksum " 1321zstd -f --no-check tmp1 1322zstd -l tmp1.zst 1323zstd -lv tmp1.zst 1324 1325rm tmp* 1326 1327 1328println "\n===> zstd long distance matching tests " 1329roundTripTest -g0 " --single-thread --long" 1330roundTripTest -g9M "2 --single-thread --long" 1331# Test parameter parsing 1332roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" 1333roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" 1334roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" 1335roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" 1336 1337 1338println "\n===> zstd long distance matching with optimal parser compressed size tests " 1339optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) 1340longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) 1341optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) 1342longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) 1343optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) 1344longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) 1345if [ "$longCSize16" -gt "$optCSize16" ]; then 1346 echo using --long on compression level 16 should not cause compressed size regression 1347 exit 1 1348elif [ "$longCSize19" -gt "$optCSize19" ]; then 1349 echo using --long on compression level 19 should not cause compressed size regression 1350 exit 1 1351elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then 1352 echo using --long on compression level 19 with wLog=23 should not cause compressed size regression 1353 exit 1 1354fi 1355 1356 1357if [ "$1" != "--test-large-data" ]; then 1358 println "Skipping large data tests" 1359 exit 0 1360fi 1361 1362 1363############################################################################# 1364 1365 1366if [ -n "$hasMT" ] 1367then 1368 println "\n===> adaptive mode " 1369 roundTripTest -g270000000 " --adapt" 1370 roundTripTest -g27000000 " --adapt=min=1,max=4" 1371 println "===> test: --adapt must fail on incoherent bounds " 1372 datagen > tmp 1373 zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" 1374 1375 println "\n===> rsyncable mode " 1376 roundTripTest -g10M " --rsyncable" 1377 roundTripTest -g10M " --rsyncable -B100K" 1378 println "===> test: --rsyncable must fail with --single-thread" 1379 zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" 1380fi 1381 1382println "\n===> patch-from=origin tests" 1383datagen -g1000 -P50 > tmp_dict 1384datagen -g1000 -P10 > tmp_patch 1385zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff 1386zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon 1387$DIFF -s tmp_patch_recon tmp_patch 1388 1389println "\n===> alternate syntax: patch-from origin" 1390zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff 1391zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon 1392$DIFF -s tmp_patch_recon tmp_patch 1393rm -rf tmp_* 1394 1395println "\n===> patch-from recursive tests" 1396mkdir tmp_dir 1397datagen > tmp_dir/tmp1 1398datagen > tmp_dir/tmp2 1399datagen > tmp_dict 1400zstd --patch-from=tmp_dict -r tmp_dir && die 1401rm -rf tmp* 1402 1403println "\n===> patch-from long mode trigger larger file test" 1404datagen -g5000000 > tmp_dict 1405datagen -g5000000 > tmp_patch 1406zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" 1407rm -rf tmp* 1408 1409println "\n===> patch-from --stream-size test" 1410datagen -g1000 -P50 > tmp_dict 1411datagen -g1000 -P10 > tmp_patch 1412cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die 1413cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff 1414rm -rf tmp* 1415 1416println "\n===> large files tests " 1417 1418roundTripTest -g270000000 1 1419roundTripTest -g250000000 2 1420roundTripTest -g230000000 3 1421 1422roundTripTest -g140000000 -P60 4 1423roundTripTest -g130000000 -P62 5 1424roundTripTest -g120000000 -P65 6 1425 1426roundTripTest -g70000000 -P70 7 1427roundTripTest -g60000000 -P71 8 1428roundTripTest -g50000000 -P73 9 1429 1430roundTripTest -g35000000 -P75 10 1431roundTripTest -g30000000 -P76 11 1432roundTripTest -g25000000 -P78 12 1433 1434roundTripTest -g18000013 -P80 13 1435roundTripTest -g18000014 -P80 14 1436roundTripTest -g18000015 -P81 15 1437roundTripTest -g18000016 -P84 16 1438roundTripTest -g18000017 -P88 17 1439roundTripTest -g18000018 -P94 18 1440roundTripTest -g18000019 -P96 19 1441 1442roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" 1443roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale 1444 1445fileRoundTripTest -g4193M -P99 1 1446 1447 1448println "\n===> zstd long, long distance matching round-trip tests " 1449roundTripTest -g270000000 "1 --single-thread --long" 1450roundTripTest -g130000000 -P60 "5 --single-thread --long" 1451roundTripTest -g35000000 -P70 "8 --single-thread --long" 1452roundTripTest -g18000001 -P80 "18 --single-thread --long" 1453# Test large window logs 1454roundTripTest -g700M -P50 "1 --single-thread --long=29" 1455roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" 1456 1457 1458if [ -n "$hasMT" ] 1459then 1460 println "\n===> zstdmt long round-trip tests " 1461 roundTripTest -g80000000 -P99 "19 -T2" " " 1462 roundTripTest -g5000000000 -P99 "1 -T2" " " 1463 roundTripTest -g500000000 -P97 "1 -T999" " " 1464 fileRoundTripTest -g4103M -P98 " -T0" " " 1465 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " 1466 # Exposes the bug in https://github.com/facebook/zstd/pull/1678 1467 # This test fails on 4 different travis builds at the time of writing 1468 # because it needs to allocate 8 GB of memory. 1469 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" 1470else 1471 println "\n**** no multithreading, skipping zstdmt tests **** " 1472fi 1473 1474 1475println "\n===> cover dictionary builder : advanced options " 1476 1477TESTFILE="$PRGDIR"/zstdcli.c 1478datagen > tmpDict 1479println "- Create first dictionary" 1480zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1481cp "$TESTFILE" tmp 1482zstd -f tmp -D tmpDict 1483zstd -d tmp.zst -D tmpDict -fo result 1484$DIFF "$TESTFILE" result 1485zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" 1486println "- Create second (different) dictionary" 1487zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1488zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1489println "- Create dictionary using shrink-dict flag" 1490zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict 1491zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 1492zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 1493println "- Create dictionary with short dictID" 1494zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1495cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1496println "- Create dictionary with size limit" 1497zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1498println "- Compare size of dictionary from 90% training samples with 80% training samples" 1499zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1500zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1501println "- Create dictionary using all samples for both training and testing" 1502zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1503println "- Test -o before --train-cover" 1504rm -f tmpDict dictionary 1505zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1506test -f tmpDict 1507zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1508test -f dictionary 1509rm -f tmp* dictionary 1510 1511rm -f tmp* 1512