1#!/bin/sh 2 3set -e 4 5die() { 6 println "$@" 1>&2 7 exit 1 8} 9 10datagen() { 11 "$DATAGEN_BIN" "$@" 12} 13 14zstd() { 15 if [ -z "$EXEC_PREFIX" ]; then 16 "$ZSTD_BIN" "$@" 17 else 18 "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 19 fi 20} 21 22sudoZstd() { 23 if [ -z "$EXEC_PREFIX" ]; then 24 sudo "$ZSTD_BIN" "$@" 25 else 26 sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" 27 fi 28} 29 30roundTripTest() { 31 if [ -n "$3" ]; then 32 cLevel="$3" 33 proba="$2" 34 else 35 cLevel="$2" 36 proba="" 37 fi 38 if [ -n "$4" ]; then 39 dLevel="$4" 40 else 41 dLevel="$cLevel" 42 fi 43 44 rm -f tmp1 tmp2 45 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" 46 datagen $1 $proba | $MD5SUM > tmp1 47 datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 48 $DIFF -q tmp1 tmp2 49} 50 51fileRoundTripTest() { 52 if [ -n "$3" ]; then 53 local_c="$3" 54 local_p="$2" 55 else 56 local_c="$2" 57 local_p="" 58 fi 59 if [ -n "$4" ]; then 60 local_d="$4" 61 else 62 local_d="$local_c" 63 fi 64 65 rm -f tmp.zst tmp.md5.1 tmp.md5.2 66 println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" 67 datagen $1 $local_p > tmp 68 < tmp $MD5SUM > tmp.md5.1 69 zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 70 $DIFF -q tmp.md5.1 tmp.md5.2 71} 72 73truncateLastByte() { 74 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" 75} 76 77println() { 78 printf '%b\n' "${*}" 79} 80 81if [ -z "${size}" ]; then 82 size= 83else 84 size=${size} 85fi 86 87SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 88PRGDIR="$SCRIPT_DIR/../programs" 89TESTDIR="$SCRIPT_DIR/../tests" 90UNAME=$(uname) 91ZSTDGREP="$PRGDIR/zstdgrep" 92 93detectedTerminal=false 94if [ -t 0 ] && [ -t 1 ] 95then 96 detectedTerminal=true 97fi 98isTerminal=${isTerminal:-$detectedTerminal} 99 100isWindows=false 101INTOVOID="/dev/null" 102case "$UNAME" in 103 GNU) DEVDEVICE="/dev/random" ;; 104 *) DEVDEVICE="/dev/zero" ;; 105esac 106case "$OS" in 107 Windows*) 108 isWindows=true 109 INTOVOID="NUL" 110 DEVDEVICE="NUL" 111 ;; 112esac 113 114case "$UNAME" in 115 Darwin) MD5SUM="md5 -r" ;; 116 FreeBSD) MD5SUM="gmd5sum" ;; 117 NetBSD) MD5SUM="md5 -n" ;; 118 OpenBSD) MD5SUM="md5" ;; 119 *) MD5SUM="md5sum" ;; 120esac 121 122MTIME="stat -c %Y" 123case "$UNAME" in 124 Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; 125esac 126 127GET_PERMS="stat -c %a" 128case "$UNAME" in 129 Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; 130esac 131 132assertFilePermissions() { 133 STAT1=$($GET_PERMS "$1") 134 STAT2=$2 135 [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" 136} 137 138assertSamePermissions() { 139 STAT1=$($GET_PERMS "$1") 140 STAT2=$($GET_PERMS "$2") 141 [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" 142} 143 144DIFF="diff" 145case "$UNAME" in 146 SunOS) DIFF="gdiff" ;; 147esac 148 149 150# check if ZSTD_BIN is defined. if not, use the default value 151if [ -z "${ZSTD_BIN}" ]; then 152 println "\nZSTD_BIN is not set. Using the default value..." 153 ZSTD_BIN="$PRGDIR/zstd" 154fi 155 156# check if DATAGEN_BIN is defined. if not, use the default value 157if [ -z "${DATAGEN_BIN}" ]; then 158 println "\nDATAGEN_BIN is not set. Using the default value..." 159 DATAGEN_BIN="$TESTDIR/datagen" 160fi 161 162ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" 163 164# assertions 165[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd follwing the instructions in README.md" 166[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " 167println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" 168 169if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' 170then 171 hasMT="" 172else 173 hasMT="true" 174fi 175 176 177 178println "\n===> simple tests " 179 180datagen > tmp 181println "test : basic compression " 182zstd -f tmp # trivial compression case, creates tmp.zst 183println "test : basic decompression" 184zstd -df tmp.zst # trivial decompression case (overwrites tmp) 185println "test : too large compression level => auto-fix" 186zstd -99 -f tmp # too large compression level, automatic sized down 187zstd -5000000000 -f tmp && die "too large numeric value : must fail" 188println "test : --fast aka negative compression levels" 189zstd --fast -f tmp # == -1 190zstd --fast=3 -f tmp # == -3 191zstd --fast=200000 -f tmp # too low compression level, automatic fixed 192zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" 193zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" 194println "test : too large numeric argument" 195zstd --fast=9999999999 -f tmp && die "should have refused numeric value" 196println "test : set compression level with environment variable ZSTD_CLEVEL" 197ZSTD_CLEVEL=12 zstd -f tmp # positive compression level 198ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level 199ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign 200ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting 201ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting 202ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting 203ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting 204ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting 205ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting 206println "test : override ZSTD_CLEVEL with command line option" 207ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option 208println "test : compress to stdout" 209zstd tmp -c > tmpCompressed 210zstd tmp --stdout > tmpCompressed # long command format 211println "test : compress to named file" 212rm -f tmpCompressed 213zstd tmp -o tmpCompressed 214test -f tmpCompressed # file must be created 215println "test : force write, correct order" 216zstd tmp -fo tmpCompressed 217println "test : forgotten argument" 218cp tmp tmp2 219zstd tmp2 -fo && die "-o must be followed by filename " 220println "test : implied stdout when input is stdin" 221println bob | zstd | zstd -d 222if [ "$isTerminal" = true ]; then 223println "test : compressed data to terminal" 224println bob | zstd && die "should have refused : compressed data to terminal" 225println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" 226zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" 227fi 228println "test : null-length file roundtrip" 229println -n '' | zstd - --stdout | zstd -d --stdout 230println "test : ensure small file doesn't add 3-bytes null block" 231datagen -g1 > tmp1 232zstd tmp1 -c | wc -c | grep "14" 233zstd < tmp1 | wc -c | grep "14" 234println "test : decompress file with wrong suffix (must fail)" 235zstd -d tmpCompressed && die "wrong suffix error not detected!" 236zstd -df tmp && die "should have refused : wrong extension" 237println "test : decompress into stdout" 238zstd -d tmpCompressed -c > tmpResult # decompression using stdout 239zstd --decompress tmpCompressed -c > tmpResult 240zstd --decompress tmpCompressed --stdout > tmpResult 241println "test : decompress from stdin into stdout" 242zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout 243zstd -dc - < tmp.zst > $INTOVOID 244zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used 245zstd -d - < tmp.zst > $INTOVOID 246println "test : impose memory limitation (must fail)" 247zstd -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" 248zstd -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 249zstd -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 250zstd -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 251println "test : overwrite protection" 252zstd -q tmp && die "overwrite check failed!" 253println "test : force overwrite" 254zstd -q -f tmp 255zstd -q --force tmp 256println "test : overwrite readonly file" 257rm -f tmpro tmpro.zst 258println foo > tmpro.zst 259println foo > tmpro 260chmod 400 tmpro.zst 261zstd -q tmpro && die "should have refused to overwrite read-only file" 262zstd -q -f tmpro 263println "test: --no-progress flag" 264zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress 265zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress 266rm -f tmpro tmpro.zst 267println "test: overwrite input file (must fail)" 268zstd tmp -fo tmp && die "zstd compression overwrote the input file" 269zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" 270println "test: detect that input file does not exist" 271zstd nothere && die "zstd hasn't detected that input file does not exist" 272println "test: --[no-]compress-literals" 273zstd tmp -c --no-compress-literals -1 | zstd -t 274zstd tmp -c --no-compress-literals --fast=1 | zstd -t 275zstd tmp -c --no-compress-literals -19 | zstd -t 276zstd tmp -c --compress-literals -1 | zstd -t 277zstd tmp -c --compress-literals --fast=1 | zstd -t 278zstd tmp -c --compress-literals -19 | zstd -t 279zstd -b --fast=1 -i0e1 tmp --compress-literals 280zstd -b --fast=1 -i0e1 tmp --no-compress-literals 281println "test: --no-check for decompression" 282zstd -f tmp -o tmp_corrupt.zst --check 283zstd -f tmp -o tmp.zst --no-check 284printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp 285zstd -d -f tmp_corrupt.zst --no-check 286zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides 287zstd -d -f tmp.zst --no-check 288 289println "\n===> zstdgrep tests" 290ln -sf "$ZSTD_BIN" zstdcat 291rm -f tmp_grep 292echo "1234" > tmp_grep 293zstd -f tmp_grep 294lines=$(ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) 295test 2 -eq $lines 296ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" 297ZCAT=./zstdcat "$ZSTDGREP" 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true 298rm -f tmp_grep* 299 300println "\n===> --exclude-compressed flag" 301rm -rf precompressedFilterTestDir 302mkdir -p precompressedFilterTestDir 303datagen $size > precompressedFilterTestDir/input.5 304datagen $size > precompressedFilterTestDir/input.6 305zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 306datagen $size > precompressedFilterTestDir/input.7 307datagen $size > precompressedFilterTestDir/input.8 308zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 309test ! -f precompressedFilterTestDir/input.5.zst.zst 310test ! -f precompressedFilterTestDir/input.6.zst.zst 311file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` 312file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` 313if [ $file2timestamp -ge $file1timestamp ]; then 314 println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." 315else 316 println "Test is not successful" 317fi 318# File Extension check. 319datagen $size > precompressedFilterTestDir/input.zstbar 320zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 321# zstd should compress input.zstbar 322test -f precompressedFilterTestDir/input.zstbar.zst 323# Check without the --exclude-compressed flag 324zstd --long --rm -r precompressedFilterTestDir 325# Files should get compressed again without the --exclude-compressed flag. 326test -f precompressedFilterTestDir/input.5.zst.zst 327test -f precompressedFilterTestDir/input.6.zst.zst 328println "Test completed" 329 330 331 332println "\n===> warning prompts should not occur if stdin is an input" 333println "y" > tmpPrompt 334println "hello world" >> tmpPrompt 335zstd tmpPrompt -f 336zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 337zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f 338zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated 339$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed 340 341echo 'yes' | zstd tmpPrompt -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files 342echo 'yes' | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 343zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" 344 345println "Test completed" 346 347 348println "\n===> recursive mode test " 349# combination of -r with empty list of input file 350zstd -c -r < tmp > tmp.zst 351 352 353println "\n===> file removal" 354zstd -f --rm tmp 355test ! -f tmp # tmp should no longer be present 356zstd -f -d --rm tmp.zst 357test ! -f tmp.zst # tmp.zst should no longer be present 358println "test : should quietly not remove non-regular file" 359println hello > tmp 360zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" 361grep -v "Refusing to remove non-regular file" tmplog 362rm -f tmplog 363zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" 364println "test : --rm on stdin" 365println a | zstd --rm > $INTOVOID # --rm should remain silent 366rm -f tmp 367zstd -f tmp && die "tmp not present : should have failed" 368test ! -f tmp.zst # tmp.zst should not be created 369println "test : -d -f do not delete destination when source is not present" 370touch tmp # create destination file 371zstd -d -f tmp.zst && die "attempt to decompress a non existing file" 372test -f tmp # destination file should still be present 373println "test : -f do not delete destination when source is not present" 374rm -f tmp # erase source file 375touch tmp.zst # create destination file 376zstd -f tmp && die "attempt to compress a non existing file" 377test -f tmp.zst # destination file should still be present 378rm -rf tmp* # may also erase tmp* directory from previous failed run 379 380 381println "\n===> decompression only tests " 382# the following test verifies that the decoder is compatible with RLE as first block 383# older versions of zstd cli are not able to decode such corner case. 384# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. 385dd bs=1048576 count=1 if=/dev/zero of=tmp 386zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" 387$DIFF -s tmp1 tmp 388rm -f tmp* 389 390 391println "\n===> compress multiple files" 392println hello > tmp1 393println world > tmp2 394zstd tmp1 tmp2 -o "$INTOVOID" -f 395zstd tmp1 tmp2 -c | zstd -t 396zstd tmp1 tmp2 -o tmp.zst 397test ! -f tmp1.zst 398test ! -f tmp2.zst 399zstd tmp1 tmp2 400zstd -t tmp1.zst tmp2.zst 401zstd -dc tmp1.zst tmp2.zst 402zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f 403zstd -d tmp1.zst tmp2.zst -o tmp 404touch tmpexists 405zstd tmp1 tmp2 -f -o tmpexists 406zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" 407println gooder > tmp_rm1 408println boi > tmp_rm2 409println worldly > tmp_rm3 410echo 'y' | zstd tmp_rm1 tmp_rm2 -o tmp_rm3.zst --rm # tests the warning prompt for --rm with multiple inputs into once source 411test ! -f tmp_rm1 412test ! -f tmp_rm2 413cp tmp_rm3.zst tmp_rm4.zst 414echo 'Y' | zstd -d tmp_rm3.zst tmp_rm4.zst -o tmp_rm_out --rm 415test ! -f tmp_rm3.zst 416test ! -f tmp_rm4.zst 417echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 418echo 'yes' | zstd tmp_rm_out tmp_rm3 -c --rm -v && die "compressing multiple files to stdout with --rm should fail unless -f is specified" 419println gooder > tmpexists1 420zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID 421 422# Bug: PR #972 423if [ "$?" -eq 139 ]; then 424 die "should not have segfaulted" 425fi 426println "\n===> multiple files and shell completion " 427datagen -s1 > tmp1 2> $INTOVOID 428datagen -s2 -g100K > tmp2 2> $INTOVOID 429datagen -s3 -g1M > tmp3 2> $INTOVOID 430println "compress tmp* : " 431zstd -f tmp* 432test -f tmp1.zst 433test -f tmp2.zst 434test -f tmp3.zst 435rm -f tmp1 tmp2 tmp3 436println "decompress tmp* : " 437zstd -df ./*.zst 438test -f tmp1 439test -f tmp2 440test -f tmp3 441println "compress tmp* into stdout > tmpall : " 442zstd -c tmp1 tmp2 tmp3 > tmpall 443test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) 444println "decompress tmpall* into stdout > tmpdec : " 445cp tmpall tmpall2 446zstd -dc tmpall* > tmpdec 447test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) 448println "compress multiple files including a missing one (notHere) : " 449zstd -f tmp1 notHere tmp2 && die "missing file not detected!" 450rm -f tmp* 451 452 453if [ "$isWindows" = false ] ; then 454 println "\n===> zstd fifo named pipe test " 455 echo "Hello World!" > tmp_original 456 mkfifo tmp_named_pipe 457 # note : fifo test doesn't work in combination with `dd` or `cat` 458 echo "Hello World!" > tmp_named_pipe & 459 zstd tmp_named_pipe -o tmp_compressed 460 zstd -d -o tmp_decompressed tmp_compressed 461 $DIFF -s tmp_original tmp_decompressed 462 rm -rf tmp* 463fi 464 465println "\n===> zstd created file permissions tests" 466if [ "$isWindows" = false ] ; then 467 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove 468 469 ORIGINAL_UMASK=$(umask) 470 umask 0000 471 472 datagen > tmp1 473 datagen > tmp2 474 assertFilePermissions tmp1 666 475 assertFilePermissions tmp2 666 476 477 println "test : copy 666 permissions in file -> file compression " 478 zstd -f tmp1 -o tmp1.zst 479 assertSamePermissions tmp1 tmp1.zst 480 println "test : copy 666 permissions in file -> file decompression " 481 zstd -f -d tmp1.zst -o tmp1.out 482 assertSamePermissions tmp1.zst tmp1.out 483 484 rm -f tmp1.zst tmp1.out 485 486 println "test : copy 400 permissions in file -> file compression (write to a read-only file) " 487 chmod 0400 tmp1 488 assertFilePermissions tmp1 400 489 zstd -f tmp1 -o tmp1.zst 490 assertSamePermissions tmp1 tmp1.zst 491 println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " 492 zstd -f -d tmp1.zst -o tmp1 493 assertSamePermissions tmp1.zst tmp1 494 495 rm -f tmp1.zst tmp1.out 496 497 println "test : check created permissions from stdin input in compression " 498 zstd -f -o tmp1.zst < tmp1 499 assertFilePermissions tmp1.zst 666 500 println "test : check created permissions from stdin input in decompression " 501 zstd -f -d -o tmp1.out < tmp1.zst 502 assertFilePermissions tmp1.out 666 503 504 rm -f tmp1.zst tmp1.out 505 506 println "test : check created permissions from multiple inputs in compression " 507 zstd -f tmp1 tmp2 -o tmp1.zst 508 assertFilePermissions tmp1.zst 666 509 println "test : check created permissions from multiple inputs in decompression " 510 cp tmp1.zst tmp2.zst 511 zstd -f -d tmp1.zst tmp2.zst -o tmp1.out 512 assertFilePermissions tmp1.out 666 513 514 rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out 515 516 println "test : check permissions on pre-existing output file in compression " 517 chmod 0600 tmp1 518 touch tmp1.zst 519 chmod 0400 tmp1.zst 520 zstd -f tmp1 -o tmp1.zst 521 assertFilePermissions tmp1.zst 600 522 println "test : check permissions on pre-existing output file in decompression " 523 chmod 0400 tmp1.zst 524 touch tmp1.out 525 chmod 0200 tmp1.out 526 zstd -f -d tmp1.zst -o tmp1.out 527 assertFilePermissions tmp1.out 400 528 529 rm -f tmp1.zst tmp1.out 530 531 umask 0666 532 chmod 0666 tmp1 tmp2 533 534 println "test : respect umask when copying permissions in file -> file compression " 535 zstd -f tmp1 -o tmp1.zst 536 assertFilePermissions tmp1.zst 0 537 println "test : respect umask when copying permissions in file -> file decompression " 538 chmod 0666 tmp1.zst 539 zstd -f -d tmp1.zst -o tmp1.out 540 assertFilePermissions tmp1.out 0 541 542 rm -f tmp1.zst tmp1.out 543 544 println "test : respect umask when compressing from stdin input " 545 zstd -f -o tmp1.zst < tmp1 546 assertFilePermissions tmp1.zst 0 547 println "test : respect umask when decompressing from stdin input " 548 chmod 0666 tmp1.zst 549 zstd -f -d -o tmp1.out < tmp1.zst 550 assertFilePermissions tmp1.out 0 551 552 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out 553 umask $ORIGINAL_UMASK 554fi 555 556if [ -n "$DEVNULLRIGHTS" ] ; then 557 # these tests requires sudo rights, which is uncommon. 558 # they are only triggered if DEVNULLRIGHTS macro is defined. 559 println "\n===> checking /dev/null permissions are unaltered " 560 datagen > tmp 561 sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions 562 sudoZstd tmp -c > $INTOVOID 563 zstd tmp -f -o tmp.zst 564 sudoZstd -d tmp.zst -c > $INTOVOID 565 sudoZstd -d tmp.zst -o $INTOVOID 566 ls -las $INTOVOID | grep "rw-rw-rw-" 567fi 568 569if [ -n "$READFROMBLOCKDEVICE" ] ; then 570 # This creates a temporary block device, which is only possible on unix-y 571 # systems, is somewhat invasive, and requires sudo. For these reasons, you 572 # have to specifically ask for this test. 573 println "\n===> checking that zstd can read from a block device" 574 datagen -g65536 > tmp.img 575 sudo losetup -fP tmp.img 576 LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:) 577 [ -z "$LOOP_DEV" ] && die "failed to get loopback device" 578 sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" 579 sudoZstd -f $LOOP_DEV -c > tmp.img.zst 580 zstd -d tmp.img.zst -o tmp.img.copy 581 sudo losetup -d $LOOP_DEV 582 $DIFF -s tmp.img tmp.img.copy || die "round trip failed" 583 rm -f tmp.img tmp.img.zst tmp.img.copy 584fi 585 586println "\n===> compress multiple files into an output directory, --output-dir-flat" 587println henlo > tmp1 588mkdir tmpInputTestDir 589mkdir tmpInputTestDir/we 590mkdir tmpInputTestDir/we/must 591mkdir tmpInputTestDir/we/must/go 592mkdir tmpInputTestDir/we/must/go/deeper 593println cool > tmpInputTestDir/we/must/go/deeper/tmp2 594mkdir tmpOutDir 595zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir 596test -f tmpOutDir/tmp1.zst 597test -f tmpOutDir/tmp2.zst 598println "test : decompress multiple files into an output directory, --output-dir-flat" 599mkdir tmpOutDirDecomp 600zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp 601test -f tmpOutDirDecomp/tmp2 602test -f tmpOutDirDecomp/tmp1 603rm -f tmpOutDirDecomp/* 604zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp 605test -f tmpOutDirDecomp/tmp2 606test -f tmpOutDirDecomp/tmp1 607rm -rf tmp* 608 609if [ "$isWindows" = false ] ; then 610 println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" 611 println "test --output-dir-mirror" > tmp1 612 mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. 613 println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 614 zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir 615 test -f tmpOutDir/tmp1.zst 616 test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst 617 618 println "test: compress input dir will be ignored if it has '..'" 619 zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 620 zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" 621 zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 622 test ! -d non-exist 623 624 println "test: compress input dir should succeed with benign uses of '..'" 625 zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout 626 test -d tmpout 627 628 println "test : decompress multiple files into an output directory, --output-dir-mirror" 629 zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp 630 test -f tmpOutDirDecomp/tmpOutDir/tmp1 631 test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 632 633 println "test: decompress input dir will be ignored if it has '..'" 634 zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" 635 test ! -d non-exist 636 637 rm -rf tmp* 638fi 639 640 641println "test : compress multiple files reading them from a file, --filelist=FILE" 642println "Hello world!, file1" > tmp1 643println "Hello world!, file2" > tmp2 644println tmp1 > tmp_fileList 645println tmp2 >> tmp_fileList 646zstd -f --filelist=tmp_fileList 647test -f tmp2.zst 648test -f tmp1.zst 649 650println "test : alternate syntax: --filelist FILE" 651zstd -f --filelist tmp_fileList 652test -f tmp2.zst 653test -f tmp1.zst 654 655println "test : reading file list from a symlink, --filelist=FILE" 656rm -f *.zst 657ln -s tmp_fileList tmp_symLink 658zstd -f --filelist=tmp_symLink 659test -f tmp2.zst 660test -f tmp1.zst 661 662println "test : compress multiple files reading them from multiple files, --filelist=FILE" 663rm -f *.zst 664println "Hello world!, file3" > tmp3 665println "Hello world!, file4" > tmp4 666println tmp3 > tmp_fileList2 667println tmp4 >> tmp_fileList2 668zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 669test -f tmp1.zst 670test -f tmp2.zst 671test -f tmp3.zst 672test -f tmp4.zst 673 674println "test : decompress multiple files reading them from a file, --filelist=FILE" 675rm -f tmp1 tmp2 676println tmp1.zst > tmpZst 677println tmp2.zst >> tmpZst 678zstd -d -f --filelist=tmpZst 679test -f tmp1 680test -f tmp2 681 682println "test : decompress multiple files reading them from multiple files, --filelist=FILE" 683rm -f tmp1 tmp2 tmp3 tmp4 684println tmp3.zst > tmpZst2 685println tmp4.zst >> tmpZst2 686zstd -d -f --filelist=tmpZst --filelist=tmpZst2 687test -f tmp1 688test -f tmp2 689test -f tmp3 690test -f tmp4 691 692println "test : survive a list of files which is text garbage (--filelist=FILE)" 693datagen > tmp_badList 694zstd -f --filelist=tmp_badList && die "should have failed : list is text garbage" 695 696println "test : survive a list of files which is binary garbage (--filelist=FILE)" 697datagen -P0 -g1M > tmp_badList 698zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console 699 700println "test : try to overflow internal list of files (--filelist=FILE)" 701touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 702ls tmp* > tmpList 703zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list 704rm -rf tmp* 705 706println "\n===> --[no-]content-size tests" 707 708datagen > tmp_contentsize 709zstd -f tmp_contentsize 710zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 711zstd -f --no-content-size tmp_contentsize 712zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 713zstd -f --content-size tmp_contentsize 714zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" 715zstd -f --content-size --no-content-size tmp_contentsize 716zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die 717rm -rf tmp* 718 719println "test : show-default-cparams regular" 720datagen > tmp 721zstd --show-default-cparams -f tmp 722rm -rf tmp* 723 724println "test : show-default-cparams recursive" 725mkdir tmp_files 726datagen -g15000 > tmp_files/tmp1 727datagen -g129000 > tmp_files/tmp2 728datagen -g257000 > tmp_files/tmp3 729zstd --show-default-cparams -f -r tmp_files 730rm -rf tmp* 731 732println "\n===> Advanced compression parameters " 733println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" 734println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" 735println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" 736println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist 737test ! -f tmp.zst # tmp.zst should not be created 738roundTripTest -g512K 739roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" 740roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" 741roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" 742roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" 743roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" 744roundTripTest -g64K "19 --zstd=strat=9" # btultra2 745 746 747println "\n===> Pass-Through mode " 748println "Hello world 1!" | zstd -df 749println "Hello world 2!" | zstd -dcf 750println "Hello world 3!" > tmp1 751zstd -dcf tmp1 752 753 754println "\n===> frame concatenation " 755println "hello " > hello.tmp 756println "world!" > world.tmp 757cat hello.tmp world.tmp > helloworld.tmp 758zstd -c hello.tmp > hello.zst 759zstd -c world.tmp > world.zst 760cat hello.zst world.zst > helloworld.zst 761zstd -dc helloworld.zst > result.tmp 762cat result.tmp 763$DIFF helloworld.tmp result.tmp 764println "frame concatenation without checksum" 765zstd -c hello.tmp > hello.zst --no-check 766zstd -c world.tmp > world.zst --no-check 767cat hello.zst world.zst > helloworld.zstd 768zstd -dc helloworld.zst > result.tmp 769$DIFF helloworld.tmp result.tmp 770println "testing zstdcat symlink" 771ln -sf "$ZSTD_BIN" zstdcat 772$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp 773$DIFF helloworld.tmp result.tmp 774ln -s helloworld.zst helloworld.link.zst 775$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp 776$DIFF helloworld.tmp result.tmp 777rm -f zstdcat 778rm -f result.tmp 779println "testing zcat symlink" 780ln -sf "$ZSTD_BIN" zcat 781$EXE_PREFIX ./zcat helloworld.zst > result.tmp 782$DIFF helloworld.tmp result.tmp 783$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp 784$DIFF helloworld.tmp result.tmp 785rm -f zcat 786rm -f ./*.tmp ./*.zstd 787println "frame concatenation tests completed" 788 789 790if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then 791println "\n**** flush write error test **** " 792 793println "println foo | zstd > /dev/full" 794println foo | zstd > /dev/full && die "write error not detected!" 795println "println foo | zstd | zstd -d > /dev/full" 796println foo | zstd | zstd -d > /dev/full && die "write error not detected!" 797 798fi 799 800 801if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then 802 803println "\n===> symbolic link test " 804 805rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 806println "hello world" > hello.tmp 807ln -s hello.tmp world.tmp 808ln -s hello.tmp world2.tmp 809zstd world.tmp hello.tmp || true 810test -f hello.tmp.zst # regular file should have been compressed! 811test ! -f world.tmp.zst # symbolic link should not have been compressed! 812zstd world.tmp || true 813test ! -f world.tmp.zst # symbolic link should not have been compressed! 814zstd world.tmp world2.tmp || true 815test ! -f world.tmp.zst # symbolic link should not have been compressed! 816test ! -f world2.tmp.zst # symbolic link should not have been compressed! 817zstd world.tmp hello.tmp -f 818test -f world.tmp.zst # symbolic link should have been compressed with --force 819rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 820 821fi 822 823 824println "\n===> test sparse file support " 825 826datagen -g5M -P100 > tmpSparse 827zstd tmpSparse -c | zstd -dv -o tmpSparseRegen 828$DIFF -s tmpSparse tmpSparseRegen 829zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse 830$DIFF -s tmpSparse tmpOutSparse 831zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse 832$DIFF -s tmpSparse tmpOutNoSparse 833ls -ls tmpSparse* # look at file size and block size on disk 834datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) 835datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd 836ls -ls tmpSparseOdd # look at file size and block size on disk 837println "\n Sparse Compatibility with Console :" 838println "Hello World 1 !" | zstd | zstd -d -c 839println "Hello World 2 !" | zstd | zstd -d | cat 840println "\n Sparse Compatibility with Append :" 841datagen -P100 -g1M > tmpSparse1M 842cat tmpSparse1M tmpSparse1M > tmpSparse2M 843zstd -v -f tmpSparse1M -o tmpSparseCompressed 844zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated 845zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated 846ls -ls tmpSparse* # look at file size and block size on disk 847$DIFF tmpSparse2M tmpSparseRegenerated 848rm -f tmpSparse* 849 850 851println "\n===> stream-size mode" 852 853datagen -g11000 > tmp 854println "test : basic file compression vs sized streaming compression" 855file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 856stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) 857if [ "$stream_size" -gt "$file_size" ]; then 858 die "hinted compression larger than expected" 859fi 860println "test : sized streaming compression and decompression" 861cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 862zstd -df tmp.zst -o tmp_decompress 863cmp tmp tmp_decompress || die "difference between original and decompressed file" 864println "test : incorrect stream size" 865cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" 866 867println "\n===> zstd zero weight dict test " 868rm -f tmp* 869cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input 870zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 871zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 872$DIFF tmp_decomp tmp_input 873rm -rf tmp* 874 875println "\n===> zstd (valid) zero weight dict test " 876rm -f tmp* 877# 0 has a non-zero weight in the dictionary 878echo "0000000000000000000000000" > tmp_input 879zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 880zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 881$DIFF tmp_decomp tmp_input 882rm -rf tmp* 883 884println "\n===> size-hint mode" 885 886datagen -g11000 > tmp 887datagen -g11000 > tmp2 888datagen > tmpDict 889println "test : basic file compression vs hinted streaming compression" 890file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 891stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) 892if [ "$stream_size" -ge "$file_size" ]; then 893 die "hinted compression larger than expected" 894fi 895println "test : hinted streaming compression and decompression" 896cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 897zstd -df tmp.zst -o tmp_decompress 898cmp tmp tmp_decompress || die "difference between original and decompressed file" 899println "test : hinted streaming compression with dictionary" 900cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict 901println "test : multiple file compression with hints and dictionary" 902zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 903zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp 904zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 905cmp tmp.zst tmp1_.zst || die "first file's output differs" 906cmp tmp2.zst tmp2_.zst || die "second file's output differs" 907println "test : incorrect hinted stream sizes" 908cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high 909cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low 910cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high 911cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low 912 913 914println "\n===> dictionary tests " 915 916println "- test with raw dict (content only) " 917datagen > tmpDict 918datagen -g1M | $MD5SUM > tmp1 919datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 920$DIFF -q tmp1 tmp2 921println "- Create first dictionary " 922TESTFILE="$PRGDIR"/zstdcli.c 923zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 924cp "$TESTFILE" tmp 925println "- Test dictionary compression with tmpDict as an input file and dictionary" 926zstd -f tmpDict -D tmpDict && die "compression error not detected!" 927println "- Dictionary compression roundtrip" 928zstd -f tmp -D tmpDict 929zstd -d tmp.zst -D tmpDict -fo result 930$DIFF "$TESTFILE" result 931println "- Dictionary compression with hlog < clog" 932zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23 933println "- Dictionary compression with btlazy2 strategy" 934zstd -f tmp -D tmpDict --zstd=strategy=6 935zstd -d tmp.zst -D tmpDict -fo result 936$DIFF "$TESTFILE" result 937if [ -n "$hasMT" ] 938then 939 println "- Test dictionary compression with multithreading " 940 datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 941fi 942println "- Create second (different) dictionary " 943zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 944zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 945println "- Create dictionary with short dictID" 946zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 947cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 948println "- Create dictionary with wrong dictID parameter order (must fail)" 949zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " 950println "- Create dictionary with size limit" 951zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v 952println "- Create dictionary with small size limit" 953zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v 954println "- Create dictionary with wrong parameter order (must fail)" 955zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " 956println "- Compress without dictID" 957zstd -f tmp -D tmpDict1 --no-dictID 958zstd -d tmp.zst -D tmpDict -fo result 959$DIFF "$TESTFILE" result 960println "- Compress multiple files with dictionary" 961rm -rf dirTestDict 962mkdir dirTestDict 963cp "$TESTDIR"/*.c dirTestDict 964cp "$PRGDIR"/*.c dirTestDict 965cp "$PRGDIR"/*.h dirTestDict 966$MD5SUM dirTestDict/* > tmph1 967zstd -f --rm dirTestDict/* -D tmpDictC 968zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default 969case "$UNAME" in 970 Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 971 *) $MD5SUM -c tmph1 ;; 972esac 973rm -rf dirTestDict 974println "- dictionary builder on bogus input" 975println "Hello World" > tmp 976zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" 977datagen -P0 -g10M > tmp 978zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" 979println "- Test -o before --train" 980rm -f tmpDict dictionary 981zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c 982test -f tmpDict 983zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c 984test -f dictionary 985println "- Test dictionary training fails" 986echo "000000000000000000000000000000000" > tmpz 987zstd --train tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 988if [ -n "$hasMT" ] 989then 990 zstd --train -T0 tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" 991 println "- Create dictionary with multithreading enabled" 992 zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 993fi 994rm -f tmp* dictionary 995 996 997println "\n===> fastCover dictionary builder : advanced options " 998TESTFILE="$PRGDIR"/zstdcli.c 999datagen > tmpDict 1000println "- Create first dictionary" 1001zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1002cp "$TESTFILE" tmp 1003zstd -f tmp -D tmpDict 1004zstd -d tmp.zst -D tmpDict -fo result 1005$DIFF "$TESTFILE" result 1006println "- Create second (different) dictionary" 1007zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1008zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1009zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" 1010println "- Create dictionary with short dictID" 1011zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1012cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1013println "- Create dictionaries with shrink-dict flag enabled" 1014zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict 1015zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 1016zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 1017println "- Create dictionary with size limit" 1018zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1019println "- Create dictionary using all samples for both training and testing" 1020zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1021println "- Create dictionary using f=16" 1022zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1023zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" 1024println "- Create dictionary using accel=2" 1025zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1026println "- Create dictionary using accel=10" 1027zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1028println "- Create dictionary with multithreading" 1029zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1030println "- Test -o before --train-fastcover" 1031rm -f tmpDict dictionary 1032zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 1033test -f tmpDict 1034zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 1035test -f dictionary 1036rm -f tmp* dictionary 1037 1038 1039println "\n===> legacy dictionary builder " 1040 1041TESTFILE="$PRGDIR"/zstdcli.c 1042datagen > tmpDict 1043println "- Create first dictionary" 1044zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1045cp "$TESTFILE" tmp 1046zstd -f tmp -D tmpDict 1047zstd -d tmp.zst -D tmpDict -fo result 1048$DIFF "$TESTFILE" result 1049zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" 1050println "- Create second (different) dictionary" 1051zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1052zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1053println "- Create dictionary with short dictID" 1054zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1055cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1056println "- Create dictionary with size limit" 1057zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1058println "- Test -o before --train-legacy" 1059rm -f tmpDict dictionary 1060zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 1061test -f tmpDict 1062zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 1063test -f dictionary 1064rm -f tmp* dictionary 1065 1066 1067println "\n===> integrity tests " 1068 1069println "test one file (tmp1.zst) " 1070datagen > tmp1 1071zstd tmp1 1072zstd -t tmp1.zst 1073zstd --test tmp1.zst 1074println "test multiple files (*.zst) " 1075zstd -t ./*.zst 1076println "test bad files (*) " 1077zstd -t ./* && die "bad files not detected !" 1078zstd -t tmp1 && die "bad file not detected !" 1079cp tmp1 tmp2.zst 1080zstd -t tmp2.zst && die "bad file not detected !" 1081datagen -g0 > tmp3 1082zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad 1083println "test --rm and --test combined " 1084zstd -t --rm tmp1.zst 1085test -f tmp1.zst # check file is still present 1086split -b16384 tmp1.zst tmpSplit. 1087zstd -t tmpSplit.* && die "bad file not detected !" 1088datagen | zstd -c | zstd -t 1089 1090 1091println "\n===> golden files tests " 1092 1093zstd -t -r "$TESTDIR/golden-decompression" 1094zstd -c -r "$TESTDIR/golden-compression" | zstd -t 1095zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t 1096 1097 1098println "\n===> benchmark mode tests " 1099 1100println "bench one file" 1101datagen > tmp1 1102zstd -bi0 tmp1 1103println "bench multiple levels" 1104zstd -i0b0e3 tmp1 1105println "bench negative level" 1106zstd -bi0 --fast tmp1 1107println "with recursive and quiet modes" 1108zstd -rqi0b1e2 tmp1 1109println "benchmark decompression only" 1110zstd -f tmp1 1111zstd -b -d -i0 tmp1.zst 1112 1113 1114println "\n===> zstd compatibility tests " 1115 1116datagen > tmp 1117rm -f tmp.zst 1118zstd --format=zstd -f tmp 1119test -f tmp.zst 1120 1121 1122println "\n===> gzip compatibility tests " 1123 1124GZIPMODE=1 1125zstd --format=gzip -V || GZIPMODE=0 1126if [ $GZIPMODE -eq 1 ]; then 1127 println "gzip support detected" 1128 GZIPEXE=1 1129 gzip -V || GZIPEXE=0 1130 if [ $GZIPEXE -eq 1 ]; then 1131 datagen > tmp 1132 zstd --format=gzip -f tmp 1133 gzip -t -v tmp.gz 1134 gzip -f tmp 1135 zstd -d -f -v tmp.gz 1136 rm -f tmp* 1137 else 1138 println "gzip binary not detected" 1139 fi 1140else 1141 println "gzip mode not supported" 1142fi 1143 1144 1145println "\n===> gzip frame tests " 1146 1147if [ $GZIPMODE -eq 1 ]; then 1148 datagen > tmp 1149 zstd -f --format=gzip tmp 1150 zstd -f tmp 1151 cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp 1152 truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1153 rm -f tmp* 1154else 1155 println "gzip mode not supported" 1156fi 1157 1158if [ $GZIPMODE -eq 1 ]; then 1159 datagen > tmp 1160 rm -f tmp.zst 1161 zstd --format=gzip --format=zstd -f tmp 1162 test -f tmp.zst 1163fi 1164 1165println "\n===> xz compatibility tests " 1166 1167LZMAMODE=1 1168zstd --format=xz -V || LZMAMODE=0 1169if [ $LZMAMODE -eq 1 ]; then 1170 println "xz support detected" 1171 XZEXE=1 1172 xz -Q -V && lzma -Q -V || XZEXE=0 1173 if [ $XZEXE -eq 1 ]; then 1174 println "Testing zstd xz and lzma support" 1175 datagen > tmp 1176 zstd --format=lzma -f tmp 1177 zstd --format=xz -f tmp 1178 xz -Q -t -v tmp.xz 1179 xz -Q -t -v tmp.lzma 1180 xz -Q -f -k tmp 1181 lzma -Q -f -k --lzma1 tmp 1182 zstd -d -f -v tmp.xz 1183 zstd -d -f -v tmp.lzma 1184 rm -f tmp* 1185 println "Creating symlinks" 1186 ln -s "$ZSTD_BIN" ./xz 1187 ln -s "$ZSTD_BIN" ./unxz 1188 ln -s "$ZSTD_BIN" ./lzma 1189 ln -s "$ZSTD_BIN" ./unlzma 1190 println "Testing xz and lzma symlinks" 1191 datagen > tmp 1192 ./xz tmp 1193 xz -Q -d tmp.xz 1194 ./lzma tmp 1195 lzma -Q -d tmp.lzma 1196 println "Testing unxz and unlzma symlinks" 1197 xz -Q tmp 1198 ./xz -d tmp.xz 1199 lzma -Q tmp 1200 ./lzma -d tmp.lzma 1201 rm -f xz unxz lzma unlzma 1202 rm -f tmp* 1203 else 1204 println "xz binary not detected" 1205 fi 1206else 1207 println "xz mode not supported" 1208fi 1209 1210 1211println "\n===> xz frame tests " 1212 1213if [ $LZMAMODE -eq 1 ]; then 1214 datagen > tmp 1215 zstd -f --format=xz tmp 1216 zstd -f --format=lzma tmp 1217 zstd -f tmp 1218 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp 1219 truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1220 truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1221 rm -f tmp* 1222else 1223 println "xz mode not supported" 1224fi 1225 1226println "\n===> lz4 compatibility tests " 1227 1228LZ4MODE=1 1229zstd --format=lz4 -V || LZ4MODE=0 1230if [ $LZ4MODE -eq 1 ]; then 1231 println "lz4 support detected" 1232 LZ4EXE=1 1233 lz4 -V || LZ4EXE=0 1234 if [ $LZ4EXE -eq 1 ]; then 1235 datagen > tmp 1236 zstd --format=lz4 -f tmp 1237 lz4 -t -v tmp.lz4 1238 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout 1239 zstd -d -f -v tmp.lz4 1240 rm -f tmp* 1241 else 1242 println "lz4 binary not detected" 1243 fi 1244else 1245 println "lz4 mode not supported" 1246fi 1247 1248 1249if [ $LZ4MODE -eq 1 ]; then 1250 println "\n===> lz4 frame tests " 1251 datagen > tmp 1252 zstd -f --format=lz4 tmp 1253 zstd -f tmp 1254 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp 1255 truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1256 rm -f tmp* 1257else 1258 println "\nlz4 mode not supported" 1259fi 1260 1261 1262println "\n===> suffix list test" 1263 1264! zstd -d tmp.abc 2> tmplg 1265 1266if [ $GZIPMODE -ne 1 ]; then 1267 grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1268fi 1269 1270if [ $LZMAMODE -ne 1 ]; then 1271 grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" 1272 grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1273fi 1274 1275if [ $LZ4MODE -ne 1 ]; then 1276 grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" 1277fi 1278 1279touch tmp1 1280zstd tmp1 -o tmp1.zstd 1281zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix 1282 1283println "\n===> tar extension tests " 1284 1285rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd 1286 1287datagen > tmp 1288tar cf tmp.tar tmp 1289zstd tmp.tar -o tmp.tzst 1290rm -f tmp.tar 1291zstd -d tmp.tzst 1292[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" 1293rm -f tmp.tar tmp.tzst 1294 1295if [ $GZIPMODE -eq 1 ]; then 1296 tar czf tmp.tgz tmp 1297 zstd -d tmp.tgz 1298 [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" 1299 rm -f tmp.tar tmp.tgz 1300fi 1301 1302if [ $LZMAMODE -eq 1 ]; then 1303 tar c tmp | zstd --format=xz > tmp.txz 1304 zstd -d tmp.txz 1305 [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" 1306 rm -f tmp.tar tmp.txz 1307fi 1308 1309if [ $LZ4MODE -eq 1 ]; then 1310 tar c tmp | zstd --format=lz4 > tmp.tlz4 1311 zstd -d tmp.tlz4 1312 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" 1313 rm -f tmp.tar tmp.tlz4 1314fi 1315 1316touch tmp.t tmp.tz tmp.tzs 1317! zstd -d tmp.t 1318! zstd -d tmp.tz 1319! zstd -d tmp.tzs 1320 1321 1322println "\n===> zstd round-trip tests " 1323 1324roundTripTest 1325roundTripTest -g15K # TableID==3 1326roundTripTest -g127K # TableID==2 1327roundTripTest -g255K # TableID==1 1328roundTripTest -g522K # TableID==0 1329roundTripTest -g519K 6 # greedy, hash chain 1330roundTripTest -g517K 16 # btlazy2 1331roundTripTest -g516K 19 # btopt 1332 1333fileRoundTripTest -g500K 1334 1335println "\n===> zstd long distance matching round-trip tests " 1336roundTripTest -g0 "2 --single-thread --long" 1337roundTripTest -g1000K "1 --single-thread --long" 1338roundTripTest -g517K "6 --single-thread --long" 1339roundTripTest -g516K "16 --single-thread --long" 1340roundTripTest -g518K "19 --single-thread --long" 1341roundTripTest -g2M "22 --single-thread --ultra --long" 1342fileRoundTripTest -g5M "3 --single-thread --long" 1343 1344 1345roundTripTest -g96K "5 --single-thread" 1346if [ -n "$hasMT" ] 1347then 1348 println "\n===> zstdmt round-trip tests " 1349 roundTripTest -g4M "1 -T0" 1350 roundTripTest -g8M "3 -T2" 1351 roundTripTest -g8M "19 --long" 1352 roundTripTest -g8000K "2 --threads=2" 1353 fileRoundTripTest -g4M "19 -T2 -B1M" 1354 1355 println "\n===> zstdmt long distance matching round-trip tests " 1356 roundTripTest -g8M "3 --long=24 -T2" 1357 1358 println "\n===> zstdmt environment variable tests " 1359 echo "multifoo" >> mt_tmp 1360 ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting 1361 ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting 1362 ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting 1363 ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1364 ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1365 ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting 1366 ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= 1367 ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage 1368 ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread 1369 rm -f mt_tmp* 1370 1371 println "\n===> ovLog tests " 1372 datagen -g2MB > tmp 1373 refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) 1374 ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) 1375 ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) 1376 if [ "$refSize" -eq "$ov9Size" ]; then 1377 echo ov9Size should be different from refSize 1378 exit 1 1379 fi 1380 if [ "$refSize" -eq "$ov1Size" ]; then 1381 echo ov1Size should be different from refSize 1382 exit 1 1383 fi 1384 if [ "$ov9Size" -ge "$ov1Size" ]; then 1385 echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" 1386 exit 1 1387 fi 1388 1389else 1390 println "\n===> no multithreading, skipping zstdmt tests " 1391fi 1392 1393rm -f tmp* 1394 1395println "\n===> zstd --list/-l single frame tests " 1396datagen > tmp1 1397datagen > tmp2 1398datagen > tmp3 1399zstd tmp* 1400zstd -l ./*.zst 1401zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header 1402zstd --list ./*.zst 1403zstd --list -v ./*.zst 1404 1405println "\n===> zstd --list/-l multiple frame tests " 1406cat tmp1.zst tmp2.zst > tmp12.zst 1407cat tmp12.zst tmp3.zst > tmp123.zst 1408zstd -l ./*.zst 1409zstd -lv ./*.zst 1410 1411println "\n===> zstd --list/-l error detection tests " 1412zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" 1413zstd --list tmp* && die "-l must fail on non-zstd file" 1414zstd -lv tmp1* && die "-l must fail on non-zstd file" 1415zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" 1416 1417println "test : detect truncated compressed file " 1418TEST_DATA_FILE=truncatable-input.txt 1419FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst 1420TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst 1421datagen -g50000 > $TEST_DATA_FILE 1422zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE 1423dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE 1424zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" 1425 1426rm -f $TEST_DATA_FILE 1427rm -f $FULL_COMPRESSED_FILE 1428rm -f $TRUNCATED_COMPRESSED_FILE 1429 1430println "\n===> zstd --list/-l errors when presented with stdin / no files" 1431zstd -l && die "-l must fail on empty list of files" 1432zstd -l - && die "-l does not work on stdin" 1433zstd -l < tmp1.zst && die "-l does not work on stdin" 1434zstd -l - < tmp1.zst && die "-l does not work on stdin" 1435zstd -l - tmp1.zst && die "-l does not work on stdin" 1436zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" 1437zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty 1438 1439println "\n===> zstd --list/-l test with null files " 1440datagen -g0 > tmp5 1441zstd tmp5 1442zstd -l tmp5.zst 1443zstd -l tmp5* && die "-l must fail on non-zstd file" 1444zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header 1445zstd -lv tmp5* && die "-l must fail on non-zstd file" 1446 1447println "\n===> zstd --list/-l test with no content size field " 1448datagen -g513K | zstd > tmp6.zst 1449zstd -l tmp6.zst 1450zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" 1451 1452println "\n===> zstd --list/-l test with no checksum " 1453zstd -f --no-check tmp1 1454zstd -l tmp1.zst 1455zstd -lv tmp1.zst 1456 1457println "\n===> zstd trace tests " 1458zstd -f --trace tmp.trace tmp1 1459zstd -f --trace tmp.trace tmp1 tmp2 tmp3 1460zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null 1461zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread 1462zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null 1463zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread 1464zstd --trace tmp.trace -t tmp1.zst 1465zstd --trace tmp.trace -t tmp1.zst tmp2.zst 1466zstd -f --trace tmp.trace -d tmp1.zst 1467zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst 1468zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 1469zstd -b1e10i0 --trace tmp.trace tmp1 1470zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 1471 1472rm -f tmp* 1473 1474 1475println "\n===> zstd long distance matching tests " 1476roundTripTest -g0 " --single-thread --long" 1477roundTripTest -g9M "2 --single-thread --long" 1478# Test parameter parsing 1479roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" 1480roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" 1481roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" 1482roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" 1483 1484 1485println "\n===> zstd long distance matching with optimal parser compressed size tests " 1486optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) 1487longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) 1488optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) 1489longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) 1490optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) 1491longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) 1492if [ "$longCSize16" -gt "$optCSize16" ]; then 1493 echo using --long on compression level 16 should not cause compressed size regression 1494 exit 1 1495elif [ "$longCSize19" -gt "$optCSize19" ]; then 1496 echo using --long on compression level 19 should not cause compressed size regression 1497 exit 1 1498elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then 1499 echo using --long on compression level 19 with wLog=23 should not cause compressed size regression 1500 exit 1 1501fi 1502 1503 1504if [ "$1" != "--test-large-data" ]; then 1505 println "Skipping large data tests" 1506 exit 0 1507fi 1508 1509 1510############################################################################# 1511 1512 1513if [ -n "$hasMT" ] 1514then 1515 println "\n===> adaptive mode " 1516 roundTripTest -g270000000 " --adapt" 1517 roundTripTest -g27000000 " --adapt=min=1,max=4" 1518 println "===> test: --adapt must fail on incoherent bounds " 1519 datagen > tmp 1520 zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" 1521 1522 println "\n===> rsyncable mode " 1523 roundTripTest -g10M " --rsyncable" 1524 roundTripTest -g10M " --rsyncable -B100K" 1525 println "===> test: --rsyncable must fail with --single-thread" 1526 zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" 1527fi 1528 1529println "\n===> patch-from=origin tests" 1530datagen -g1000 -P50 > tmp_dict 1531datagen -g1000 -P10 > tmp_patch 1532zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff 1533zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon 1534$DIFF -s tmp_patch_recon tmp_patch 1535 1536println "\n===> alternate syntax: patch-from origin" 1537zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff 1538zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon 1539$DIFF -s tmp_patch_recon tmp_patch 1540rm -rf tmp_* 1541 1542println "\n===> patch-from recursive tests" 1543mkdir tmp_dir 1544datagen > tmp_dir/tmp1 1545datagen > tmp_dir/tmp2 1546datagen > tmp_dict 1547zstd --patch-from=tmp_dict -r tmp_dir && die 1548rm -rf tmp* 1549 1550println "\n===> patch-from long mode trigger larger file test" 1551datagen -g5000000 > tmp_dict 1552datagen -g5000000 > tmp_patch 1553zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" 1554rm -rf tmp* 1555 1556println "\n===> patch-from very large dictionary and file test" 1557datagen -g550000000 -P0 > tmp_dict 1558datagen -g100000000 -P1 > tmp_patch 1559zstd --long=30 -1f --patch-from tmp_dict tmp_patch 1560zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon 1561$DIFF -s tmp_patch_recon tmp_patch 1562rm -rf tmp* 1563 1564println "\n===> patch-from --stream-size test" 1565datagen -g1000 -P50 > tmp_dict 1566datagen -g1000 -P10 > tmp_patch 1567cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die 1568cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff 1569rm -rf tmp* 1570 1571println "\n===> large files tests " 1572 1573roundTripTest -g270000000 1 1574roundTripTest -g250000000 2 1575roundTripTest -g230000000 3 1576 1577roundTripTest -g140000000 -P60 4 1578roundTripTest -g130000000 -P62 5 1579roundTripTest -g120000000 -P65 6 1580 1581roundTripTest -g70000000 -P70 7 1582roundTripTest -g60000000 -P71 8 1583roundTripTest -g50000000 -P73 9 1584 1585roundTripTest -g35000000 -P75 10 1586roundTripTest -g30000000 -P76 11 1587roundTripTest -g25000000 -P78 12 1588 1589roundTripTest -g18000013 -P80 13 1590roundTripTest -g18000014 -P80 14 1591roundTripTest -g18000015 -P81 15 1592roundTripTest -g18000016 -P84 16 1593roundTripTest -g18000017 -P88 17 1594roundTripTest -g18000018 -P94 18 1595roundTripTest -g18000019 -P96 19 1596 1597roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" 1598roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale 1599 1600fileRoundTripTest -g4193M -P99 1 1601 1602 1603println "\n===> zstd long, long distance matching round-trip tests " 1604roundTripTest -g270000000 "1 --single-thread --long" 1605roundTripTest -g130000000 -P60 "5 --single-thread --long" 1606roundTripTest -g35000000 -P70 "8 --single-thread --long" 1607roundTripTest -g18000001 -P80 "18 --single-thread --long" 1608# Test large window logs 1609roundTripTest -g700M -P50 "1 --single-thread --long=29" 1610roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" 1611 1612 1613if [ -n "$hasMT" ] 1614then 1615 println "\n===> zstdmt long round-trip tests " 1616 roundTripTest -g80000000 -P99 "19 -T2" " " 1617 roundTripTest -g5000000000 -P99 "1 -T2" " " 1618 roundTripTest -g500000000 -P97 "1 -T999" " " 1619 fileRoundTripTest -g4103M -P98 " -T0" " " 1620 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " 1621 # Exposes the bug in https://github.com/facebook/zstd/pull/1678 1622 # This test fails on 4 different travis builds at the time of writing 1623 # because it needs to allocate 8 GB of memory. 1624 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" 1625else 1626 println "\n**** no multithreading, skipping zstdmt tests **** " 1627fi 1628 1629 1630println "\n===> cover dictionary builder : advanced options " 1631 1632TESTFILE="$PRGDIR"/zstdcli.c 1633datagen > tmpDict 1634println "- Create first dictionary" 1635zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1636cp "$TESTFILE" tmp 1637zstd -f tmp -D tmpDict 1638zstd -d tmp.zst -D tmpDict -fo result 1639$DIFF "$TESTFILE" result 1640zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" 1641println "- Create second (different) dictionary" 1642zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1643zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1644println "- Create dictionary using shrink-dict flag" 1645zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict 1646zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 1647zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 1648println "- Create dictionary with short dictID" 1649zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1650cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1651println "- Create dictionary with size limit" 1652zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1653println "- Compare size of dictionary from 90% training samples with 80% training samples" 1654zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1655zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1656println "- Create dictionary using all samples for both training and testing" 1657zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1658println "- Test -o before --train-cover" 1659rm -f tmpDict dictionary 1660zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1661test -f tmpDict 1662zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1663test -f dictionary 1664rm -f tmp* dictionary 1665 1666rm -f tmp* 1667