1#! /bin/sh 2# 3# Copyright by The HDF Group. 4# Copyright by the Board of Trustees of the University of Illinois. 5# All rights reserved. 6# 7# This file is part of HDF5. The full HDF5 copyright notice, including 8# terms governing use, modification, and redistribution, is contained in 9# the COPYING file, which can be found at the root of the source code 10# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. 11# If you do not have access to either file, you may request a copy from 12# help@hdfgroup.org. 13# 14# Tests for the h5repack tool 15# 16# Modification: 17# Pedro Vicente Nunes, 11/15/2006 18# Added $FILEN variables for file names 19# 20 21srcdir=@srcdir@ 22 23USE_FILTER_SZIP="@USE_FILTER_SZIP@" 24USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" 25 26TESTNAME=h5repack 27EXIT_SUCCESS=0 28EXIT_FAILURE=1 29 30H5REPACK=h5repack # The tool name 31H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary 32 33H5DIFF=../h5diff/h5diff # The h5diff tool name 34H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary 35 36H5DUMP=../h5dump/h5dump # The h5dump tool name 37H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary 38 39RM='rm -rf' 40CMP='cmp' 41DIFF='diff -c' 42GREP='grep' 43CP='cp' 44DIRNAME='dirname' 45LS='ls' 46AWK='awk' 47 48H5DETECTSZIP=testh5repack_detect_szip 49H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP 50 51 52nerrors=0 53verbose=yes 54 55# source dirs 56SRC_TOOLS="$srcdir/.." 57SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" 58 59# testfiles source dirs for tools 60SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" 61SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" 62SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles" 63SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles" 64SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles" 65SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles" 66SRC_H5STAT_TESTFILES="$SRC_TOOLS/h5stat/testfiles" 67SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles" 68 69TESTDIR=./testpack 70test -d $TESTDIR || mkdir $TESTDIR 71 72###################################################################### 73# test files 74# -------------------------------------------------------------------- 75# All the test files copy from source directory to test directory 76# NOTE: Keep this framework to add/remove test files. 77# Any test files from other tools can be used in this framework. 78# This list are also used for checking exist. 79# Comment '#' without space can be used. 80# -------------------------------------------------------------------- 81LIST_HDF5_TEST_FILES=" 82$SRC_H5REPACK_TESTFILES/h5repack_attr.h5 83$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5 84$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5 85$SRC_H5REPACK_TESTFILES/h5repack_early.h5 86$SRC_H5REPACK_TESTFILES/h5repack_ext.h5 87$SRC_H5REPACK_TESTFILES/h5repack_fill.h5 88$SRC_H5REPACK_TESTFILES/h5repack_filters.h5 89$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5 90$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5 91$SRC_H5REPACK_TESTFILES/h5repack_layout.h5 92$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5 93$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5 94$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5 95$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5 96$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5 97$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5 98$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5 99$SRC_H5REPACK_TESTFILES/h5repack_objs.h5 100$SRC_H5REPACK_TESTFILES/h5repack_refs.h5 101$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 102$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 103$SRC_H5REPACK_TESTFILES/h5repack_szip.h5 104$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 105$SRC_TOOLS_TESTFILES/tfamily00000.h5 106$SRC_TOOLS_TESTFILES/tfamily00001.h5 107$SRC_TOOLS_TESTFILES/tfamily00002.h5 108$SRC_TOOLS_TESTFILES/tfamily00003.h5 109$SRC_TOOLS_TESTFILES/tfamily00004.h5 110$SRC_TOOLS_TESTFILES/tfamily00005.h5 111$SRC_TOOLS_TESTFILES/tfamily00006.h5 112$SRC_TOOLS_TESTFILES/tfamily00007.h5 113$SRC_TOOLS_TESTFILES/tfamily00008.h5 114$SRC_TOOLS_TESTFILES/tfamily00009.h5 115$SRC_TOOLS_TESTFILES/tfamily00010.h5 116$SRC_TOOLS_TESTFILES/tordergr.h5 117" 118 119LIST_OTHER_TEST_FILES=" 120$SRC_H5REPACK_TESTFILES/h5repack-help.txt 121$SRC_H5REPACK_TESTFILES/h5repack_ext.bin 122$SRC_H5REPACK_TESTFILES/ublock.bin 123$SRC_H5REPACK_TESTFILES/h5repack.info 124$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl 125$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl 126$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl 127$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst 128$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl 129$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst 130" 131 132# 133# copy test files and expected output files from source dirs to test dir 134# 135COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES" 136 137COPY_TESTFILES_TO_TESTDIR() 138{ 139 # copy test files. Used -f to make sure get a new copy 140 for tstfile in $COPY_TESTFILES 141 do 142 # ignore '#' comment 143 echo $tstfile | tr -d ' ' | grep '^#' > /dev/null 144 RET=$? 145 if [ $RET -eq 1 ]; then 146 # skip cp if srcdir is same as destdir 147 # this occurs when build/test performed in source dir and 148 # make cp fail 149 SDIR=`$DIRNAME $tstfile` 150 INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` 151 INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` 152 if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then 153 $CP -f $tstfile $TESTDIR 154 if [ $? -ne 0 ]; then 155 echo "Error: FAILED to copy $tstfile ." 156 157 # Comment out this to CREATE expected file 158 exit $EXIT_FAILURE 159 fi 160 fi 161 fi 162 done 163} 164 165CLEAN_TESTFILES_AND_TESTDIR() 166{ 167 # skip rm if srcdir is same as destdir 168 # this occurs when build/test performed in source dir and 169 # make cp fail 170 SDIR=$SRC_H5REPACK_TESTFILES 171 INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` 172 INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` 173 if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then 174 $RM $TESTDIR 175 fi 176} 177 178# Print a $* message left justified in a field of 70 characters 179# 180MESSAGE() { 181 SPACES=" " 182 echo "$* $SPACES" | cut -c1-70 | tr -d '\012' 183} 184 185# Print a line-line message left justified in a field of 70 characters 186# beginning with the word "Testing". 187# 188TESTING() { 189 MESSAGE "Testing $*" 190} 191 192# Print a line-line message left justified in a field of 70 characters 193# beginning with the word "Verifying". 194# 195VERIFY() { 196 MESSAGE "Verifying $*" 197} 198 199# Print a message that a test has been skipped (because a required filter 200# was unavailable) 201SKIP() { 202 TESTING $H5REPACK $@ 203 echo " -SKIP-" 204} 205 206# Call the h5diff tool 207# 208DIFFTEST() 209{ 210 VERIFY h5diff output $@ 211 ( 212 cd $TESTDIR 213 $RUNSERIAL $H5DIFF_BIN -q "$@" 214 ) 215 RET=$? 216 if [ $RET != 0 ] ; then 217 echo "*FAILED*" 218 nerrors="`expr $nerrors + 1`" 219 else 220 echo " PASSED" 221 fi 222 223} 224 225# Call h5repack 226# 227 228 229# call TOOLTEST_MAIN and delete $output file 230TOOLTEST() 231{ 232 echo $@ 233 infile=$2 234 outfile=out-$1.$2 235 shift 236 shift 237 238 # Run test. 239 TESTING $H5REPACK $@ 240 ( 241 cd $TESTDIR 242 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 243 ) 244 RET=$? 245 if [ $RET != 0 ] ; then 246 echo "*FAILED*" 247 nerrors="`expr $nerrors + 1`" 248 else 249 echo " PASSED" 250 DIFFTEST $infile $outfile 251 fi 252 rm -f $outfile 253} 254 255#------------------------------------------ 256# Verifying layouts of a dataset 257VERIFY_LAYOUT_DSET() 258{ 259 layoutfile=layout-$1.$2 260 dset=$3 261 expectlayout=$4 262 infile=$2 263 outfile=out-$1.$2 264 shift 265 shift 266 shift 267 shift 268 269 TESTING $H5REPACK $@ 270 ( 271 cd $TESTDIR 272 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 273 ) 274 RET=$? 275 if [ $RET != 0 ] ; then 276 echo "*FAILED*" 277 nerrors="`expr $nerrors + 1`" 278 else 279 echo " PASSED" 280 DIFFTEST $infile $outfile 281 fi 282 283 #--------------------------------- 284 # check the layout from a dataset 285 VERIFY "a dataset layout" 286 ( 287 cd $TESTDIR 288 $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile 289 ) 290 $GREP $expectlayout $TESTDIR/$layoutfile > /dev/null 291 if [ $? -eq 0 ]; then 292 echo " PASSED" 293 else 294 echo " FAILED" 295 nerrors="`expr $nerrors + 1`" 296 fi 297 298 # clean up tmp files 299 rm -f $outfile 300 rm -f $layoutfile 301} 302 303#---------------------------------------- 304# Verifying layouts from entire file 305VERIFY_LAYOUT_ALL() 306{ 307 infile=$2 308 outfile=out-$1.$2 309 layoutfile=layout-$1.$2 310 expectlayout=$3 311 shift 312 shift 313 shift 314 315 TESTING $H5REPACK $@ 316 ( 317 cd $TESTDIR 318 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 319 ) 320 RET=$? 321 if [ $RET != 0 ] ; then 322 echo "*FAILED*" 323 nerrors="`expr $nerrors + 1`" 324 else 325 echo " PASSED" 326 DIFFTEST $infile $outfile 327 fi 328 329 330 #--------------------------------- 331 # check the layout from a dataset 332 # check if the other layouts still exsit 333 VERIFY "layouts" 334 ( 335 cd $TESTDIR 336 echo 337 # if CONTIGUOUS 338 if [ $expectlayout = "CONTIGUOUS" ]; then 339 TESTING $H5DUMP_BIN -pH $outfile 340 ( 341 $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile 342 ) 343 $GREP "COMPACT" $layoutfile > /dev/null 344 if [ $? -eq 0 ]; then 345 echo " FAILED" 346 nerrors="`expr $nerrors + 1`" 347 else 348 $GREP "CHUNKED" $layoutfile > /dev/null 349 if [ $? -eq 0 ]; then 350 echo " FAILED" 351 nerrors="`expr $nerrors + 1`" 352 else 353 echo " PASSED" 354 fi 355 fi 356 else 357 # if COMPACT 358 if [ $expectlayout = "COMPACT" ]; then 359 TESTING $H5DUMP_BIN -pH $outfile 360 ( 361 $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile 362 ) 363 $GREP "CHUNKED" $layoutfile > /dev/null 364 if [ $? -eq 0 ]; then 365 echo " FAILED" 366 nerrors="`expr $nerrors + 1`" 367 else 368 $GREP "CONTIGUOUS" $layoutfile > /dev/null 369 if [ $? -eq 0 ]; then 370 echo " FAILED" 371 nerrors="`expr $nerrors + 1`" 372 else 373 echo " PASSED" 374 fi 375 fi 376 else 377 # if CHUNKED 378 if [ $expectlayout = "CHUNKED" ]; then 379 TESTING $H5DUMP_BIN -pH $outfile 380 ( 381 $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile 382 ) 383 $GREP "CONTIGUOUS" $layoutfile > /dev/null 384 if [ $? -eq 0 ]; then 385 echo " FAILED" 386 nerrors="`expr $nerrors + 1`" 387 else 388 $GREP "COMPACT" $layoutfile > /dev/null 389 if [ $? -eq 0 ]; then 390 echo " FAILED" 391 nerrors="`expr $nerrors + 1`" 392 else 393 echo " PASSED" 394 fi 395 fi 396 fi 397 fi 398 fi 399 ) 400 401 # clean up tmp files 402 rm -f $outfile 403 rm -f $layoutfile 404} 405 406# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file 407# 408TOOLTEST0() 409{ 410 infile=$2 411 outfile=out-$1.$2 412 shift 413 shift 414 415 # Run test. 416 TESTING $H5REPACK $@ 417 ( 418 cd $TESTDIR 419 $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@" 420 ) 421 RET=$? 422 if [ $RET != 0 ] ; then 423 echo "*FAILED*" 424 nerrors="`expr $nerrors + 1`" 425 else 426 echo " PASSED" 427 DIFFTEST $infile $outfile 428 fi 429 rm -f $outfile 430} 431 432 433# same as TOOLTEST, but it uses without -i -o options 434# used to test the family driver, where these files reside 435# 436TOOLTEST1() 437{ 438 infile=$2 439 outfile=out-$1.$2 440 shift 441 shift 442 443 # Run test. 444 TESTING $H5REPACK $@ 445 ( 446 cd $TESTDIR 447 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 448 ) 449 RET=$? 450 if [ $RET != 0 ] ; then 451 echo "*FAILED*" 452 nerrors="`expr $nerrors + 1`" 453 else 454 echo " PASSED" 455 DIFFTEST $infile $outfile 456 fi 457 rm -f $outfile 458} 459 460# This is same as TOOLTEST() with comparing display output 461# from -v option 462# 463TOOLTESTV() 464{ 465 expect="$TESTDIR/$2-$1.tst" 466 actual="$TESTDIR/`basename $2 .ddl`.out" 467 actual_err="$TESTDIR/`basename $2 .ddl`.err" 468 469 infile=$2 470 outfile=out-$1.$2 471 shift 472 shift 473 474 # Run test. 475 TESTING $H5REPACK $@ 476 ( 477 cd $TESTDIR 478 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 479 ) >$actual 2>$actual_err 480 RET=$? 481 if [ $RET != 0 ] ; then 482 echo "*FAILED*" 483 nerrors="`expr $nerrors + 1`" 484 else 485 echo " PASSED" 486 DIFFTEST $infile $outfile 487 fi 488 489 # display output compare 490 STDOUT_FILTER $actual 491 cat $actual_err >> $actual 492 493 VERIFY output from $H5REPACK $@ 494 if cmp -s $expect $actual; then 495 echo " PASSED" 496 else 497 echo "*FAILED*" 498 echo " Expected result (*.tst) differs from actual result (*.out)" 499 nerrors="`expr $nerrors + 1`" 500 test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' 501 fi 502 503 rm -f $actual $actual_err 504 rm -f $outfile 505} 506 507# This is same as TOOLTESTV() with comparing h5dump output 508# from -pH option 509# 510TOOLTEST_DUMP() 511{ 512 infile=$2 513 outfile=out-$1.$2 514 expect="$TESTDIR/$1.$2.ddl" 515 actual="$TESTDIR/out-$1.$2.out" 516 actual_err="$TESTDIR/out-$1.$2.err" 517 518 shift 519 shift 520 521 # Run test. 522 TESTING $H5REPACK $@ 523 ( 524 cd $TESTDIR 525 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 526 ) >$actual 2>$actual_err 527 RET=$? 528 if [ $RET != 0 ] ; then 529 echo "*FAILED*" 530 nerrors="`expr $nerrors + 1`" 531 else 532 echo " PASSED" 533 VERIFY h5dump output $@ 534 ( 535 cd $TESTDIR 536 $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile 537 ) >$actual 2>$actual_err 538 cat $actual_err >> $actual 539 540 RET=$? 541 542 fi 543 544 if cmp -s $expect $actual; then 545 echo " PASSED" 546 else 547 echo "*FAILED*" 548 echo " Expected result (*.ddl) differs from actual result (*.out)" 549 nerrors="`expr $nerrors + 1`" 550 test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' 551 fi 552 553 rm -f $actual $actual_err 554 rm -f $outfile 555} 556 557# TOOLTEST_META: 558# Test metadata block size option. 559# Reason to create a function here is to localize all special steps related to 560# metadata block size option in one place. This is a quick solution. More 561# thought out solution needed when there is more time. 562# 563# $1: test input file 564# $2:$: metadata options (either "-M size" or "--metadata_block_size=size") 565# 566# Algorithm: 567# Run it once without the metadata option ($2-$); 568# Save the result output file; 569# Run it second time with the metadata option; 570# Verify the output file of second run is larger than the one of 1st run. 571TOOLTEST_META() 572{ 573 infile=$2 574 outfile=out-$1.$2 575 shift 576 shift 577 578 # Run test. 579 TESTING $H5REPACK $@ 580 ( 581 cd $TESTDIR 582 $RUNSERIAL $H5REPACK_BIN $infile $outfile 583 ) 584 RET=$? 585 if [ $RET != 0 ] ; then 586 echo "*FAILED*" 587 nerrors="`expr $nerrors + 1`" 588 else 589 echo " PASSED" 590 DIFFTEST $infile $outfile 591 fi 592 # get the size of the first output file 593 size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` 594 595 # 2nd run with metadata option 596 # Run test. 597 TESTING $H5REPACK $@ 598 ( 599 cd $TESTDIR 600 $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile 601 ) 602 RET=$? 603 if [ $RET != 0 ] ; then 604 echo "*FAILED*" 605 nerrors="`expr $nerrors + 1`" 606 else 607 echo " PASSED" 608 DIFFTEST $infile $outfile 609 fi 610 # get the size of the second output file 611 size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1` 612 613 # verify sizes. 614 MESSAGE "Verify the sizes of both output files ($size1 vs $size2)" 615 if [ $size1 -lt $size2 ]; then 616 # pass 617 echo " PASSED" 618 else 619 #fail 620 echo "*FAILED*" 621 nerrors="`expr $nerrors + 1`" 622 fi 623 624 rm -f $outfile 625} 626 627# ADD_HELP_TEST 628TOOLTEST_HELP() { 629 630 expect="$TESTDIR/$1" 631 actual="$TESTDIR/`basename $1 .txt`.out" 632 actual_err="$TESTDIR/`basename $1 .txt`.err" 633 shift 634 635 # Run test. 636 TESTING $H5REPACK $@ 637 ( 638 cd $TESTDIR 639 $RUNSERIAL $H5REPACK_BIN "$@" 640 ) >$actual 2>$actual_err 641 cat $actual_err >> $actual 642 643 if [ ! -f $expect ]; then 644 # Create the expect data file if it doesn't yet exist. 645 echo " CREATED" 646 cp $actual $expect-CREATED 647 echo " Expected result (*.txt) missing" 648 nerrors="`expr $nerrors + 1`" 649 elif cmp -s $expect $actual; then 650 echo " PASSED" 651 else 652 echo "*FAILED*" 653 echo " Expected output (*.txt) differs from actual output (*.out)" 654 nerrors="`expr $nerrors + 1`" 655 fi 656 657 # Clean up output file 658 if test -z "$HDF5_NOCLEANUP"; then 659 rm -f $actual $actual_err 660 fi 661 662} 663 664# This is different from $srcdir/../../bin/output_filter.sh 665STDOUT_FILTER() { 666 result_file=$1 667 tmp_file=/tmp/h5test_tmp_$$ 668 # Filter name of files. 669 cp $result_file $tmp_file 670 sed -e '/^Opening file/d' -e '/^Making file/d' \ 671 < $tmp_file > $result_file 672 # cleanup 673 rm -f $tmp_file 674} 675 676# 677# The tests 678# We use the files generated by h5repacktst 679# Each run generates "<file>.out.h5" and the tool h5diff is used to 680# compare the input and output files 681# 682# the tests are the same as the program h5repacktst, but run from the CLI 683# 684 685# See which filters are usable (and skip tests for filters we 686# don't have). Do this by searching H5pubconf.h to see which 687# filters are defined. 688 689# detect whether the encoder is present. 690USE_FILTER_SZIP_ENCODER="no"; 691if test $USE_FILTER_SZIP = "yes"; then 692USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN` 693fi 694 695############################################################################## 696### T H E T E S T S 697############################################################################## 698# prepare for test 699COPY_TESTFILES_TO_TESTDIR 700 701TOOLTEST_HELP h5repack-help.txt -h 702 703# copy files (these files have no filters) 704TOOLTEST fill h5repack_fill.h5 705TOOLTEST objs h5repack_objs.h5 706TOOLTEST attr h5repack_attr.h5 707TOOLTEST hlink h5repack_hlink.h5 708TOOLTEST layout h5repack_layout.h5 709TOOLTEST early h5repack_early.h5 710 711# nested 8bit enum in both deflated and non-deflated datafiles 712if [ $USE_FILTER_DEFLATE != "yes" ]; then 713TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5 714else 715TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5 716fi 717 718# use h5repack_layout.h5 to write some filters (this file has no filters) 719 720# gzip with individual object 721arg="h5repack_layout.h5 -f dset1:GZIP=1 -l dset1:CHUNK=20x10" 722if test $USE_FILTER_DEFLATE != "yes" ; then 723 SKIP $arg 724else 725 TOOLTEST gzip_individual $arg 726fi 727 728# gzip for all 729arg="h5repack_layout.h5 -f GZIP=1" 730if test $USE_FILTER_DEFLATE != "yes" ; then 731 SKIP $arg 732else 733 TOOLTEST gzip_all $arg 734fi 735 736# szip with individual object 737arg="h5repack_layout.h5 -f dset2:SZIP=8,EC -l dset2:CHUNK=20x10" 738if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then 739 SKIP $arg 740else 741 TOOLTEST szip_individual $arg 742fi 743 744# szip for all 745arg="h5repack_layout.h5 -f SZIP=8,NN" 746if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then 747 SKIP $arg 748else 749 TOOLTEST szip_all $arg 750fi 751 752# shuffle with individual object 753arg="h5repack_layout.h5 -f dset2:SHUF -l dset2:CHUNK=20x10" 754TOOLTEST shuffle_individual $arg 755 756 757# shuffle for all 758arg="h5repack_layout.h5 -f SHUF" 759TOOLTEST shuffle_all $arg 760 761# fletcher32 with individual object 762arg="h5repack_layout.h5 -f dset2:FLET -l dset2:CHUNK=20x10" 763TOOLTEST fletcher_individual $arg 764 765# fletcher32 for all 766arg="h5repack_layout.h5 -f FLET" 767TOOLTEST fletcher_all $arg 768 769# all filters 770arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10" 771if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then 772 SKIP $arg 773else 774 TOOLTEST all_filters $arg 775fi 776 777# verbose gzip with individual object 778arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9" 779if test $USE_FILTER_DEFLATE != "yes" ; then 780 SKIP $arg 781else 782 # compare output 783 TOOLTESTV gzip_verbose_filters $arg 784fi 785 786########################################################### 787# the following tests assume the input files have filters 788########################################################### 789 790# szip copy 791arg="h5repack_szip.h5" 792if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then 793 SKIP $arg 794else 795 TOOLTEST szip_copy $arg 796fi 797 798# szip remove 799arg="h5repack_szip.h5 --filter=dset_szip:NONE" 800if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then 801 SKIP $arg 802else 803 TOOLTEST szip_remove $arg 804fi 805 806# deflate copy 807arg="h5repack_deflate.h5" 808if test $USE_FILTER_DEFLATE != "yes" ; then 809 SKIP $arg 810else 811 TOOLTEST deflate_copy $arg 812fi 813 814# deflate remove 815arg="h5repack_deflate.h5 -f dset_deflate:NONE" 816if test $USE_FILTER_DEFLATE != "yes" ; then 817 SKIP $arg 818else 819 TOOLTEST deflate_remove $arg 820fi 821 822# shuffle copy 823arg="h5repack_shuffle.h5" 824TOOLTEST shuffle_copy $arg 825 826# shuffle remove 827arg="h5repack_shuffle.h5 -f dset_shuffle:NONE" 828TOOLTEST shuffle_remove $arg 829 830# fletcher32 copy 831arg="h5repack_fletcher.h5" 832TOOLTEST fletcher_copy $arg 833 834# fletcher32 remove 835arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE" 836TOOLTEST fletcher_remove $arg 837 838# nbit copy 839arg="h5repack_nbit.h5" 840TOOLTEST nbit_copy $arg 841 842# nbit remove 843arg="h5repack_nbit.h5 -f dset_nbit:NONE" 844TOOLTEST nbit_remove $arg 845 846# nbit add 847arg="h5repack_nbit.h5 -f dset_int31:NBIT" 848TOOLTEST nbit_add $arg 849 850# scaleoffset copy 851arg="h5repack_soffset.h5" 852TOOLTEST scale_copy $arg 853 854# scaleoffset add 855arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN" 856TOOLTEST scale_add $arg 857 858# scaleoffset remove 859arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE" 860TOOLTEST scale_remove $arg 861 862# remove all filters 863arg="h5repack_filters.h5 -f NONE" 864if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then 865 SKIP $arg 866else 867 TOOLTEST remove_all $arg 868fi 869 870#filter conversions 871 872arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN" 873if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then 874 SKIP $arg 875else 876 TOOLTEST deflate_convert $arg 877fi 878 879arg="h5repack_szip.h5 -f dset_szip:GZIP=1" 880if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then 881 SKIP $arg 882else 883 TOOLTEST szip_convert $arg 884fi 885 886 887#limit 888arg="h5repack_layout.h5 -f GZIP=1 -m 1024" 889if test $USE_FILTER_DEFLATE != "yes" ; then 890 SKIP $arg 891else 892 TOOLTEST_DUMP deflate_limit $arg 893fi 894 895#file 896arg="h5repack_layout.h5 -e h5repack.info" 897if test $USE_FILTER_DEFLATE != "yes" ; then 898 SKIP $arg 899else 900 TOOLTEST deflate_file $arg 901fi 902 903#crtorder 904arg="tordergr.h5 -L" 905TOOLTEST_DUMP crtorder $arg 906 907######################################################### 908# layout options (these files have no filters) 909######################################################### 910VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10 911 912VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10 913 914VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI 915 916VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI 917 918VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA 919 920VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA 921 922################################################################ 923# layout conversions (file has no filters) 924############################################################### 925 926VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI 927 928VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5 929 930VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA 931 932VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA 933 934VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6 935 936VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI 937 938VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA 939 940VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI 941 942VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13 943 944# test convert small size dataset ( < 1k) to compact layout without -m 945VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA 946 947VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA 948 949#--------------------------------------------------------------------------- 950# Test file contains chunked datasets (need multiple dsets) with 951# unlimited max dims. (HDFFV-7933) 952# Use first dset to test. 953#--------------------------------------------------------------------------- 954# chunk to chunk - specify chunk dim bigger than any current dim 955VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300 956 957# chunk to contiguous 958VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI 959 960# chunk to compact - convert big dataset (should be > 64k) for this purpose, 961# should remain as original layout (chunk) 962VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA 963 964#-------------------------------------------------------------------------- 965# Test -f for some specific cases. Chunked dataset with unlimited max dims. 966# (HDFFV-8012) 967#-------------------------------------------------------------------------- 968# - should not fail 969# - should not change max dims from unlimit 970 971# chunk dim is bigger than dataset dim. ( dset size < 64k ) 972VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE 973# chunk dim is bigger than dataset dim. ( dset size > 64k ) 974VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE 975 976# chunk dims are smaller than dataset dims. ( dset size < 64k ) 977#TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE 978VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE 979 980# file input - should not fail 981TOOLTEST error4 h5repack_layout3.h5 -f NONE 982 983#-------------------------------------------------------------------------- 984# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset 985# (dset size < 64K) and with unlimited max dims on a condition as follow. 986# (HDFFV-8214) 987#-------------------------------------------------------------------------- 988 989# chunk dim is bigger than dataset dim. should succeed. 990VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI 991# chunk dim is smaller than dataset dim. should succeed. 992VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI 993 994 995# Native option 996# Do not use FILE1, as the named dtype will be converted to native, and h5diff will 997# report a difference. 998TOOLTEST native_fill h5repack_fill.h5 -n 999TOOLTEST native_attr h5repack_attr.h5 -n 1000 1001 1002# latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters) 1003arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]" 1004if test $USE_FILTER_DEFLATE != "yes" ; then 1005 SKIP $arg 1006else 1007 VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype] 1008fi 1009 1010# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) 1011arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" 1012if test $USE_FILTER_DEFLATE != "yes" ; then 1013 SKIP $arg 1014else 1015 VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype] 1016fi 1017 1018# several global filters 1019 1020arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF" 1021if test $USE_FILTER_DEFLATE != "yes" ; then 1022 SKIP $arg 1023else 1024 TOOLTEST global_filters $arg 1025fi 1026 1027# syntax of -i infile -o outfile 1028# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) 1029arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" 1030if test $USE_FILTER_DEFLATE != "yes" ; then 1031 SKIP $arg 1032else 1033 TOOLTEST0 old_style_layout_short_switches $arg 1034fi 1035 1036# add a userblock to file 1037arg="h5repack_objs.h5 -u ublock.bin -b 2048" 1038TOOLTEST add_userblock $arg 1039 1040# add alignment 1041arg="h5repack_objs.h5 -t 1 -a 1 " 1042TOOLTEST add_alignment $arg 1043 1044# Check repacking file with old version of layout message (should get upgraded 1045# to new version and be readable, etc.) 1046TOOLTEST upgrade_layout h5repack_layouto.h5 1047 1048# test for datum size > H5TOOLS_MALLOCSIZE 1049TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1 1050 1051# Check repacking file with committed datatypes in odd configurations 1052TOOLTEST committed_dt h5repack_named_dtypes.h5 1053 1054# tests family driver (file is located in common testfiles folder, uses TOOLTEST1 1055TOOLTEST1 family tfamily%05d.h5 1056 1057# test various references (bug 1814 and 1726) 1058TOOLTEST bug1814 h5repack_refs.h5 1059 1060# test attribute with various references (bug1797 / HDFFV-5932) 1061# the references in attribute of compund or vlen datatype 1062TOOLTEST HDFFV-5932 h5repack_attr_refs.h5 1063 1064# Add test for memory leak in attirbute. This test is verified by CTEST. 1065# 1. leak from vlen string 1066# 2. leak from compound type without reference member 1067# (HDFFV-7840, ) 1068# Note: this test is experimental for sharing test file among tools 1069TOOLTEST HDFFV-7840 h5diff_attr1.h5 1070 1071# tests for metadata block size option 1072TOOLTEST_META meta_short h5repack_layout.h5 -M 8192 1073TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192 1074 1075# Clean up temporary files/directories 1076CLEAN_TESTFILES_AND_TESTDIR 1077 1078if test $nerrors -eq 0 ; then 1079 echo "All $TESTNAME tests passed." 1080 exit $EXIT_SUCCESS 1081else 1082 echo "$TESTNAME tests failed with $nerrors errors." 1083 exit $EXIT_FAILURE 1084fi 1085 1086