1#! /bin/sh
2#
3# Copyright by The HDF Group.
4# Copyright by the Board of Trustees of the University of Illinois.
5# All rights reserved.
6#
7# This file is part of HDF5.  The full HDF5 copyright notice, including
8# terms governing use, modification, and redistribution, is contained in
9# the COPYING file, which can be found at the root of the source code
10# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
11# If you do not have access to either file, you may request a copy from
12# help@hdfgroup.org.
13#
14# Tests for the h5repack tool
15#
16# Modification:
17#  Pedro Vicente Nunes, 11/15/2006
18#  Added $FILEN variables for file names
19#
20
21srcdir=@srcdir@
22
23USE_FILTER_SZIP="@USE_FILTER_SZIP@"
24USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
25
26TESTNAME=h5repack
27EXIT_SUCCESS=0
28EXIT_FAILURE=1
29
30H5REPACK=h5repack               # The tool name
31H5REPACK_BIN=`pwd`/$H5REPACK    # The path of the tool binary
32
33H5DIFF=../h5diff/h5diff         # The h5diff tool name
34H5DIFF_BIN=`pwd`/$H5DIFF        # The path of the h5diff tool binary
35
36H5DUMP=../h5dump/h5dump         # The h5dump tool name
37H5DUMP_BIN=`pwd`/$H5DUMP        # The path of the h5dump tool binary
38
39RM='rm -rf'
40CMP='cmp'
41DIFF='diff -c'
42GREP='grep'
43CP='cp'
44DIRNAME='dirname'
45LS='ls'
46AWK='awk'
47
48H5DETECTSZIP=testh5repack_detect_szip
49H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP
50
51
52nerrors=0
53verbose=yes
54
55# source dirs
56SRC_TOOLS="$srcdir/.."
57SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"
58
59# testfiles source dirs for tools
60SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
61SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
62SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles"
63SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles"
64SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles"
65SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles"
66SRC_H5STAT_TESTFILES="$SRC_TOOLS/h5stat/testfiles"
67SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles"
68
69TESTDIR=./testpack
70test -d $TESTDIR || mkdir $TESTDIR
71
72######################################################################
73# test files
74# --------------------------------------------------------------------
75# All the test files copy from source directory to test directory
76# NOTE: Keep this framework to add/remove test files.
77#       Any test files from other tools can be used in this framework.
78#       This list are also used for checking exist.
79#       Comment '#' without space can be used.
80# --------------------------------------------------------------------
81LIST_HDF5_TEST_FILES="
82$SRC_H5REPACK_TESTFILES/h5repack_attr.h5
83$SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5
84$SRC_H5REPACK_TESTFILES/h5repack_deflate.h5
85$SRC_H5REPACK_TESTFILES/h5repack_early.h5
86$SRC_H5REPACK_TESTFILES/h5repack_ext.h5
87$SRC_H5REPACK_TESTFILES/h5repack_fill.h5
88$SRC_H5REPACK_TESTFILES/h5repack_filters.h5
89$SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5
90$SRC_H5REPACK_TESTFILES/h5repack_hlink.h5
91$SRC_H5REPACK_TESTFILES/h5repack_layout.h5
92$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5
93$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5
94$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5
95$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5
96$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum.h5
97$SRC_H5REPACK_TESTFILES/h5repack_nested_8bit_enum_deflated.h5
98$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5
99$SRC_H5REPACK_TESTFILES/h5repack_objs.h5
100$SRC_H5REPACK_TESTFILES/h5repack_refs.h5
101$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
102$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
103$SRC_H5REPACK_TESTFILES/h5repack_szip.h5
104$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
105$SRC_TOOLS_TESTFILES/tfamily00000.h5
106$SRC_TOOLS_TESTFILES/tfamily00001.h5
107$SRC_TOOLS_TESTFILES/tfamily00002.h5
108$SRC_TOOLS_TESTFILES/tfamily00003.h5
109$SRC_TOOLS_TESTFILES/tfamily00004.h5
110$SRC_TOOLS_TESTFILES/tfamily00005.h5
111$SRC_TOOLS_TESTFILES/tfamily00006.h5
112$SRC_TOOLS_TESTFILES/tfamily00007.h5
113$SRC_TOOLS_TESTFILES/tfamily00008.h5
114$SRC_TOOLS_TESTFILES/tfamily00009.h5
115$SRC_TOOLS_TESTFILES/tfamily00010.h5
116$SRC_TOOLS_TESTFILES/tordergr.h5
117"
118
119LIST_OTHER_TEST_FILES="
120$SRC_H5REPACK_TESTFILES/h5repack-help.txt
121$SRC_H5REPACK_TESTFILES/h5repack_ext.bin
122$SRC_H5REPACK_TESTFILES/ublock.bin
123$SRC_H5REPACK_TESTFILES/h5repack.info
124$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
125$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
126$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
127$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
128$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
129$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
130"
131
132#
133# copy test files and expected output files from source dirs to test dir
134#
135COPY_TESTFILES="$LIST_HDF5_TEST_FILES $LIST_OTHER_TEST_FILES"
136
137COPY_TESTFILES_TO_TESTDIR()
138{
139    # copy test files. Used -f to make sure get a new copy
140    for tstfile in $COPY_TESTFILES
141    do
142        # ignore '#' comment
143        echo $tstfile | tr -d ' ' | grep '^#' > /dev/null
144        RET=$?
145        if [ $RET -eq 1 ]; then
146            # skip cp if srcdir is same as destdir
147            # this occurs when build/test performed in source dir and
148            # make cp fail
149            SDIR=`$DIRNAME $tstfile`
150            INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
151            INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
152            if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
153                $CP -f $tstfile $TESTDIR
154                if [ $? -ne 0 ]; then
155                    echo "Error: FAILED to copy $tstfile ."
156
157                    # Comment out this to CREATE expected file
158                    exit $EXIT_FAILURE
159                fi
160            fi
161        fi
162    done
163}
164
165CLEAN_TESTFILES_AND_TESTDIR()
166{
167    # skip rm if srcdir is same as destdir
168    # this occurs when build/test performed in source dir and
169    # make cp fail
170    SDIR=$SRC_H5REPACK_TESTFILES
171    INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'`
172    INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'`
173    if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then
174        $RM $TESTDIR
175    fi
176}
177
178# Print a $* message left justified in a field of 70 characters
179#
180MESSAGE() {
181   SPACES="                                                               "
182   echo "$* $SPACES" | cut -c1-70 | tr -d '\012'
183}
184
185# Print a line-line message left justified in a field of 70 characters
186# beginning with the word "Testing".
187#
188TESTING() {
189    MESSAGE "Testing $*"
190}
191
192# Print a line-line message left justified in a field of 70 characters
193# beginning with the word "Verifying".
194#
195VERIFY() {
196    MESSAGE "Verifying $*"
197}
198
199# Print a message that a test has been skipped (because a required filter
200# was unavailable)
201SKIP() {
202    TESTING $H5REPACK $@
203    echo " -SKIP-"
204}
205
206# Call the h5diff tool
207#
208DIFFTEST()
209{
210    VERIFY h5diff output $@
211    (
212        cd $TESTDIR
213        $RUNSERIAL $H5DIFF_BIN -q  "$@"
214    )
215    RET=$?
216    if [ $RET != 0 ] ; then
217         echo "*FAILED*"
218         nerrors="`expr $nerrors + 1`"
219    else
220         echo " PASSED"
221    fi
222
223}
224
225# Call h5repack
226#
227
228
229# call TOOLTEST_MAIN and delete $output file
230TOOLTEST()
231{
232    echo $@
233    infile=$2
234    outfile=out-$1.$2
235    shift
236    shift
237
238    # Run test.
239    TESTING $H5REPACK $@
240    (
241        cd $TESTDIR
242        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
243    )
244    RET=$?
245    if [ $RET != 0 ] ; then
246        echo "*FAILED*"
247        nerrors="`expr $nerrors + 1`"
248    else
249        echo " PASSED"
250        DIFFTEST $infile $outfile
251    fi
252    rm -f $outfile
253}
254
255#------------------------------------------
256# Verifying layouts of a dataset
257VERIFY_LAYOUT_DSET()
258{
259    layoutfile=layout-$1.$2
260    dset=$3
261    expectlayout=$4
262    infile=$2
263    outfile=out-$1.$2
264    shift
265    shift
266    shift
267    shift
268
269    TESTING  $H5REPACK $@
270    (
271        cd $TESTDIR
272        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
273    )
274    RET=$?
275    if [ $RET != 0 ] ; then
276        echo "*FAILED*"
277        nerrors="`expr $nerrors + 1`"
278    else
279        echo " PASSED"
280        DIFFTEST $infile $outfile
281    fi
282
283    #---------------------------------
284    # check the layout from a dataset
285    VERIFY  "a dataset layout"
286    (
287        cd $TESTDIR
288        $RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
289    )
290    $GREP $expectlayout $TESTDIR/$layoutfile > /dev/null
291    if [ $? -eq 0 ]; then
292        echo " PASSED"
293    else
294        echo " FAILED"
295        nerrors="`expr $nerrors + 1`"
296    fi
297
298    # clean up tmp files
299    rm -f $outfile
300    rm -f $layoutfile
301}
302
303#----------------------------------------
304# Verifying layouts from entire file
305VERIFY_LAYOUT_ALL()
306{
307    infile=$2
308    outfile=out-$1.$2
309    layoutfile=layout-$1.$2
310    expectlayout=$3
311    shift
312    shift
313    shift
314
315    TESTING  $H5REPACK $@
316    (
317        cd $TESTDIR
318        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
319    )
320    RET=$?
321    if [ $RET != 0 ] ; then
322        echo "*FAILED*"
323        nerrors="`expr $nerrors + 1`"
324    else
325        echo " PASSED"
326        DIFFTEST $infile $outfile
327    fi
328
329
330    #---------------------------------
331    # check the layout from a dataset
332    # check if the other layouts still exsit
333    VERIFY  "layouts"
334    (
335        cd $TESTDIR
336        echo
337        # if CONTIGUOUS
338        if [ $expectlayout = "CONTIGUOUS" ]; then
339            TESTING $H5DUMP_BIN -pH $outfile
340            (
341                $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
342            )
343            $GREP "COMPACT" $layoutfile  > /dev/null
344            if [ $? -eq 0 ]; then
345                echo " FAILED"
346                nerrors="`expr $nerrors + 1`"
347            else
348                $GREP "CHUNKED" $layoutfile  > /dev/null
349                if [ $? -eq 0 ]; then
350                    echo " FAILED"
351                    nerrors="`expr $nerrors + 1`"
352                else
353                    echo " PASSED"
354                fi
355            fi
356        else
357            # if COMPACT
358            if [ $expectlayout = "COMPACT" ]; then
359                TESTING $H5DUMP_BIN -pH $outfile
360                (
361                    $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
362                )
363                $GREP "CHUNKED" $layoutfile  > /dev/null
364                if [ $? -eq 0 ]; then
365                    echo " FAILED"
366                    nerrors="`expr $nerrors + 1`"
367                else
368                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
369                    if [ $? -eq 0 ]; then
370                        echo " FAILED"
371                        nerrors="`expr $nerrors + 1`"
372                    else
373                        echo " PASSED"
374                    fi
375                fi
376            else
377                # if CHUNKED
378                if [ $expectlayout = "CHUNKED" ]; then
379                    TESTING $H5DUMP_BIN -pH $outfile
380                    (
381                        $RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
382                    )
383                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
384                    if [ $? -eq 0 ]; then
385                        echo " FAILED"
386                        nerrors="`expr $nerrors + 1`"
387                    else
388                        $GREP "COMPACT" $layoutfile  > /dev/null
389                        if [ $? -eq 0 ]; then
390                            echo " FAILED"
391                            nerrors="`expr $nerrors + 1`"
392                        else
393                            echo " PASSED"
394                        fi
395                    fi
396                fi
397           fi
398        fi
399    )
400
401    # clean up tmp files
402    rm -f $outfile
403    rm -f $layoutfile
404}
405
406# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
407#
408TOOLTEST0()
409{
410    infile=$2
411    outfile=out-$1.$2
412    shift
413    shift
414
415    # Run test.
416    TESTING $H5REPACK $@
417    (
418        cd $TESTDIR
419        $RUNSERIAL $H5REPACK_BIN -i $infile -o $outfile "$@"
420    )
421    RET=$?
422    if [ $RET != 0 ] ; then
423        echo "*FAILED*"
424        nerrors="`expr $nerrors + 1`"
425    else
426        echo " PASSED"
427        DIFFTEST $infile $outfile
428    fi
429    rm -f $outfile
430}
431
432
433# same as TOOLTEST, but it uses without -i -o options
434# used to test the family driver, where these files reside
435#
436TOOLTEST1()
437{
438    infile=$2
439    outfile=out-$1.$2
440    shift
441    shift
442
443    # Run test.
444    TESTING $H5REPACK $@
445    (
446        cd $TESTDIR
447        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
448    )
449    RET=$?
450    if [ $RET != 0 ] ; then
451    echo "*FAILED*"
452    nerrors="`expr $nerrors + 1`"
453    else
454    echo " PASSED"
455    DIFFTEST $infile $outfile
456    fi
457    rm -f $outfile
458}
459
460# This is same as TOOLTEST() with comparing display output
461# from -v option
462#
463TOOLTESTV()
464{
465    expect="$TESTDIR/$2-$1.tst"
466    actual="$TESTDIR/`basename $2 .ddl`.out"
467    actual_err="$TESTDIR/`basename $2 .ddl`.err"
468
469    infile=$2
470    outfile=out-$1.$2
471    shift
472    shift
473
474    # Run test.
475    TESTING $H5REPACK $@
476    (
477        cd $TESTDIR
478        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
479    ) >$actual 2>$actual_err
480    RET=$?
481    if [ $RET != 0 ] ; then
482    echo "*FAILED*"
483    nerrors="`expr $nerrors + 1`"
484    else
485    echo " PASSED"
486    DIFFTEST $infile $outfile
487    fi
488
489    # display output compare
490    STDOUT_FILTER $actual
491    cat $actual_err >> $actual
492
493    VERIFY output from $H5REPACK $@
494    if cmp -s $expect $actual; then
495     echo " PASSED"
496    else
497     echo "*FAILED*"
498     echo "    Expected result (*.tst) differs from actual result (*.out)"
499     nerrors="`expr $nerrors + 1`"
500     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
501    fi
502
503   rm -f $actual $actual_err
504   rm -f $outfile
505}
506
507# This is same as TOOLTESTV() with comparing h5dump output
508# from -pH option
509#
510TOOLTEST_DUMP()
511{
512    infile=$2
513    outfile=out-$1.$2
514    expect="$TESTDIR/$1.$2.ddl"
515    actual="$TESTDIR/out-$1.$2.out"
516    actual_err="$TESTDIR/out-$1.$2.err"
517
518    shift
519    shift
520
521    # Run test.
522    TESTING $H5REPACK $@
523    (
524        cd $TESTDIR
525        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
526    ) >$actual 2>$actual_err
527    RET=$?
528    if [ $RET != 0 ] ; then
529    echo "*FAILED*"
530    nerrors="`expr $nerrors + 1`"
531    else
532     echo " PASSED"
533     VERIFY h5dump output $@
534     (
535        cd $TESTDIR
536        $RUNSERIAL $H5DUMP_BIN -q creation_order -pH $outfile
537     ) >$actual 2>$actual_err
538     cat $actual_err >> $actual
539
540     RET=$?
541
542    fi
543
544    if cmp -s $expect $actual; then
545     echo " PASSED"
546    else
547     echo "*FAILED*"
548     echo "    Expected result (*.ddl) differs from actual result (*.out)"
549     nerrors="`expr $nerrors + 1`"
550     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
551    fi
552
553   rm -f $actual $actual_err
554   rm -f $outfile
555}
556
557# TOOLTEST_META:
558# Test metadata block size option.
559# Reason to create a function here is to localize all special steps related to
560# metadata block size option in one place.  This is a quick solution.  More
561# thought out solution needed when there is more time.
562#
563# $1: test input file
564# $2:$: metadata options (either "-M size" or "--metadata_block_size=size")
565#
566# Algorithm:
567# Run it once without the metadata option ($2-$);
568# Save the result output file;
569# Run it second time with the metadata option;
570# Verify the output file of second run is larger than the one of 1st run.
571TOOLTEST_META()
572{
573    infile=$2
574    outfile=out-$1.$2
575    shift
576    shift
577
578    # Run test.
579    TESTING $H5REPACK $@
580    (
581        cd $TESTDIR
582        $RUNSERIAL $H5REPACK_BIN $infile $outfile
583    )
584    RET=$?
585    if [ $RET != 0 ] ; then
586        echo "*FAILED*"
587        nerrors="`expr $nerrors + 1`"
588    else
589        echo " PASSED"
590        DIFFTEST $infile $outfile
591    fi
592    # get the size of the first output file
593    size1=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`
594
595    # 2nd run with metadata option
596    # Run test.
597    TESTING $H5REPACK $@
598    (
599        cd $TESTDIR
600        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
601    )
602    RET=$?
603    if [ $RET != 0 ] ; then
604        echo "*FAILED*"
605        nerrors="`expr $nerrors + 1`"
606    else
607        echo " PASSED"
608        DIFFTEST $infile $outfile
609    fi
610    # get the size of the second output file
611    size2=`wc -c $TESTDIR/$outfile | cut -d' ' -f1`
612
613    # verify sizes.
614    MESSAGE "Verify the sizes of both output files ($size1 vs $size2)"
615    if [ $size1 -lt $size2 ]; then
616        # pass
617    echo " PASSED"
618    else
619    #fail
620    echo "*FAILED*"
621        nerrors="`expr $nerrors + 1`"
622    fi
623
624    rm -f $outfile
625}
626
627# ADD_HELP_TEST
628TOOLTEST_HELP() {
629
630    expect="$TESTDIR/$1"
631    actual="$TESTDIR/`basename $1 .txt`.out"
632    actual_err="$TESTDIR/`basename $1 .txt`.err"
633    shift
634
635    # Run test.
636    TESTING $H5REPACK $@
637    (
638      cd $TESTDIR
639      $RUNSERIAL $H5REPACK_BIN "$@"
640    ) >$actual 2>$actual_err
641    cat $actual_err >> $actual
642
643    if [ ! -f $expect ]; then
644    # Create the expect data file if it doesn't yet exist.
645      echo " CREATED"
646      cp $actual $expect-CREATED
647      echo "    Expected result (*.txt) missing"
648      nerrors="`expr $nerrors + 1`"
649    elif cmp -s $expect $actual; then
650      echo " PASSED"
651    else
652      echo "*FAILED*"
653      echo "    Expected output (*.txt) differs from actual output (*.out)"
654      nerrors="`expr $nerrors + 1`"
655    fi
656
657    # Clean up output file
658    if test -z "$HDF5_NOCLEANUP"; then
659     rm -f $actual $actual_err
660    fi
661
662}
663
664# This is different from $srcdir/../../bin/output_filter.sh
665STDOUT_FILTER() {
666    result_file=$1
667    tmp_file=/tmp/h5test_tmp_$$
668    # Filter name of files.
669    cp $result_file $tmp_file
670    sed -e '/^Opening file/d' -e '/^Making file/d' \
671    < $tmp_file > $result_file
672    # cleanup
673    rm -f $tmp_file
674}
675
676#
677# The tests
678# We use the files generated by h5repacktst
679# Each run generates "<file>.out.h5" and the tool h5diff is used to
680# compare the input and output files
681#
682# the tests are the same as the program h5repacktst, but run from the CLI
683#
684
685# See which filters are usable (and skip tests for filters we
686# don't have).  Do this by searching H5pubconf.h to see which
687# filters are defined.
688
689# detect whether the encoder is present.
690USE_FILTER_SZIP_ENCODER="no";
691if test $USE_FILTER_SZIP = "yes"; then
692USE_FILTER_SZIP_ENCODER=`$RUNSERIAL $H5DETECTSZIP_BIN`
693fi
694
695##############################################################################
696###              T H E   T E S T S
697##############################################################################
698# prepare for test
699COPY_TESTFILES_TO_TESTDIR
700
701TOOLTEST_HELP h5repack-help.txt -h
702
703# copy files (these files have no filters)
704TOOLTEST fill h5repack_fill.h5
705TOOLTEST objs h5repack_objs.h5
706TOOLTEST attr h5repack_attr.h5
707TOOLTEST hlink h5repack_hlink.h5
708TOOLTEST layout h5repack_layout.h5
709TOOLTEST early h5repack_early.h5
710
711# nested 8bit enum in both deflated and non-deflated datafiles
712if [ $USE_FILTER_DEFLATE != "yes" ]; then
713TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum.h5
714else
715TOOLTEST nested_8bit_enum h5repack_nested_8bit_enum_deflated.h5
716fi
717
718# use h5repack_layout.h5 to write some filters  (this file has  no filters)
719
720# gzip with individual object
721arg="h5repack_layout.h5 -f dset1:GZIP=1  -l dset1:CHUNK=20x10"
722if test  $USE_FILTER_DEFLATE != "yes" ; then
723 SKIP $arg
724else
725 TOOLTEST gzip_individual $arg
726fi
727
728# gzip for all
729arg="h5repack_layout.h5 -f GZIP=1"
730if test  $USE_FILTER_DEFLATE != "yes" ; then
731 SKIP $arg
732else
733 TOOLTEST gzip_all $arg
734fi
735
736# szip with individual object
737arg="h5repack_layout.h5 -f dset2:SZIP=8,EC  -l dset2:CHUNK=20x10"
738if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
739 SKIP $arg
740else
741 TOOLTEST szip_individual $arg
742fi
743
744# szip for all
745arg="h5repack_layout.h5 -f SZIP=8,NN"
746if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
747 SKIP $arg
748else
749 TOOLTEST szip_all $arg
750fi
751
752# shuffle with individual object
753arg="h5repack_layout.h5 -f dset2:SHUF  -l dset2:CHUNK=20x10"
754TOOLTEST shuffle_individual $arg
755
756
757# shuffle for all
758arg="h5repack_layout.h5 -f SHUF"
759TOOLTEST shuffle_all $arg
760
761# fletcher32  with individual object
762arg="h5repack_layout.h5 -f dset2:FLET  -l dset2:CHUNK=20x10"
763TOOLTEST fletcher_individual $arg
764
765# fletcher32 for all
766arg="h5repack_layout.h5 -f FLET"
767TOOLTEST fletcher_all $arg
768
769# all filters
770arg="h5repack_layout.h5 -f dset2:SHUF -f dset2:FLET -f dset2:SZIP=8,NN -f dset2:GZIP=1 -l dset2:CHUNK=20x10"
771if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
772 SKIP $arg
773else
774 TOOLTEST all_filters $arg
775fi
776
777# verbose gzip with individual object
778arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9"
779if test  $USE_FILTER_DEFLATE != "yes" ; then
780 SKIP $arg
781else
782 # compare output
783 TOOLTESTV gzip_verbose_filters $arg
784fi
785
786###########################################################
787# the following tests assume the input files have filters
788###########################################################
789
790# szip copy
791arg="h5repack_szip.h5"
792if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
793 SKIP $arg
794else
795 TOOLTEST szip_copy $arg
796fi
797
798# szip remove
799arg="h5repack_szip.h5 --filter=dset_szip:NONE"
800if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" ; then
801 SKIP $arg
802else
803 TOOLTEST szip_remove $arg
804fi
805
806# deflate copy
807arg="h5repack_deflate.h5"
808if test $USE_FILTER_DEFLATE != "yes" ; then
809 SKIP $arg
810else
811 TOOLTEST deflate_copy $arg
812fi
813
814# deflate remove
815arg="h5repack_deflate.h5 -f dset_deflate:NONE"
816if test $USE_FILTER_DEFLATE != "yes" ; then
817 SKIP $arg
818else
819 TOOLTEST deflate_remove $arg
820fi
821
822# shuffle copy
823arg="h5repack_shuffle.h5"
824TOOLTEST shuffle_copy $arg
825
826# shuffle remove
827arg="h5repack_shuffle.h5 -f dset_shuffle:NONE"
828TOOLTEST shuffle_remove $arg
829
830# fletcher32 copy
831arg="h5repack_fletcher.h5"
832TOOLTEST fletcher_copy $arg
833
834# fletcher32 remove
835arg="h5repack_fletcher.h5 -f dset_fletcher32:NONE"
836TOOLTEST fletcher_remove $arg
837
838# nbit copy
839arg="h5repack_nbit.h5"
840TOOLTEST nbit_copy $arg
841
842# nbit remove
843arg="h5repack_nbit.h5 -f dset_nbit:NONE"
844TOOLTEST nbit_remove $arg
845
846# nbit add
847arg="h5repack_nbit.h5 -f dset_int31:NBIT"
848TOOLTEST nbit_add $arg
849
850# scaleoffset copy
851arg="h5repack_soffset.h5"
852TOOLTEST scale_copy $arg
853
854# scaleoffset add
855arg="h5repack_soffset.h5 -f dset_none:SOFF=31,IN"
856TOOLTEST scale_add $arg
857
858# scaleoffset remove
859arg="h5repack_soffset.h5 -f dset_scaleoffset:NONE"
860TOOLTEST scale_remove $arg
861
862# remove all  filters
863arg="h5repack_filters.h5 -f NONE"
864if test $USE_FILTER_DEFLATE != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes" ; then
865 SKIP $arg
866else
867 TOOLTEST remove_all $arg
868fi
869
870#filter conversions
871
872arg="h5repack_deflate.h5 -f dset_deflate:SZIP=8,NN"
873if test $USE_FILTER_SZIP_ENCODER != "yes" -o $USE_FILTER_SZIP != "yes" -o $USE_FILTER_DEFLATE != "yes" ; then
874 SKIP $arg
875else
876 TOOLTEST deflate_convert $arg
877fi
878
879arg="h5repack_szip.h5 -f dset_szip:GZIP=1"
880if test $USE_FILTER_SZIP != "yes" -o $USE_FILTER_SZIP_ENCODER != "yes"  -o $USE_FILTER_DEFLATE != "yes" ; then
881 SKIP $arg
882else
883 TOOLTEST szip_convert $arg
884fi
885
886
887#limit
888arg="h5repack_layout.h5 -f GZIP=1 -m 1024"
889if test $USE_FILTER_DEFLATE != "yes"  ; then
890 SKIP $arg
891else
892 TOOLTEST_DUMP deflate_limit $arg
893fi
894
895#file
896arg="h5repack_layout.h5 -e h5repack.info"
897if test $USE_FILTER_DEFLATE != "yes" ; then
898 SKIP $arg
899else
900 TOOLTEST deflate_file $arg
901fi
902
903#crtorder
904arg="tordergr.h5 -L"
905TOOLTEST_DUMP crtorder $arg
906
907#########################################################
908# layout options (these files have no filters)
909#########################################################
910VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10
911
912VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10
913
914VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI
915
916VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI
917
918VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA
919
920VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA
921
922################################################################
923# layout conversions (file has no filters)
924###############################################################
925
926VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI
927
928VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5
929
930VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA
931
932VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA
933
934VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6
935
936VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI
937
938VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA
939
940VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI
941
942VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13
943
944# test convert small size dataset ( < 1k) to compact layout without -m
945VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA
946
947VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA
948
949#---------------------------------------------------------------------------
950# Test file contains chunked datasets (need multiple dsets) with
951# unlimited max dims.   (HDFFV-7933)
952# Use first dset to test.
953#---------------------------------------------------------------------------
954# chunk to chunk - specify chunk dim bigger than any current dim
955VERIFY_LAYOUT_DSET chunk2chunk h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300
956
957# chunk to contiguous
958VERIFY_LAYOUT_DSET chunk2conti h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI
959
960# chunk to compact - convert big dataset (should be > 64k) for this purpose,
961# should remain as original layout (chunk)
962VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA
963
964#--------------------------------------------------------------------------
965# Test -f for some specific cases. Chunked dataset with unlimited max dims.
966# (HDFFV-8012)
967#--------------------------------------------------------------------------
968# - should not fail
969# - should not change max dims from unlimit
970
971# chunk dim is bigger than dataset dim. ( dset size < 64k )
972VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE
973# chunk dim is bigger than dataset dim. ( dset size > 64k )
974VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE
975
976# chunk dims are smaller than dataset dims. ( dset size < 64k )
977#TOOLTEST_MAIN h5repack_layout3.h5  -f chunk_unlimit3:NONE
978VERIFY_LAYOUT_DSET error3 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED -f chunk_unlimit3:NONE
979
980# file input - should not fail
981TOOLTEST error4 h5repack_layout3.h5 -f NONE
982
983#--------------------------------------------------------------------------
984# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset
985# (dset size < 64K) and with unlimited max dims on a condition as follow.
986# (HDFFV-8214)
987#--------------------------------------------------------------------------
988
989# chunk dim is bigger than dataset dim. should succeed.
990VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI
991# chunk dim is smaller than dataset dim. should succeed.
992VERIFY_LAYOUT_DSET ckdim_smaller h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI
993
994
995# Native option
996# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
997# report a difference.
998TOOLTEST native_fill h5repack_fill.h5 -n
999TOOLTEST native_attr h5repack_attr.h5 -n
1000
1001
1002# latest file format with long switches. use FILE4=h5repack_layout.h5 (no filters)
1003arg="h5repack_layout.h5 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]"
1004if test $USE_FILTER_DEFLATE != "yes" ; then
1005 SKIP $arg
1006else
1007 VERIFY_LAYOUT_ALL layout_long_switches h5repack_layout.h5 CHUNKED --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest --compact=8 --indexed=6 --ssize=8[:dtype]
1008fi
1009
1010# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
1011arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
1012if test $USE_FILTER_DEFLATE != "yes" ; then
1013 SKIP $arg
1014else
1015 VERIFY_LAYOUT_ALL layout_short_switches h5repack_layout.h5 CHUNKED -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]
1016fi
1017
1018# several global filters
1019
1020arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF"
1021if test $USE_FILTER_DEFLATE != "yes" ; then
1022 SKIP $arg
1023else
1024 TOOLTEST global_filters $arg
1025fi
1026
1027# syntax of -i infile -o outfile
1028# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
1029arg="h5repack_layout.h5 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
1030if test $USE_FILTER_DEFLATE != "yes" ; then
1031 SKIP $arg
1032else
1033 TOOLTEST0 old_style_layout_short_switches $arg
1034fi
1035
1036# add a userblock to file
1037arg="h5repack_objs.h5 -u ublock.bin -b 2048"
1038TOOLTEST add_userblock $arg
1039
1040# add alignment
1041arg="h5repack_objs.h5 -t 1 -a 1 "
1042TOOLTEST add_alignment $arg
1043
1044# Check repacking file with old version of layout message (should get upgraded
1045#       to new version and be readable, etc.)
1046TOOLTEST upgrade_layout h5repack_layouto.h5
1047
1048# test for datum size > H5TOOLS_MALLOCSIZE
1049TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1
1050
1051# Check repacking file with committed datatypes in odd configurations
1052TOOLTEST committed_dt h5repack_named_dtypes.h5
1053
1054# tests family driver (file is located in common testfiles folder, uses TOOLTEST1
1055TOOLTEST1 family tfamily%05d.h5
1056
1057# test various references (bug 1814 and 1726)
1058TOOLTEST bug1814 h5repack_refs.h5
1059
1060# test attribute with various references (bug1797 / HDFFV-5932)
1061# the references in attribute of compund or vlen datatype
1062TOOLTEST HDFFV-5932 h5repack_attr_refs.h5
1063
1064# Add test for memory leak in attirbute. This test is verified by CTEST.
1065# 1. leak from vlen string
1066# 2. leak from compound type without reference member
1067# (HDFFV-7840, )
1068# Note: this test is experimental for sharing test file among tools
1069TOOLTEST HDFFV-7840 h5diff_attr1.h5
1070
1071# tests for metadata block size option
1072TOOLTEST_META meta_short h5repack_layout.h5 -M 8192
1073TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192
1074
1075# Clean up temporary files/directories
1076CLEAN_TESTFILES_AND_TESTDIR
1077
1078if test $nerrors -eq 0 ; then
1079    echo "All $TESTNAME tests passed."
1080    exit $EXIT_SUCCESS
1081else
1082    echo "$TESTNAME tests failed with $nerrors errors."
1083    exit $EXIT_FAILURE
1084fi
1085
1086