1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 2 * Copyright by The HDF Group. * 3 * Copyright by the Board of Trustees of the University of Illinois. * 4 * All rights reserved. * 5 * * 6 * This file is part of HDF5. The full HDF5 copyright notice, including * 7 * terms governing use, modification, and redistribution, is contained in * 8 * the COPYING file, which can be found at the root of the source code * 9 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * 10 * If you do not have access to either file, you may request a copy from * 11 * help@hdfgroup.org. * 12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 13 14 15 #ifndef H5REPACK_H__ 16 #define H5REPACK_H__ 17 18 #include "H5private.h" 19 #include "hdf5.h" 20 #include "h5trav.h" 21 22 #define H5FOPENERROR "unable to open file" 23 #define PFORMAT "%-7s %-7s %-7s\n" /* chunk info, compression info, name*/ 24 #define PFORMAT1 "%-7s %-7s %-7s" /* chunk info, compression info, name*/ 25 #define MAX_NC_NAME 256 /* max length of a name */ 26 #define MAX_VAR_DIMS 32 /* max per variable dimensions */ 27 #define FORMAT_OBJ " %-27s %s\n" /* obj type, name */ 28 #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ 29 #define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ 30 31 /* File space default information */ 32 #define FS_PAGESIZE_DEF 4096 33 #define FS_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR 34 #define FS_PERSIST_DEF FALSE 35 #define FS_THRESHOLD_DEF 1 36 37 38 /*------------------------------------------------------------------------- 39 * data structures for command line options 40 *------------------------------------------------------------------------- 41 */ 42 43 /* a list of names */ 44 typedef struct { 45 char obj[MAX_NC_NAME]; 46 } obj_list_t; 47 48 /* 49 the type of filter and additional parameter 50 type can be one of the filters 51 H5Z_FILTER_NONE 0, uncompress if compressed 52 H5Z_FILTER_DEFLATE 1 , deflation like gzip 53 H5Z_FILTER_SHUFFLE 2 , shuffle the data 54 H5Z_FILTER_FLETCHER32 3 , letcher32 checksum of EDC 55 H5Z_FILTER_SZIP 4 , szip compression 56 H5Z_FILTER_NBIT 5 , nbit compression 57 H5Z_FILTER_SCALEOFFSET 6 , scaleoffset compression 58 */ 59 60 #define CD_VALUES 20 61 62 typedef struct { 63 H5Z_filter_t filtn; /* filter identification number */ 64 unsigned filt_flag; /* filter definition flag */ 65 unsigned cd_values[CD_VALUES]; /* filter client data values */ 66 size_t cd_nelmts; /* filter client number of values */ 67 } filter_info_t; 68 69 /* chunk lengths along each dimension and rank */ 70 typedef struct { 71 hsize_t chunk_lengths[MAX_VAR_DIMS]; 72 int rank; 73 } chunk_info_t; 74 75 /* we currently define a maximum value for the filters array, 76 that corresponds to the current library filters */ 77 #define H5_REPACK_MAX_NFILTERS 6 78 79 /* information for one object, contains PATH, CHUNK info and FILTER info */ 80 typedef struct { 81 char path[MAX_NC_NAME]; /* name of object */ 82 filter_info_t filter[H5_REPACK_MAX_NFILTERS]; /* filter array */ 83 int nfilters; /* current number of filters */ 84 H5D_layout_t layout; /* layout information */ 85 chunk_info_t chunk; /* chunk information */ 86 hid_t refobj_id; /* object ID, references */ 87 } pack_info_t; 88 89 /* store a table of all objects */ 90 typedef struct { 91 unsigned int size; 92 unsigned int nelems; 93 pack_info_t *objs; 94 } pack_opttbl_t; 95 96 97 /*------------------------------------------------------------------------- 98 * command line options 99 *------------------------------------------------------------------------- 100 */ 101 102 /* all the above, ready to go to the hrepack call */ 103 typedef struct { 104 pack_opttbl_t *op_tbl; /*table with all -c and -f options */ 105 int all_layout; /*apply the layout to all objects */ 106 int all_filter; /*apply the filter to all objects */ 107 filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ 108 int n_filter_g; /*number of global filters */ 109 chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ 110 H5D_layout_t layout_g; /*global layout information for the ALL case */ 111 int verbose; /*verbose mode */ 112 hsize_t min_comp; /*minimum size to compress, in bytes */ 113 int use_native; /*use a native type in write */ 114 hbool_t latest; /*pack file with the latest file format */ 115 H5F_libver_t low_bound; /* The file's low bound as in H5Fset_libver_bounds() */ 116 H5F_libver_t high_bound; /* The file's high bound as in H5Fset_libver_bounds() */ 117 int grp_compact; /* Set the maximum number of links to store as header messages in the group */ 118 int grp_indexed; /* Set the minimum number of links to store in the indexed format */ 119 int msg_size[8]; /* Minimum size of shared messages: dataspace, 120 datatype, fill value, filter pipleline, attribute */ 121 const char *ublock_filename; /* user block file name */ 122 hsize_t ublock_size; /* user block size */ 123 hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ 124 hsize_t threshold; /* alignment threshold for H5Pset_alignment */ 125 hsize_t alignment; /* alignment for H5Pset_alignment */ 126 H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ 127 int fs_persist; /* Free space section threshold */ 128 long fs_threshold; /* Free space section threshold */ 129 long long fs_pagesize; /* File space page size */ 130 } pack_opt_t; 131 132 133 typedef struct named_dt_t { 134 haddr_t addr_in; /* Address of the named dtype in the in file */ 135 hid_t id_out; /* Open identifier for the dtype in the out file */ 136 struct named_dt_t *next; /* Next dtype */ 137 } named_dt_t; 138 139 /*------------------------------------------------------------------------- 140 * public functions 141 *------------------------------------------------------------------------- 142 */ 143 144 #ifdef __cplusplus 145 extern "C" { 146 #endif 147 148 int h5repack(const char* infile, const char* outfile, pack_opt_t *options); 149 int h5repack_addfilter(const char* str, pack_opt_t *options); 150 int h5repack_addlayout(const char* str, pack_opt_t *options); 151 int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest); 152 int h5repack_end(pack_opt_t *options); 153 int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); 154 int h5repack_cmp_pl(const char *fname1, const char *fname2); 155 156 /* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr() 157 * and struct named_dt_t were located in h5repack_copy.c as static prior to 158 * bugfix1726. 159 * Made shared functions as copy_attr() was needed in h5repack_refs.c. 160 * However copy_attr() may be obsoleted when H5Acopy is available and put back 161 * others to static in h5repack_copy.c. 162 */ 163 hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options); 164 int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err); 165 int copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, 166 trav_table_t *travt, pack_opt_t *options); 167 168 #ifdef __cplusplus 169 } 170 #endif 171 172 173 174 /*------------------------------------------------------------------------- 175 * private functions 176 *------------------------------------------------------------------------- 177 */ 178 179 180 /*------------------------------------------------------------------------- 181 * copy module 182 *------------------------------------------------------------------------- 183 */ 184 185 int copy_objects (const char* fnamein, 186 const char* fnameout, 187 pack_opt_t *options); 188 189 int do_copy_refobjs(hid_t fidin, 190 hid_t fidout, 191 trav_table_t *travt, 192 pack_opt_t *options); 193 194 /*------------------------------------------------------------------------- 195 * filters and verify module 196 *------------------------------------------------------------------------- 197 */ 198 void init_packobject(pack_info_t *obj); 199 200 201 /*------------------------------------------------------------------------- 202 * filters and copy module 203 *------------------------------------------------------------------------- 204 */ 205 206 int apply_filters(const char* name, /* object name from traverse list */ 207 int rank, /* rank of dataset */ 208 hsize_t *dims, /* dimensions of dataset */ 209 size_t msize, /* size of type */ 210 hid_t dcpl_id, /* dataset creation property list */ 211 pack_opt_t *options, /* repack options */ 212 int *has_filter); /* (OUT) object NAME has a filter */ 213 214 215 /*------------------------------------------------------------------------- 216 * options table 217 *------------------------------------------------------------------------- 218 */ 219 int options_table_init(pack_opttbl_t **tbl); 220 int options_table_free(pack_opttbl_t *table); 221 int options_add_layout(obj_list_t *obj_list, 222 unsigned n_objs, 223 pack_info_t *pack, 224 pack_opttbl_t *table); 225 int options_add_filter(obj_list_t *obj_list, 226 unsigned n_objs, 227 filter_info_t filt, 228 pack_opttbl_t *table); 229 pack_info_t* options_get_object(const char *path, 230 pack_opttbl_t *table); 231 232 /*------------------------------------------------------------------------- 233 * parse functions 234 *------------------------------------------------------------------------- 235 */ 236 237 obj_list_t* parse_filter(const char *str, 238 unsigned *n_objs, 239 filter_info_t *filt, 240 pack_opt_t *options, 241 int *is_glb); 242 243 obj_list_t* parse_layout(const char *str, 244 unsigned *n_objs, 245 pack_info_t *pack, /* info about object */ 246 pack_opt_t *options); 247 248 249 250 251 #endif /* H5REPACK_H__ */ 252 253