1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF5. The full HDF5 copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the COPYING file, which can be found at the root of the source code *
9 * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
10 * If you do not have access to either file, you may request a copy from *
11 * help@hdfgroup.org. *
12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13
14 #include "H5Zmodule.h" /* This source code file is part of the H5Z module */
15
16
17 #include "H5private.h" /* Generic Functions */
18 #include "H5Eprivate.h" /* Error handling */
19 #include "H5Fprivate.h" /* File access */
20 #include "H5Iprivate.h" /* IDs */
21 #include "H5MMprivate.h" /* Memory management */
22 #include "H5Oprivate.h" /* Object headers */
23 #include "H5Pprivate.h" /* Property lists */
24 #include "H5Sprivate.h" /* Dataspaces */
25 #include "H5Tprivate.h" /* Datatypes */
26 #include "H5Zpkg.h" /* Data filters */
27
28 #ifdef H5_HAVE_FILTER_SZIP
29
30 #ifdef H5_HAVE_SZLIB_H
31 # include "szlib.h"
32 #endif
33
34 /* Local function prototypes */
35 static htri_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
36 static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id);
37 static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts,
38 const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf);
39
40 /* This message derives from H5Z */
41 H5Z_class2_t H5Z_SZIP[1] = {{
42 H5Z_CLASS_T_VERS, /* H5Z_class_t version */
43 H5Z_FILTER_SZIP, /* Filter id number */
44 1, /* Assume encoder present: check before registering */
45 1, /* decoder_present flag (set to true) */
46 "szip", /* Filter name for debugging */
47 H5Z_can_apply_szip, /* The "can apply" callback */
48 H5Z_set_local_szip, /* The "set local" callback */
49 H5Z_filter_szip, /* The actual filter function */
50 }};
51
52
53
54 /*-------------------------------------------------------------------------
55 * Function: H5Z_can_apply_szip
56 *
57 * Purpose: Check the parameters for szip compression for validity and
58 * whether they fit a particular dataset.
59 *
60 * Note: This function currently range-checks for datatypes with
61 * 8-bit boundaries (8, 16, 24, etc.). It appears that the szip
62 * library can actually handle 1-24, 32 & 64 bit samples. If
63 * this becomes important, we should make the checks below more
64 * sophisticated and have them check for n-bit datatypes of the
65 * correct size, etc. - QAK
66 *
67 * Return: Success: Non-negative
68 * Failure: Negative
69 *
70 * Programmer: Quincey Koziol
71 * Monday, April 7, 2003
72 *
73 *-------------------------------------------------------------------------
74 */
75 static htri_t
H5Z_can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id,hid_t type_id,hid_t H5_ATTR_UNUSED space_id)76 H5Z_can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id)
77 {
78 const H5T_t *type; /* Datatype */
79 unsigned dtype_size; /* Datatype's size (in bits) */
80 H5T_order_t dtype_order; /* Datatype's endianness order */
81 htri_t ret_value = TRUE; /* Return value */
82
83 FUNC_ENTER_NOAPI(FAIL)
84
85 /* Get datatype */
86 if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
87 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
88
89 /* Get datatype's size, for checking the "bits-per-pixel" */
90 if((dtype_size = (8 * H5T_get_size(type))) == 0)
91 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
92
93 /* Range check datatype's size */
94 if(dtype_size > 32 && dtype_size != 64)
95 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype size")
96
97 /* Get datatype's endianness order */
98 if((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
99 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order")
100
101 /* Range check datatype's endianness order */
102 /* (Note: this may not handle non-atomic datatypes well) */
103 if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
104 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype endianness order")
105
106 done:
107 FUNC_LEAVE_NOAPI(ret_value)
108 } /* end H5Z_can_apply_szip() */
109
110
111 /*-------------------------------------------------------------------------
112 * Function: H5Z_set_local_szip
113 *
114 * Purpose: Set the "local" dataset parameters for szip compression.
115 *
116 * Return: Success: Non-negative
117 * Failure: Negative
118 *
119 * Programmer: Quincey Koziol
120 * Monday, April 7, 2003
121 *
122 * Modifications: Used new logic to set the size of the scanline parameter.
123 * Now SZIP compression can be applied to the chunk
124 * of any shape and size with only one restriction: the number
125 * of elements in the chunk has to be not less than number
126 * of elements (pixels) in the block (cd_values[H5Z_SZIP_PARM_PPB]
127 * parameter).
128 * Elena Pourmal, July 20, 2004
129 *
130 *-------------------------------------------------------------------------
131 */
132 static herr_t
H5Z_set_local_szip(hid_t dcpl_id,hid_t type_id,hid_t space_id)133 H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id)
134 {
135 H5P_genplist_t *dcpl_plist; /* Property list pointer */
136 const H5T_t *type; /* Datatype */
137 const H5S_t *ds; /* Dataspace */
138 unsigned flags; /* Filter flags */
139 size_t cd_nelmts = H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */
140 unsigned cd_values[H5Z_SZIP_TOTAL_NPARMS]; /* Filter parameters */
141 hsize_t dims[H5O_LAYOUT_NDIMS]; /* Dataspace (i.e. chunk) dimensions */
142 int ndims; /* Number of (chunk) dimensions */
143 H5T_order_t dtype_order; /* Datatype's endianness order */
144 size_t dtype_size; /* Datatype's size (in bits) */
145 size_t dtype_precision; /* Datatype's precision (in bits) */
146 size_t dtype_offset; /* Datatype's offset (in bits) */
147 hsize_t scanline; /* Size of dataspace's fastest changing dimension */
148 herr_t ret_value = SUCCEED; /* Return value */
149
150 FUNC_ENTER_NOAPI(FAIL)
151
152 /* Get the plist structure */
153 if(NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
154 HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
155
156 /* Get datatype */
157 if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
158 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
159
160 /* Get the filter's current parameters */
161 if(H5P_get_filter_by_id(dcpl_plist, H5Z_FILTER_SZIP, &flags, &cd_nelmts, cd_values, 0, NULL, NULL) < 0)
162 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters")
163
164 /* Get datatype's size, for checking the "bits-per-pixel" */
165 if((dtype_size = (8 * H5T_get_size(type))) == 0)
166 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size");
167
168 /* Get datatype's precision, in case is less than full bits */
169 if((dtype_precision = H5T_get_precision(type)) == 0)
170 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype precision");
171
172 if(dtype_precision < dtype_size) {
173 dtype_offset = H5T_get_offset(type);
174 if(dtype_offset != 0)
175 dtype_precision = dtype_size;
176 } /* end if */
177 if(dtype_precision > 24) {
178 if(dtype_precision <= 32)
179 dtype_precision = 32;
180 else if(dtype_precision <= 64)
181 dtype_precision = 64;
182 } /* end if */
183
184 /* Set "local" parameter for this dataset's "bits-per-pixel" */
185 cd_values[H5Z_SZIP_PARM_BPP] = dtype_precision;
186
187 /* Get dataspace */
188 if(NULL == (ds = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE)))
189 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace")
190
191 /* Get dimensions for dataspace */
192 if((ndims = H5S_get_simple_extent_dims(ds, dims, NULL)) < 0)
193 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions")
194
195 /* Set "local" parameter for this dataset's "pixels-per-scanline" */
196 /* (Use the chunk's fastest changing dimension size) */
197 HDassert(ndims > 0);
198 scanline = dims[ndims - 1];
199
200 /* Adjust scanline if it is smaller than number of pixels per block or
201 if it is bigger than maximum pixels per scanline, or there are more than
202 SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */
203
204 /* Check the pixels per block against the 'scanline' size */
205 if(scanline < cd_values[H5Z_SZIP_PARM_PPB]) {
206 hssize_t npoints; /* Number of points in the dataspace */
207
208 /* Get number of elements for the dataspace; use
209 total number of elements in the chunk to define the new 'scanline' size */
210 if((npoints = H5S_GET_EXTENT_NPOINTS(ds)) < 0)
211 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")
212 if(npoints < cd_values[H5Z_SZIP_PARM_PPB])
213 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk")
214 scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints);
215 }
216 else {
217 if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE)
218 scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline);
219 else
220 scanline = cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE;
221 } /* end else */
222
223 /* Assign the final value to the scanline */
224 H5_CHECKED_ASSIGN(cd_values[H5Z_SZIP_PARM_PPS], unsigned, scanline, hsize_t);
225
226 /* Get datatype's endianness order */
227 if((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
228 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
229
230 /* Set the correct endianness flag for szip */
231 /* (Note: this may not handle non-atomic datatypes well) */
232 cd_values[H5Z_SZIP_PARM_MASK] &= ~(SZ_LSB_OPTION_MASK|SZ_MSB_OPTION_MASK);
233 switch(dtype_order) {
234 case H5T_ORDER_LE: /* Little-endian byte order */
235 cd_values[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK;
236 break;
237
238 case H5T_ORDER_BE: /* Big-endian byte order */
239 cd_values[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK;
240 break;
241
242 case H5T_ORDER_ERROR:
243 case H5T_ORDER_VAX:
244 case H5T_ORDER_MIXED:
245 case H5T_ORDER_NONE:
246 default:
247 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
248 } /* end switch */
249
250 /* Modify the filter's parameters for this dataset */
251 if(H5P_modify_filter(dcpl_plist, H5Z_FILTER_SZIP, flags, H5Z_SZIP_TOTAL_NPARMS, cd_values) < 0)
252 HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local szip parameters")
253
254 done:
255 FUNC_LEAVE_NOAPI(ret_value)
256 } /* end H5Z_set_local_szip() */
257
258
259 /*-------------------------------------------------------------------------
260 * Function: H5Z_filter_szip
261 *
262 * Purpose: Implement an I/O filter around the 'rice' algorithm in
263 * libsz
264 *
265 * Return: Success: Size of buffer filtered
266 * Failure: 0
267 *
268 * Programmer: Kent Yang
269 * Tuesday, April 1, 2003
270 *
271 *-------------------------------------------------------------------------
272 */
273 static size_t
H5Z_filter_szip(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],size_t nbytes,size_t * buf_size,void ** buf)274 H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
275 size_t nbytes, size_t *buf_size, void **buf)
276 {
277 size_t ret_value = 0; /* Return value */
278 size_t size_out = 0; /* Size of output buffer */
279 unsigned char *outbuf = NULL; /* Pointer to new output buffer */
280 unsigned char *newbuf = NULL; /* Pointer to input buffer */
281 SZ_com_t sz_param; /* szip parameter block */
282
283 FUNC_ENTER_NOAPI(0)
284
285 /* Sanity check to make certain that we haven't drifted out of date with
286 * the mask options from the szlib.h header */
287 HDassert(H5_SZIP_ALLOW_K13_OPTION_MASK==SZ_ALLOW_K13_OPTION_MASK);
288 HDassert(H5_SZIP_CHIP_OPTION_MASK==SZ_CHIP_OPTION_MASK);
289 HDassert(H5_SZIP_EC_OPTION_MASK==SZ_EC_OPTION_MASK);
290 HDassert(H5_SZIP_LSB_OPTION_MASK==SZ_LSB_OPTION_MASK);
291 HDassert(H5_SZIP_MSB_OPTION_MASK==SZ_MSB_OPTION_MASK);
292 HDassert(H5_SZIP_NN_OPTION_MASK==SZ_NN_OPTION_MASK);
293 HDassert(H5_SZIP_RAW_OPTION_MASK==SZ_RAW_OPTION_MASK);
294
295 /* Check arguments */
296 if (cd_nelmts!=4)
297 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid number of filter parameters")
298
299 /* Copy the filter parameters into the szip parameter block */
300 H5_CHECKED_ASSIGN(sz_param.options_mask, int, cd_values[H5Z_SZIP_PARM_MASK], unsigned);
301 H5_CHECKED_ASSIGN(sz_param.bits_per_pixel, int, cd_values[H5Z_SZIP_PARM_BPP], unsigned);
302 H5_CHECKED_ASSIGN(sz_param.pixels_per_block, int, cd_values[H5Z_SZIP_PARM_PPB], unsigned);
303 H5_CHECKED_ASSIGN(sz_param.pixels_per_scanline, int, cd_values[H5Z_SZIP_PARM_PPS], unsigned);
304
305 /* Input; uncompress */
306 if (flags & H5Z_FLAG_REVERSE) {
307 uint32_t stored_nalloc; /* Number of bytes the compressed block will expand into */
308 size_t nalloc; /* Number of bytes the compressed block will expand into */
309
310 /* Get the size of the uncompressed buffer */
311 newbuf = (unsigned char *)(*buf);
312 UINT32DECODE(newbuf,stored_nalloc);
313 H5_CHECKED_ASSIGN(nalloc, size_t, stored_nalloc, uint32_t);
314
315 /* Allocate space for the uncompressed buffer */
316 if(NULL == (outbuf = (unsigned char *)H5MM_malloc(nalloc)))
317 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for szip decompression")
318
319 /* Decompress the buffer */
320 size_out=nalloc;
321 if(SZ_BufftoBuffDecompress(outbuf, &size_out, newbuf, nbytes-4, &sz_param) != SZ_OK)
322 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "szip_filter: decompression failed")
323 HDassert(size_out==nalloc);
324
325 /* Free the input buffer */
326 H5MM_xfree(*buf);
327
328 /* Set return values */
329 *buf = outbuf;
330 outbuf = NULL;
331 *buf_size = nalloc;
332 ret_value = size_out;
333 }
334 /* Output; compress */
335 else {
336 unsigned char *dst = NULL; /* Temporary pointer to new output buffer */
337
338 /* Allocate space for the compressed buffer & header (assume data won't get bigger) */
339 if(NULL == (dst=outbuf = (unsigned char *)H5MM_malloc(nbytes+4)))
340 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate szip destination buffer")
341
342 /* Encode the uncompressed length */
343 H5_CHECK_OVERFLOW(nbytes,size_t,uint32_t);
344 UINT32ENCODE(dst,nbytes);
345
346 /* Compress the buffer */
347 size_out = nbytes;
348 if(SZ_OK!= SZ_BufftoBuffCompress(dst, &size_out, *buf, nbytes, &sz_param))
349 HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow")
350 HDassert(size_out<=nbytes);
351
352 /* Free the input buffer */
353 H5MM_xfree(*buf);
354
355 /* Set return values */
356 *buf = outbuf;
357 outbuf = NULL;
358 *buf_size = nbytes+4;
359 ret_value = size_out+4;
360 }
361
362 done:
363 if(outbuf)
364 H5MM_xfree(outbuf);
365 FUNC_LEAVE_NOAPI(ret_value)
366 }
367
368 #endif /* H5_HAVE_FILTER_SZIP */
369
370