1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF5. The full HDF5 copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the COPYING file, which can be found at the root of the source code *
9 * distribution tree, or in https://www.hdfgroup.org/licenses. *
10 * If you do not have access to either file, you may request a copy from *
11 * help@hdfgroup.org. *
12 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
13
14 #include "H5Zmodule.h" /* This source code file is part of the H5Z module */
15
16 #include "H5private.h" /* Generic Functions */
17 #include "H5ACprivate.h" /* Metadata cache */
18 #include "H5Eprivate.h" /* Error handling */
19 #include "H5Iprivate.h" /* IDs */
20 #include "H5MMprivate.h" /* Memory management */
21 #include "H5Pprivate.h" /* Property lists */
22 #include "H5Oprivate.h" /* Object headers */
23 #include "H5Sprivate.h" /* Dataspaces */
24 #include "H5Tprivate.h" /* Datatypes */
25 #include "H5Zpkg.h" /* Data filters */
26
27 /* Struct of parameters needed for compressing/decompressing one atomic datatype */
28 typedef struct {
29 unsigned size; /* datatype size */
30 uint32_t minbits; /* minimum bits to compress one value of such datatype */
31 unsigned mem_order; /* current memory endianness order */
32 } parms_atomic;
33
34 enum H5Z_scaleoffset_t {
35 t_bad = 0,
36 t_uchar = 1,
37 t_ushort,
38 t_uint,
39 t_ulong,
40 t_ulong_long,
41 t_schar,
42 t_short,
43 t_int,
44 t_long,
45 t_long_long,
46 t_float,
47 t_double
48 };
49
50 /* Local function prototypes */
51 static htri_t H5Z__can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
52 static enum H5Z_scaleoffset_t H5Z__scaleoffset_get_type(unsigned dtype_class, unsigned dtype_size,
53 unsigned dtype_sign);
54 static herr_t H5Z__scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist, H5T_t *type,
55 enum H5Z_scaleoffset_t scale_type, unsigned cd_values[],
56 int need_convert);
57 static herr_t H5Z__set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
58 static size_t H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
59 size_t nbytes, size_t *buf_size, void **buf);
60 static void H5Z__scaleoffset_convert(void *buf, unsigned d_nelmts, unsigned dtype_size);
61 static H5_ATTR_CONST unsigned H5Z__scaleoffset_log2(unsigned long long num);
62 static void H5Z__scaleoffset_precompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
63 unsigned filavail, const unsigned cd_values[], uint32_t *minbits,
64 unsigned long long *minval);
65 static void H5Z__scaleoffset_postdecompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
66 unsigned filavail, const unsigned cd_values[], uint32_t minbits,
67 unsigned long long minval);
68 static herr_t H5Z__scaleoffset_precompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
69 unsigned filavail, const unsigned cd_values[],
70 uint32_t *minbits, unsigned long long *minval, double D_val);
71 static herr_t H5Z__scaleoffset_postdecompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
72 unsigned filavail, const unsigned cd_values[],
73 uint32_t minbits, unsigned long long minval, double D_val);
74 static void H5Z__scaleoffset_next_byte(size_t *j, unsigned *buf_len);
75 static void H5Z__scaleoffset_decompress_one_byte(unsigned char *data, size_t data_offset, unsigned k,
76 unsigned begin_i, const unsigned char *buffer, size_t *j,
77 unsigned *buf_len, parms_atomic p, unsigned dtype_len);
78 static void H5Z__scaleoffset_compress_one_byte(const unsigned char *data, size_t data_offset, unsigned k,
79 unsigned begin_i, unsigned char *buffer, size_t *j,
80 unsigned *buf_len, parms_atomic p, unsigned dtype_len);
81 static void H5Z__scaleoffset_decompress_one_atomic(unsigned char *data, size_t data_offset,
82 unsigned char *buffer, size_t *j, unsigned *buf_len,
83 parms_atomic p);
84 static void H5Z__scaleoffset_compress_one_atomic(unsigned char *data, size_t data_offset,
85 unsigned char *buffer, size_t *j, unsigned *buf_len,
86 parms_atomic p);
87 static void H5Z__scaleoffset_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
88 parms_atomic p);
89 static void H5Z__scaleoffset_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
90 size_t buffer_size, parms_atomic p);
91
92 /* This message derives from H5Z */
93 H5Z_class2_t H5Z_SCALEOFFSET[1] = {{
94 H5Z_CLASS_T_VERS, /* H5Z_class_t version */
95 H5Z_FILTER_SCALEOFFSET, /* Filter id number */
96 1, /* Assume encoder present: check before registering */
97 1, /* decoder_present flag (set to true) */
98 "scaleoffset", /* Filter name for debugging */
99 H5Z__can_apply_scaleoffset, /* The "can apply" callback */
100 H5Z__set_local_scaleoffset, /* The "set local" callback */
101 H5Z__filter_scaleoffset, /* The actual filter function */
102 }};
103
104 /* Local macros */
105 #define H5Z_SCALEOFFSET_TOTAL_NPARMS 20 /* Total number of parameters for filter */
106 #define H5Z_SCALEOFFSET_PARM_SCALETYPE 0 /* "User" parameter for scale type */
107 #define H5Z_SCALEOFFSET_PARM_SCALEFACTOR 1 /* "User" parameter for scale factor */
108 #define H5Z_SCALEOFFSET_PARM_NELMTS 2 /* "Local" parameter for number of elements in the chunk */
109 #define H5Z_SCALEOFFSET_PARM_CLASS 3 /* "Local" parameter for datatype class */
110 #define H5Z_SCALEOFFSET_PARM_SIZE 4 /* "Local" parameter for datatype size */
111 #define H5Z_SCALEOFFSET_PARM_SIGN 5 /* "Local" parameter for integer datatype sign */
112 #define H5Z_SCALEOFFSET_PARM_ORDER 6 /* "Local" parameter for datatype byte order */
113 #define H5Z_SCALEOFFSET_PARM_FILAVAIL 7 /* "Local" parameter for dataset fill value existence */
114 #define H5Z_SCALEOFFSET_PARM_FILVAL 8 /* "Local" parameter for start location to store dataset fill value */
115
116 #define H5Z_SCALEOFFSET_CLS_INTEGER 0 /* Integer (datatype class) */
117 #define H5Z_SCALEOFFSET_CLS_FLOAT 1 /* Floatig-point (datatype class) */
118
119 #define H5Z_SCALEOFFSET_SGN_NONE 0 /* Unsigned integer type */
120 #define H5Z_SCALEOFFSET_SGN_2 1 /* Two's complement signed integer type */
121
122 #define H5Z_SCALEOFFSET_ORDER_LE 0 /* Little endian (datatype byte order) */
123 #define H5Z_SCALEOFFSET_ORDER_BE 1 /* Big endian (datatype byte order) */
124
125 #define H5Z_SCALEOFFSET_FILL_UNDEFINED 0 /* Fill value is not defined */
126 #define H5Z_SCALEOFFSET_FILL_DEFINED 1 /* Fill value is defined */
127
128 /* Store fill value in cd_values[] */
129 #define H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
130 { \
131 unsigned _i = H5Z_SCALEOFFSET_PARM_FILVAL; /* index into cd_values */ \
132 uint32_t _cd_value; /* Current cd_value */ \
133 char * _fv_p; /* Pointer to current byte in fill_val */ \
134 size_t _copy_size = 4; /* # of bytes to copy this iteration */ \
135 size_t _size_rem = sizeof(type); /* # of bytes left to copy to cd_values */ \
136 \
137 /* Store the fill value as the last entry in cd_values[] \
138 * Store byte by byte from least significant byte to most significant byte \
139 * Plenty of space left for the fill value (from index 8 to 19) \
140 * H5O_pline_encode will byte-swap each individual cd value, but we still \
141 * need to swap the cd values as a whole if we are on a BE machine. Note \
142 * that we need to make sure to put the data only in the lowest 4 bytes of \
143 * each, if sizeof(unsigned) > 4. \
144 */ \
145 if (H5T_native_order_g == H5T_ORDER_LE) { \
146 _fv_p = (char *)&(fill_val); \
147 /* Copy 4 bytes at a time to each cd value */ \
148 do { \
149 if (_size_rem < 4) { \
150 /* Amount left to copy is smaller than a cd_value, adjust copy \
151 * size and initialize cd_value as it will not be fully \
152 * overwritten */ \
153 _copy_size = _size_rem; \
154 _cd_value = (uint32_t)0; \
155 } /* end if */ \
156 \
157 /* Copy the value */ \
158 H5MM_memcpy(&_cd_value, _fv_p, _copy_size); \
159 (cd_values)[_i] = (unsigned)_cd_value; \
160 \
161 /* Next field */ \
162 _i++; \
163 _fv_p += _copy_size; \
164 _size_rem -= _copy_size; \
165 } while (_size_rem); \
166 } /* end if */ \
167 else { \
168 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
169 \
170 /* Copy 4 bytes at a time to each cd value, but start at the end \
171 * (highest address) of fill_val */ \
172 _fv_p = ((char *)&(fill_val)) + sizeof(type) - MIN(4, _size_rem); \
173 while (_size_rem >= 4) { \
174 /* Copy the value */ \
175 H5MM_memcpy(&_cd_value, _fv_p, _copy_size); \
176 (cd_values)[_i] = (unsigned)_cd_value; \
177 \
178 /* Next field */ \
179 _i++; \
180 _size_rem -= 4; \
181 if (_size_rem >= 4) \
182 _fv_p -= 4; \
183 else \
184 _fv_p -= _size_rem; \
185 } /* end while */ \
186 \
187 HDassert(_fv_p == (char *)&(fill_val)); \
188 if (_size_rem) { \
189 /* Amount left to copy is smaller than a cd_value, initialize \
190 * _cd_value as it will not be fully overwritten and copy to the end \
191 * of _cd value as it is BE. */ \
192 _cd_value = (uint32_t)0; \
193 H5MM_memcpy((char *)&_cd_value + 4 - _size_rem, _fv_p, _size_rem); \
194 (cd_values)[_i] = (unsigned)_cd_value; \
195 } /* end if */ \
196 } /* end else */ \
197 }
198
199 /* Set the fill value parameter in cd_values[] for unsigned integer type */
200 #define H5Z_scaleoffset_set_filval_1(type, dcpl_plist, dt, cd_values, need_convert) \
201 do { \
202 type fill_val; \
203 \
204 /* Get dataset fill value */ \
205 if (H5P_get_fill_value(dcpl_plist, dt, &fill_val) < 0) \
206 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
207 \
208 if (need_convert) \
209 H5Z__scaleoffset_convert(&fill_val, 1, sizeof(type)); \
210 \
211 H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
212 } while (0)
213
214 /* Set the fill value parameter in cd_values[] for signed integer type */
215 #define H5Z_scaleoffset_set_filval_2(type, dcpl_plist, dt, cd_values, need_convert) \
216 do { \
217 type fill_val; \
218 \
219 /* Get dataset fill value */ \
220 if (H5P_get_fill_value(dcpl_plist, dt, &fill_val) < 0) \
221 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
222 \
223 if (need_convert) \
224 H5Z__scaleoffset_convert(&fill_val, 1, sizeof(type)); \
225 \
226 H5Z_scaleoffset_save_filval(unsigned type, cd_values, fill_val) \
227 } while (0)
228
229 /* Set the fill value parameter in cd_values[] for character integer type */
230 #define H5Z_scaleoffset_set_filval_3(type, dcpl_plist, dt, cd_values, need_convert) \
231 do { \
232 type fill_val; \
233 \
234 /* Get dataset fill value */ \
235 if (H5P_get_fill_value(dcpl_plist, dt, &fill_val) < 0) \
236 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
237 \
238 /* Store the fill value as the last entry in cd_values[] */ \
239 (cd_values)[H5Z_SCALEOFFSET_PARM_FILVAL] = (unsigned)((unsigned char)fill_val); \
240 } while (0)
241
242 /* Set the fill value parameter in cd_values[] for floating-point type */
243 #define H5Z_scaleoffset_set_filval_4(type, dcpl_plist, dt, cd_values, need_convert) \
244 do { \
245 type fill_val; \
246 \
247 /* Get dataset fill value */ \
248 if (H5P_get_fill_value(dcpl_plist, dt, &fill_val) < 0) \
249 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
250 \
251 if (need_convert) \
252 H5Z__scaleoffset_convert(&fill_val, 1, sizeof(type)); \
253 \
254 H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
255 } while (0)
256
257 /* Get the fill value for integer type */
258 #define H5Z_scaleoffset_get_filval_1(type, cd_values, fill_val) \
259 { \
260 unsigned _i = H5Z_SCALEOFFSET_PARM_FILVAL; /* index into cd_values */ \
261 uint32_t _cd_value; /* Current cd_value */ \
262 char * _fv_p; /* Pointer to current byte in fill_val */ \
263 size_t _copy_size = 4; /* # of bytes to copy this iteration */ \
264 size_t _size_rem = sizeof(type); /* # of bytes left to copy to filval */ \
265 \
266 /* Retrieve the fill value from the last entry in cd_values[] \
267 * Store byte by byte from least significant byte to most significant byte \
268 * Plenty of space left for the fill value (from index 8 to 19) \
269 * H5O_pline_encode will byte-swap each individual cd value, but we still \
270 * need to swap the cd values as a whole if we are on a BE machine. Note \
271 * that we need to make sure to put the data only in the lowest 4 bytes of \
272 * each, if sizeof(unsigned) > 4. \
273 */ \
274 if (H5T_native_order_g == H5T_ORDER_LE) { \
275 _fv_p = (char *)&(fill_val); \
276 /* Copy 4 bytes at a time to each cd value */ \
277 do { \
278 if (_size_rem < 4) \
279 /* Amount left to copy is smaller than a cd_value, adjust copy \
280 * size and initialize cd_value as it will not be fully \
281 * overwritten */ \
282 _copy_size = _size_rem; \
283 \
284 /* Copy the value */ \
285 _cd_value = (uint32_t)(cd_values)[_i]; \
286 H5MM_memcpy(_fv_p, &_cd_value, _copy_size); \
287 \
288 /* Next field */ \
289 _i++; \
290 _fv_p += _copy_size; \
291 _size_rem -= _copy_size; \
292 } while (_size_rem); \
293 } /* end if */ \
294 else { \
295 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
296 \
297 /* Copy 4 bytes at a time to each cd value, but start at the end \
298 * (highest address) of fill_val */ \
299 _fv_p = ((char *)&(fill_val)) + sizeof(type) - MIN(4, _size_rem); \
300 while (_size_rem >= 4) { \
301 /* Copy the value */ \
302 _cd_value = (uint32_t)(cd_values)[_i]; \
303 H5MM_memcpy(_fv_p, &_cd_value, _copy_size); \
304 \
305 /* Next field */ \
306 _i++; \
307 _size_rem -= 4; \
308 if (_size_rem >= 4) \
309 _fv_p -= 4; \
310 else \
311 _fv_p -= _size_rem; \
312 } /* end while */ \
313 \
314 HDassert(_fv_p == (char *)&(fill_val)); \
315 if (_size_rem) { \
316 /* Amount left to copy is smaller than a cd_value, initialize \
317 * _cd_value as it will not be fully overwritten and copy to the end \
318 * of _cd value as it is BE. */ \
319 _cd_value = (uint32_t)(cd_values)[_i]; \
320 H5MM_memcpy(_fv_p, (char *)&_cd_value + 4 - _size_rem, _size_rem); \
321 } /* end if */ \
322 } /* end else */ \
323 }
324
325 /* Get the fill value for floating-point type */
326 #define H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
327 { \
328 if (sizeof(type) <= sizeof(long long)) \
329 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) else HGOTO_ERROR( \
330 H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
331 }
332
333 /* Find maximum and minimum values of a buffer with fill value defined for integer type */
334 #define H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min) \
335 { \
336 i = 0; \
337 while (i < d_nelmts && buf[i] == filval) \
338 i++; \
339 if (i < d_nelmts) \
340 min = max = buf[i]; \
341 for (; i < d_nelmts; i++) { \
342 if (buf[i] == filval) \
343 continue; /* ignore fill value */ \
344 if (buf[i] > max) \
345 max = buf[i]; \
346 if (buf[i] < min) \
347 min = buf[i]; \
348 } \
349 }
350
351 /* Find maximum and minimum values of a buffer with fill value undefined */
352 #define H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
353 { \
354 min = max = buf[0]; \
355 for (i = 0; i < d_nelmts; i++) { \
356 if (buf[i] > max) \
357 max = buf[i]; \
358 if (buf[i] < min) \
359 min = buf[i]; \
360 } \
361 }
362
363 /* Find maximum and minimum values of a buffer with fill value defined for floating-point type */
364 #define H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \
365 { \
366 i = 0; \
367 while (i < d_nelmts && HDfabs(buf[i] - filval) < HDpow(10.0, -D_val)) \
368 i++; \
369 if (i < d_nelmts) \
370 min = max = buf[i]; \
371 for (; i < d_nelmts; i++) { \
372 if (HDfabs(buf[i] - filval) < HDpow(10.0, -D_val)) \
373 continue; /* ignore fill value */ \
374 if (buf[i] > max) \
375 max = buf[i]; \
376 if (buf[i] < min) \
377 min = buf[i]; \
378 } \
379 }
380
381 /* Find minimum value of a buffer with fill value defined for integer type */
382 #define H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval, min) \
383 { \
384 i = 0; \
385 while (i < d_nelmts && buf[i] == filval) \
386 i++; \
387 if (i < d_nelmts) \
388 min = buf[i]; \
389 for (; i < d_nelmts; i++) { \
390 if (buf[i] == filval) \
391 continue; /* ignore fill value */ \
392 if (buf[i] < min) \
393 min = buf[i]; \
394 } \
395 }
396
397 /* Find minimum value of a buffer with fill value undefined */
398 #define H5Z_scaleoffset_min_2(i, d_nelmts, buf, min) \
399 { \
400 min = buf[0]; \
401 for (i = 0; i < d_nelmts; i++) \
402 if (buf[i] < min) \
403 min = buf[i]; \
404 }
405
406 /* Check and handle special situation for unsigned integer type */
407 #define H5Z_scaleoffset_check_1(type, max, min, minbits) \
408 { \
409 if (max - min > (type)(~(type)0 - 2)) { \
410 *minbits = sizeof(type) * 8; \
411 return; \
412 } \
413 }
414
415 /* Check and handle special situation for signed integer type */
416 #define H5Z_scaleoffset_check_2(type, max, min, minbits) \
417 { \
418 if ((unsigned type)(max - min) > (unsigned type)(~(unsigned type)0 - 2)) { \
419 *minbits = sizeof(type) * 8; \
420 return; \
421 } \
422 }
423
424 /* Check and handle special situation for floating-point type */
425 #define H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) \
426 { \
427 if (sizeof(type) == sizeof(int)) { \
428 if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \
429 pow_fun(2.0f, (type)(sizeof(int) * 8 - 1))) { \
430 *minbits = sizeof(int) * 8; \
431 goto done; \
432 } \
433 } \
434 else if (sizeof(type) == sizeof(long)) { \
435 if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \
436 pow_fun(2.0f, (type)(sizeof(long) * 8 - 1))) { \
437 *minbits = sizeof(long) * 8; \
438 goto done; \
439 } \
440 } \
441 else if (sizeof(type) == sizeof(long long)) { \
442 if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \
443 pow_fun(2.0f, (type)(sizeof(long long) * 8 - 1))) { \
444 *minbits = sizeof(long long) * 8; \
445 goto done; \
446 } \
447 } \
448 else \
449 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
450 }
451
452 /* Precompress for unsigned integer type */
453 #define H5Z_scaleoffset_precompress_1(type, data, d_nelmts, filavail, cd_values, minbits, minval) \
454 do { \
455 type * buf = (type *)data, min = 0, max = 0, span, filval = 0; \
456 unsigned i; \
457 \
458 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
459 H5Z_scaleoffset_get_filval_1(type, cd_values, \
460 filval) if (*minbits == H5Z_SO_INT_MINBITS_DEFAULT) \
461 { /* minbits not set yet, calculate max, min, and minbits */ \
462 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min) \
463 H5Z_scaleoffset_check_1(type, max, min, minbits) span = (type)(max - min + 1); \
464 *minbits = H5Z__scaleoffset_log2((unsigned long long)(span + 1)); \
465 } \
466 else /* minbits already set, only calculate min */ \
467 H5Z_scaleoffset_min_1( \
468 i, d_nelmts, buf, filval, \
469 min) if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
470 for (i = 0; i < d_nelmts; i++) buf[i] = \
471 (type)((buf[i] == filval) ? (((type)1 << *minbits) - 1) : (buf[i] - min)); \
472 } \
473 else { /* fill value undefined */ \
474 if (*minbits == \
475 H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */ \
476 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
477 H5Z_scaleoffset_check_1(type, max, min, minbits) span = (type)(max - min + 1); \
478 *minbits = H5Z__scaleoffset_log2((unsigned long long)span); \
479 } \
480 else /* minbits already set, only calculate min */ \
481 H5Z_scaleoffset_min_2( \
482 i, d_nelmts, buf, \
483 min) if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
484 for (i = 0; i < d_nelmts; i++) buf[i] = (type)(buf[i] - min); \
485 } \
486 *minval = min; \
487 } while (0)
488
489 /* Precompress for signed integer type */
490 #define H5Z_scaleoffset_precompress_2(type, data, d_nelmts, filavail, cd_values, minbits, minval) \
491 do { \
492 type * buf = (type *)data, min = 0, max = 0, filval = 0; \
493 unsigned type span; \
494 unsigned i; \
495 \
496 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
497 H5Z_scaleoffset_get_filval_1(type, cd_values, \
498 filval) if (*minbits == H5Z_SO_INT_MINBITS_DEFAULT) \
499 { /* minbits not set yet, calculate max, min, and minbits */ \
500 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min) \
501 H5Z_scaleoffset_check_2(type, max, min, minbits) span = (unsigned type)(max - min + 1); \
502 *minbits = H5Z__scaleoffset_log2((unsigned long long)(span + 1)); \
503 } \
504 else /* minbits already set, only calculate min */ \
505 H5Z_scaleoffset_min_1( \
506 i, d_nelmts, buf, filval, \
507 min) if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
508 for (i = 0; i < d_nelmts; i++) buf[i] = (type)( \
509 (buf[i] == filval) ? (type)(((unsigned type)1 << *minbits) - 1) : (buf[i] - min)); \
510 } \
511 else { /* fill value undefined */ \
512 if (*minbits == \
513 H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */ \
514 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
515 H5Z_scaleoffset_check_2(type, max, min, minbits) span = (unsigned type)(max - min + 1); \
516 *minbits = H5Z__scaleoffset_log2((unsigned long long)span); \
517 } \
518 else /* minbits already set, only calculate min */ \
519 H5Z_scaleoffset_min_2( \
520 i, d_nelmts, buf, \
521 min) if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
522 for (i = 0; i < d_nelmts; i++) buf[i] = (type)(buf[i] - min); \
523 } \
524 *minval = (unsigned long long)min; \
525 } while (0)
526
527 /* Modify values of data in precompression if fill value defined for floating-point type */
528 #define H5Z_scaleoffset_modify_1(i, type, pow_fun, abs_fun, lround_fun, llround_fun, buf, d_nelmts, filval, \
529 minbits, min, D_val) \
530 { \
531 if (sizeof(type) == sizeof(int)) \
532 for (i = 0; i < d_nelmts; i++) { \
533 if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \
534 *(int *)((void *)&buf[i]) = (int)(((unsigned int)1 << *minbits) - 1); \
535 else \
536 *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \
537 min * pow_fun(10.0f, (type)D_val)); \
538 } \
539 else if (sizeof(type) == sizeof(long)) \
540 for (i = 0; i < d_nelmts; i++) { \
541 if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \
542 *(long *)((void *)&buf[i]) = (long)(((unsigned long)1 << *minbits) - 1); \
543 else \
544 *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \
545 min * pow_fun(10.0f, (type)D_val)); \
546 } \
547 else if (sizeof(type) == sizeof(long long)) \
548 for (i = 0; i < d_nelmts; i++) { \
549 if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \
550 *(long long *)((void *)&buf[i]) = (long long)(((unsigned long long)1 << *minbits) - 1); \
551 else \
552 *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \
553 min * pow_fun(10.0f, (type)D_val)); \
554 } \
555 else \
556 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
557 }
558
559 /* Modify values of data in precompression if fill value undefined for floating-point type */
560 #define H5Z_scaleoffset_modify_2(i, type, pow_fun, lround_fun, llround_fun, buf, d_nelmts, min, D_val) \
561 { \
562 if (sizeof(type) == sizeof(int)) \
563 for (i = 0; i < d_nelmts; i++) \
564 *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \
565 min * pow_fun(10.0f, (type)D_val)); \
566 else if (sizeof(type) == sizeof(long)) \
567 for (i = 0; i < d_nelmts; i++) \
568 *(long *)((void *)&buf[i]) = \
569 lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)); \
570 else if (sizeof(type) == sizeof(long long)) \
571 for (i = 0; i < d_nelmts; i++) \
572 *(long long *)((void *)&buf[i]) = \
573 llround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)); \
574 else \
575 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
576 }
577
578 /* Save the minimum value for floating-point type */
579 #define H5Z_scaleoffset_save_min(i, type, minval, min) \
580 { \
581 if (sizeof(type) <= sizeof(long long)) \
582 /* Save min value to corresponding position \
583 * byte-order will be swapped as appropriate, but be sure to \
584 * account for offset in BE if sizes differ \
585 */ \
586 if (H5T_native_order_g == H5T_ORDER_LE) \
587 H5MM_memcpy(minval, &min, sizeof(type)); \
588 else { \
589 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
590 H5MM_memcpy(((char *)minval) + (sizeof(long long) - sizeof(type)), &min, sizeof(type)); \
591 } /* end else */ \
592 else \
593 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
594 }
595
596 /* Precompress for floating-point type using variable-minimum-bits method */
597 #define H5Z_scaleoffset_precompress_3(type, pow_fun, abs_fun, round_fun, lround_fun, llround_fun, data, \
598 d_nelmts, filavail, cd_values, minbits, minval, D_val) \
599 do { \
600 type * buf = (type *)data, min = 0, max = 0, filval = 0; \
601 unsigned long long span; \
602 unsigned i; \
603 \
604 *minval = 0; \
605 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
606 H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
607 H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \
608 H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \
609 (unsigned long long)(llround_fun(max * pow_fun(10.0f, (type)D_val) - \
610 min * pow_fun(10.0f, (type)D_val)) + \
611 1); \
612 *minbits = H5Z__scaleoffset_log2(span + 1); \
613 if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
614 H5Z_scaleoffset_modify_1(i, type, pow_fun, abs_fun, lround_fun, llround_fun, buf, d_nelmts, \
615 filval, minbits, min, D_val) \
616 } \
617 else { /* fill value undefined */ \
618 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
619 H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \
620 (unsigned long long)(llround_fun(max * pow_fun(10.0f, (type)D_val) - \
621 min * pow_fun(10.0f, (type)D_val)) + \
622 1); \
623 *minbits = H5Z__scaleoffset_log2(span); \
624 if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
625 H5Z_scaleoffset_modify_2(i, type, pow_fun, lround_fun, llround_fun, buf, d_nelmts, min, \
626 D_val) \
627 } \
628 H5Z_scaleoffset_save_min(i, type, minval, min) \
629 } while (0)
630
631 /* Postdecompress for unsigned integer type */
632 #define H5Z_scaleoffset_postdecompress_1(type, data, d_nelmts, filavail, cd_values, minbits, minval) \
633 do { \
634 type * buf = (type *)data, filval = 0; \
635 unsigned i; \
636 \
637 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
638 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) for (i = 0; i < d_nelmts; i++) buf[i] = \
639 (type)((buf[i] == (((type)1 << minbits) - 1)) ? filval : (buf[i] + minval)); \
640 } \
641 else /* fill value undefined */ \
642 for (i = 0; i < d_nelmts; i++) \
643 buf[i] = (type)(buf[i] + (type)(minval)); \
644 } while (0)
645
646 /* Postdecompress for signed integer type */
647 #define H5Z_scaleoffset_postdecompress_2(type, data, d_nelmts, filavail, cd_values, minbits, minval) \
648 do { \
649 type * buf = (type *)data, filval = 0; \
650 unsigned i; \
651 \
652 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
653 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) for (i = 0; i < d_nelmts; i++) buf[i] = \
654 (type)(((unsigned type)buf[i] == (((unsigned type)1 << minbits) - 1)) ? filval \
655 : (buf[i] + minval)); \
656 } \
657 else /* fill value undefined */ \
658 for (i = 0; i < d_nelmts; i++) \
659 buf[i] = (type)(buf[i] + (type)(minval)); \
660 } while (0)
661
662 /* Retrive minimum value of floating-point type */
663 #define H5Z_scaleoffset_get_min(type, minval, min) \
664 { \
665 if (sizeof(type) <= sizeof(long long)) \
666 /* retrieve min value from corresponding position \
667 * byte-order has already been swapped as appropriate, but be sure to \
668 * account for offset in BE if sizes differ \
669 */ \
670 if (H5T_native_order_g == H5T_ORDER_LE) \
671 H5MM_memcpy(&min, &minval, sizeof(type)); \
672 else { \
673 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
674 H5MM_memcpy(&min, ((char *)&minval) + (sizeof(long long) - sizeof(type)), sizeof(type)); \
675 } /* end else */ \
676 else \
677 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
678 }
679
680 /* Modify values of data in postdecompression if fill value defined for floating-point type */
681 #define H5Z_scaleoffset_modify_3(i, type, pow_fun, buf, d_nelmts, filval, minbits, min, D_val) \
682 { \
683 if (sizeof(type) == sizeof(int)) \
684 for (i = 0; i < d_nelmts; i++) \
685 buf[i] = \
686 (type)((*(int *)((void *)&buf[i]) == (int)(((unsigned int)1 << minbits) - 1)) \
687 ? filval \
688 : (type)(*(int *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
689 else if (sizeof(type) == sizeof(long)) \
690 for (i = 0; i < d_nelmts; i++) \
691 buf[i] = \
692 (type)((*(long *)((void *)&buf[i]) == (long)(((unsigned long)1 << minbits) - 1)) \
693 ? filval \
694 : (type)(*(long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
695 else if (sizeof(type) == sizeof(long long)) \
696 for (i = 0; i < d_nelmts; i++) \
697 buf[i] = (type)( \
698 (*(long long *)((void *)&buf[i]) == (long long)(((unsigned long long)1 << minbits) - 1)) \
699 ? filval \
700 : (type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
701 else \
702 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
703 }
704
705 /* Modify values of data in postdecompression if fill value undefined for floating-point type */
706 #define H5Z_scaleoffset_modify_4(i, type, pow_fun, buf, d_nelmts, min, D_val) \
707 { \
708 if (sizeof(type) == sizeof(int)) \
709 for (i = 0; i < d_nelmts; i++) \
710 buf[i] = ((type)(*(int *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
711 else if (sizeof(type) == sizeof(long)) \
712 for (i = 0; i < d_nelmts; i++) \
713 buf[i] = ((type)(*(long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
714 else if (sizeof(type) == sizeof(long long)) \
715 for (i = 0; i < d_nelmts; i++) \
716 buf[i] = ((type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \
717 else \
718 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
719 }
720
721 /* Postdecompress for floating-point type using variable-minimum-bits method */
722 #define H5Z_scaleoffset_postdecompress_3(type, pow_fun, data, d_nelmts, filavail, cd_values, minbits, \
723 minval, D_val) \
724 do { \
725 type * buf = (type *)data, filval = 0, min = 0; \
726 unsigned i; \
727 \
728 H5Z_scaleoffset_get_min(type, minval, min) \
729 \
730 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) \
731 { /* fill value defined */ \
732 H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
733 H5Z_scaleoffset_modify_3(i, type, pow_fun, buf, d_nelmts, filval, minbits, min, D_val) \
734 } \
735 else /* fill value undefined */ \
736 H5Z_scaleoffset_modify_4(i, type, pow_fun, buf, d_nelmts, min, D_val) \
737 } while (0)
738
739 /*-------------------------------------------------------------------------
740 * Function: H5Z__can_apply_scaleoffset
741 *
742 * Purpose: Check the parameters for scaleoffset compression for
743 * validity and whether they fit a particular dataset.
744 *
745 * Return: Success: Non-negative
746 * Failure: Negative
747 *
748 * Programmer: Xiaowen Wu
749 * Friday, February 4, 2005
750 *
751 *-------------------------------------------------------------------------
752 */
753 static htri_t
H5Z__can_apply_scaleoffset(hid_t H5_ATTR_UNUSED dcpl_id,hid_t type_id,hid_t H5_ATTR_UNUSED space_id)754 H5Z__can_apply_scaleoffset(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id)
755 {
756 const H5T_t *type; /* Datatype */
757 H5T_class_t dtype_class; /* Datatype's class */
758 H5T_order_t dtype_order; /* Datatype's endianness order */
759 htri_t ret_value = TRUE; /* Return value */
760
761 FUNC_ENTER_STATIC
762
763 /* Get datatype */
764 if (NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
765 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
766
767 /* Get datatype's class, for checking the "datatype class" */
768 if ((dtype_class = H5T_get_class(type, TRUE)) == H5T_NO_CLASS)
769 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype class")
770
771 /* Get datatype's size, for checking the "datatype size" */
772 if (H5T_get_size(type) == 0)
773 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
774
775 if (dtype_class == H5T_INTEGER || dtype_class == H5T_FLOAT) {
776 /* Get datatype's endianness order */
777 if ((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
778 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order")
779
780 /* Range check datatype's endianness order */
781 if (dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
782 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "bad datatype endianness order")
783 }
784 else
785 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "datatype class not supported by scaleoffset")
786
787 done:
788 FUNC_LEAVE_NOAPI(ret_value)
789 } /* end H5Z__can_apply_scaleoffset() */
790
791 /*-------------------------------------------------------------------------
792 * Function: H5Z__scaleoffset_get_type
793 *
794 * Purpose: Get the specific integer type based on datatype size and sign
795 * or floating-point type based on size
796 *
797 * Return: Success: id number of integer type
798 * Failure: 0
799 *
800 * Programmer: Xiaowen Wu
801 * Wednesday, April 13, 2005
802 *
803 *-------------------------------------------------------------------------
804 */
805 static enum H5Z_scaleoffset_t
H5Z__scaleoffset_get_type(unsigned dtype_class,unsigned dtype_size,unsigned dtype_sign)806 H5Z__scaleoffset_get_type(unsigned dtype_class, unsigned dtype_size, unsigned dtype_sign)
807 {
808 enum H5Z_scaleoffset_t type = t_bad; /* integer type */
809 enum H5Z_scaleoffset_t ret_value = t_bad; /* Return value */
810
811 FUNC_ENTER_STATIC
812
813 if (dtype_class == H5Z_SCALEOFFSET_CLS_INTEGER) {
814 if (dtype_sign == H5Z_SCALEOFFSET_SGN_NONE) { /* unsigned integer */
815 if (dtype_size == sizeof(unsigned char))
816 type = t_uchar;
817 else if (dtype_size == sizeof(unsigned short))
818 type = t_ushort;
819 else if (dtype_size == sizeof(unsigned int))
820 type = t_uint;
821 else if (dtype_size == sizeof(unsigned long))
822 type = t_ulong;
823 #if H5_SIZEOF_LONG != H5_SIZEOF_LONG_LONG
824 else if (dtype_size == sizeof(unsigned long long))
825 type = t_ulong_long;
826 #endif /* H5_SIZEOF_LONG != H5_SIZEOF_LONG_LONG */
827 else
828 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
829 }
830
831 if (dtype_sign == H5Z_SCALEOFFSET_SGN_2) { /* signed integer */
832 if (dtype_size == sizeof(signed char))
833 type = t_schar;
834 else if (dtype_size == sizeof(short))
835 type = t_short;
836 else if (dtype_size == sizeof(int))
837 type = t_int;
838 else if (dtype_size == sizeof(long))
839 type = t_long;
840 #if H5_SIZEOF_LONG != H5_SIZEOF_LONG_LONG
841 else if (dtype_size == sizeof(long long))
842 type = t_long_long;
843 #endif /* H5_SIZEOF_LONG != H5_SIZEOF_LONG_LONG */
844 else
845 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
846 }
847 }
848
849 if (dtype_class == H5Z_SCALEOFFSET_CLS_FLOAT) {
850 if (dtype_size == sizeof(float))
851 type = t_float;
852 else if (dtype_size == sizeof(double))
853 type = t_double;
854 else
855 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
856 }
857
858 /* Set return value */
859 ret_value = type;
860
861 done:
862 FUNC_LEAVE_NOAPI(ret_value)
863 }
864
865 /*-------------------------------------------------------------------------
866 * Function: H5Z__scaleoffset_set_parms_fillval
867 *
868 * Purpose: Get the fill value of the dataset and store in cd_values[]
869 *
870 * Return: Success: Non-negative
871 * Failure: Negative
872 *
873 * Programmer: Xiaowen Wu
874 * Monday, March 7, 2005
875 *
876 *-------------------------------------------------------------------------
877 */
878 static herr_t
H5Z__scaleoffset_set_parms_fillval(H5P_genplist_t * dcpl_plist,H5T_t * type,enum H5Z_scaleoffset_t scale_type,unsigned cd_values[],int need_convert)879 H5Z__scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist, H5T_t *type, enum H5Z_scaleoffset_t scale_type,
880 unsigned cd_values[], int need_convert)
881 {
882 herr_t ret_value = SUCCEED; /* Return value */
883
884 FUNC_ENTER_STATIC
885
886 if (scale_type == t_uchar)
887 H5Z_scaleoffset_set_filval_3(unsigned char, dcpl_plist, type, cd_values, need_convert);
888 else if (scale_type == t_ushort)
889 H5Z_scaleoffset_set_filval_1(unsigned short, dcpl_plist, type, cd_values, need_convert);
890 else if (scale_type == t_uint)
891 H5Z_scaleoffset_set_filval_1(unsigned int, dcpl_plist, type, cd_values, need_convert);
892 else if (scale_type == t_ulong)
893 H5Z_scaleoffset_set_filval_1(unsigned long, dcpl_plist, type, cd_values, need_convert);
894 else if (scale_type == t_ulong_long)
895 H5Z_scaleoffset_set_filval_1(unsigned long long, dcpl_plist, type, cd_values, need_convert);
896 else if (scale_type == t_schar)
897 H5Z_scaleoffset_set_filval_3(signed char, dcpl_plist, type, cd_values, need_convertd);
898 else if (scale_type == t_short)
899 H5Z_scaleoffset_set_filval_2(short, dcpl_plist, type, cd_values, need_convert);
900 else if (scale_type == t_int)
901 H5Z_scaleoffset_set_filval_2(int, dcpl_plist, type, cd_values, need_convert);
902 else if (scale_type == t_long)
903 H5Z_scaleoffset_set_filval_2(long, dcpl_plist, type, cd_values, need_convert);
904 else if (scale_type == t_long_long)
905 H5Z_scaleoffset_set_filval_2(long long, dcpl_plist, type, cd_values, need_convert);
906 else if (scale_type == t_float)
907 H5Z_scaleoffset_set_filval_4(float, dcpl_plist, type, cd_values, need_convert);
908 else if (scale_type == t_double)
909 H5Z_scaleoffset_set_filval_4(double, dcpl_plist, type, cd_values, need_convert);
910
911 done:
912 FUNC_LEAVE_NOAPI(ret_value)
913 } /* end H5Z__scaleoffset_set_parms_fillval() */
914
915 /*-------------------------------------------------------------------------
916 * Function: H5Z__set_local_scaleoffset
917 *
918 * Purpose: Set the "local" dataset parameters for scaleoffset
919 * compression.
920 *
921 * Return: Success: Non-negative
922 * Failure: Negative
923 *
924 * Programmer: Xiaowen Wu
925 * Friday, February 4, 2005
926 *
927 *-------------------------------------------------------------------------
928 */
929 static herr_t
H5Z__set_local_scaleoffset(hid_t dcpl_id,hid_t type_id,hid_t space_id)930 H5Z__set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id)
931 {
932 H5P_genplist_t * dcpl_plist; /* Property list pointer */
933 H5T_t * type; /* Datatype */
934 const H5S_t * ds; /* Dataspace */
935 unsigned flags; /* Filter flags */
936 size_t cd_nelmts = H5Z_SCALEOFFSET_USER_NPARMS; /* Number of filter parameters */
937 unsigned cd_values[H5Z_SCALEOFFSET_TOTAL_NPARMS]; /* Filter parameters */
938 hssize_t npoints; /* Number of points in the dataspace */
939 H5T_class_t dtype_class; /* Datatype's class */
940 H5T_order_t dtype_order; /* Datatype's endianness order */
941 size_t dtype_size; /* Datatype's size (in bytes) */
942 H5T_sign_t dtype_sign; /* Datatype's sign */
943 enum H5Z_scaleoffset_t scale_type; /* Specific datatype */
944 H5D_fill_value_t status; /* Status of fill value in property list */
945 herr_t ret_value = SUCCEED; /* Return value */
946
947 FUNC_ENTER_STATIC
948
949 /* Get the plist structure */
950 if (NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
951 HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
952
953 /* Get datatype */
954 if (NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
955 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
956
957 /* Initialize the parameters to a known state */
958 HDmemset(cd_values, 0, sizeof(cd_values));
959
960 /* Get the filter's current parameters */
961 if (H5P_get_filter_by_id(dcpl_plist, H5Z_FILTER_SCALEOFFSET, &flags, &cd_nelmts, cd_values, (size_t)0,
962 NULL, NULL) < 0)
963 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get scaleoffset parameters")
964
965 /* Get dataspace */
966 if (NULL == (ds = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE)))
967 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace")
968
969 /* Get total number of elements in the chunk */
970 if ((npoints = H5S_GET_EXTENT_NPOINTS(ds)) < 0)
971 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")
972
973 /* Set "local" parameter for this dataset's number of elements */
974 H5_CHECKED_ASSIGN(cd_values[H5Z_SCALEOFFSET_PARM_NELMTS], unsigned, npoints, hssize_t);
975
976 /* Get datatype's class */
977 if ((dtype_class = H5T_get_class(type, TRUE)) == H5T_NO_CLASS)
978 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype class")
979
980 /* Set "local" parameter for datatype's class */
981 switch (dtype_class) {
982 case H5T_INTEGER:
983 cd_values[H5Z_SCALEOFFSET_PARM_CLASS] = H5Z_SCALEOFFSET_CLS_INTEGER;
984 break;
985
986 case H5T_FLOAT:
987 cd_values[H5Z_SCALEOFFSET_PARM_CLASS] = H5Z_SCALEOFFSET_CLS_FLOAT;
988 break;
989
990 case H5T_NO_CLASS:
991 case H5T_TIME:
992 case H5T_STRING:
993 case H5T_BITFIELD:
994 case H5T_OPAQUE:
995 case H5T_COMPOUND:
996 case H5T_REFERENCE:
997 case H5T_ENUM:
998 case H5T_VLEN:
999 case H5T_ARRAY:
1000 case H5T_NCLASSES:
1001 default:
1002 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "datatype class not supported by scaleoffset")
1003 } /* end switch */
1004
1005 /* Get datatype's size */
1006 if ((dtype_size = H5T_get_size(type)) == 0)
1007 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
1008
1009 /* Set "local" parameter for datatype size */
1010 H5_CHECK_OVERFLOW(dtype_size, size_t, unsigned);
1011 cd_values[H5Z_SCALEOFFSET_PARM_SIZE] = (unsigned)dtype_size;
1012
1013 if (dtype_class == H5T_INTEGER) {
1014 /* Get datatype's sign */
1015 if ((dtype_sign = H5T_get_sign(type)) == H5T_SGN_ERROR)
1016 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype sign")
1017
1018 /* Set "local" parameter for integer datatype sign */
1019 switch (dtype_sign) {
1020 case H5T_SGN_NONE:
1021 cd_values[H5Z_SCALEOFFSET_PARM_SIGN] = H5Z_SCALEOFFSET_SGN_NONE;
1022 break;
1023
1024 case H5T_SGN_2:
1025 cd_values[H5Z_SCALEOFFSET_PARM_SIGN] = H5Z_SCALEOFFSET_SGN_2;
1026 break;
1027
1028 case H5T_SGN_ERROR:
1029 case H5T_NSGN:
1030 default:
1031 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad integer sign")
1032 } /* end switch */
1033 } /* end if */
1034
1035 /* Get datatype's endianness order */
1036 if ((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
1037 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
1038
1039 /* Set "local" parameter for datatype endianness */
1040 switch (dtype_order) {
1041 case H5T_ORDER_LE: /* Little-endian byte order */
1042 cd_values[H5Z_SCALEOFFSET_PARM_ORDER] = H5Z_SCALEOFFSET_ORDER_LE;
1043 break;
1044
1045 case H5T_ORDER_BE: /* Big-endian byte order */
1046 cd_values[H5Z_SCALEOFFSET_PARM_ORDER] = H5Z_SCALEOFFSET_ORDER_BE;
1047 break;
1048
1049 case H5T_ORDER_ERROR:
1050 case H5T_ORDER_VAX:
1051 case H5T_ORDER_MIXED:
1052 case H5T_ORDER_NONE:
1053 default:
1054 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
1055 } /* end switch */
1056
1057 /* Check whether fill value is defined for dataset */
1058 if (H5P_fill_value_defined(dcpl_plist, &status) < 0)
1059 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to determine if fill value is defined")
1060
1061 /* Set local parameter for availability of fill value */
1062 if (status == H5D_FILL_VALUE_UNDEFINED)
1063 cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL] = H5Z_SCALEOFFSET_FILL_UNDEFINED;
1064 else {
1065 int need_convert = FALSE; /* Flag indicating conversion of byte order */
1066
1067 cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL] = H5Z_SCALEOFFSET_FILL_DEFINED;
1068
1069 /* Check if memory byte order matches dataset datatype byte order */
1070 if (H5T_native_order_g != dtype_order)
1071 need_convert = TRUE;
1072
1073 /* Before getting fill value, get its type */
1074 if ((scale_type = H5Z__scaleoffset_get_type(cd_values[H5Z_SCALEOFFSET_PARM_CLASS],
1075 cd_values[H5Z_SCALEOFFSET_PARM_SIZE],
1076 cd_values[H5Z_SCALEOFFSET_PARM_SIGN])) == 0)
1077 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot use C integer datatype for cast")
1078
1079 /* Get dataset fill value and store in cd_values[] */
1080 if (H5Z__scaleoffset_set_parms_fillval(dcpl_plist, type, scale_type, cd_values, need_convert) < 0)
1081 HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "unable to set fill value")
1082 } /* end else */
1083
1084 /* Modify the filter's parameters for this dataset */
1085 if (H5P_modify_filter(dcpl_plist, H5Z_FILTER_SCALEOFFSET, flags, (size_t)H5Z_SCALEOFFSET_TOTAL_NPARMS,
1086 cd_values) < 0)
1087 HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local scaleoffset parameters")
1088
1089 done:
1090 FUNC_LEAVE_NOAPI(ret_value)
1091 } /* end H5Z__set_local_scaleoffset() */
1092
1093 /*-------------------------------------------------------------------------
1094 * Function: H5Z__filter_scaleoffset
1095 *
1096 * Purpose: Implement an I/O filter for storing packed integer
1097 * data using scale and offset method.
1098 *
1099 * Return: Success: Size of buffer filtered
1100 * Failure: 0
1101 *
1102 * Programmer: Xiaowen Wu
1103 * Monday, February 7, 2005
1104 *
1105 *-------------------------------------------------------------------------
1106 */
1107 static size_t
H5Z__filter_scaleoffset(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],size_t nbytes,size_t * buf_size,void ** buf)1108 H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes,
1109 size_t *buf_size, void **buf)
1110 {
1111 size_t ret_value = 0; /* return value */
1112 size_t size_out = 0; /* size of output buffer */
1113 unsigned d_nelmts = 0; /* number of data elements in the chunk */
1114 unsigned dtype_class; /* datatype class */
1115 unsigned dtype_sign; /* integer datatype sign */
1116 unsigned filavail; /* flag indicating if fill value is defined or not */
1117 H5Z_SO_scale_type_t scale_type = H5Z_SO_FLOAT_DSCALE; /* scale type */
1118 int scale_factor = 0; /* scale factor */
1119 double D_val = 0.0; /* decimal scale factor */
1120 uint32_t minbits = 0; /* minimum number of bits to store values */
1121 unsigned long long minval = 0; /* minimum value of input buffer */
1122 enum H5Z_scaleoffset_t type; /* memory type corresponding to dataset datatype */
1123 int need_convert = FALSE; /* flag indicating conversion of byte order */
1124 unsigned char * outbuf = NULL; /* pointer to new output buffer */
1125 unsigned buf_offset = 21; /* buffer offset because of parameters stored in file */
1126 unsigned i; /* index */
1127 parms_atomic p; /* parameters needed for compress/decompress functions */
1128
1129 FUNC_ENTER_STATIC
1130
1131 /* check arguments */
1132 if (cd_nelmts != H5Z_SCALEOFFSET_TOTAL_NPARMS)
1133 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scaleoffset number of parameters")
1134
1135 /* Check if memory byte order matches dataset datatype byte order */
1136 switch (H5T_native_order_g) {
1137 case H5T_ORDER_LE: /* memory is little-endian byte order */
1138 if (cd_values[H5Z_SCALEOFFSET_PARM_ORDER] == H5Z_SCALEOFFSET_ORDER_BE)
1139 need_convert = TRUE;
1140 break;
1141
1142 case H5T_ORDER_BE: /* memory is big-endian byte order */
1143 if (cd_values[H5Z_SCALEOFFSET_PARM_ORDER] == H5Z_SCALEOFFSET_ORDER_LE)
1144 need_convert = TRUE;
1145 break;
1146
1147 case H5T_ORDER_ERROR:
1148 case H5T_ORDER_VAX:
1149 case H5T_ORDER_MIXED:
1150 case H5T_ORDER_NONE:
1151 default:
1152 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "bad H5T_NATIVE_INT endianness order")
1153 } /* end switch */
1154
1155 /* copy filter parameters to local variables */
1156 d_nelmts = cd_values[H5Z_SCALEOFFSET_PARM_NELMTS];
1157 dtype_class = cd_values[H5Z_SCALEOFFSET_PARM_CLASS];
1158 dtype_sign = cd_values[H5Z_SCALEOFFSET_PARM_SIGN];
1159 filavail = cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL];
1160 scale_factor = (int)cd_values[H5Z_SCALEOFFSET_PARM_SCALEFACTOR];
1161 scale_type = (H5Z_SO_scale_type_t)cd_values[H5Z_SCALEOFFSET_PARM_SCALETYPE];
1162
1163 /* check and assign proper values set by user to related parameters
1164 * scale type can be H5Z_SO_FLOAT_DSCALE (0), H5Z_SO_FLOAT_ESCALE (1) or H5Z_SO_INT (other)
1165 * H5Z_SO_FLOAT_DSCALE : floating-point type, variable-minimum-bits method,
1166 * scale factor is decimal scale factor
1167 * H5Z_SO_FLOAT_ESCALE : floating-point type, fixed-minimum-bits method,
1168 * scale factor is the fixed minimum number of bits
1169 * H5Z_SO_INT : integer type, scale_factor is minimum number of bits
1170 */
1171 if (dtype_class == H5Z_SCALEOFFSET_CLS_FLOAT) { /* floating-point type */
1172 if (scale_type != H5Z_SO_FLOAT_DSCALE && scale_type != H5Z_SO_FLOAT_ESCALE)
1173 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scale type")
1174 }
1175
1176 if (dtype_class == H5Z_SCALEOFFSET_CLS_INTEGER) { /* integer type */
1177 if (scale_type != H5Z_SO_INT)
1178 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scale type")
1179
1180 /* if scale_factor is less than 0 for integer, library will reset it to 0
1181 * in this case, library will calculate the minimum-bits
1182 */
1183 if (scale_factor < 0)
1184 scale_factor = 0;
1185 }
1186
1187 /* fixed-minimum-bits method is not implemented and is forbidden */
1188 if (scale_type == H5Z_SO_FLOAT_ESCALE)
1189 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "E-scaling method not supported")
1190
1191 if (scale_type == H5Z_SO_FLOAT_DSCALE) { /* floating-point type, variable-minimum-bits */
1192 D_val = (double)scale_factor;
1193 }
1194 else { /* integer type, or floating-point type with fixed-minimum-bits method */
1195 if (scale_factor > (int)(cd_values[H5Z_SCALEOFFSET_PARM_SIZE] * 8))
1196 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "minimum number of bits exceeds maximum")
1197
1198 /* no need to process data */
1199 if (scale_factor == (int)(cd_values[H5Z_SCALEOFFSET_PARM_SIZE] * 8)) {
1200 ret_value = *buf_size;
1201 goto done;
1202 }
1203 minbits = (uint32_t)scale_factor;
1204 }
1205
1206 /* prepare parameters to pass to compress/decompress functions */
1207 p.size = cd_values[H5Z_SCALEOFFSET_PARM_SIZE];
1208 p.mem_order = H5T_native_order_g;
1209
1210 /* input; decompress */
1211 if (flags & H5Z_FLAG_REVERSE) {
1212 /* retrieve values of minbits and minval from input compressed buffer
1213 * retrieve them corresponding to how they are stored during compression
1214 */
1215 uint32_t minbits_mask = 0;
1216 unsigned long long minval_mask = 0;
1217 unsigned minval_size = 0;
1218
1219 minbits = 0;
1220 for (i = 0; i < 4; i++) {
1221 minbits_mask = ((unsigned char *)*buf)[i];
1222 minbits_mask <<= i * 8;
1223 minbits |= minbits_mask;
1224 }
1225
1226 /* retrieval of minval takes into consideration situation where sizeof
1227 * unsigned long long (datatype of minval) may change from compression
1228 * to decompression, only smaller size is used
1229 */
1230 minval_size = sizeof(unsigned long long) <= ((unsigned char *)*buf)[4] ? sizeof(unsigned long long)
1231 : ((unsigned char *)*buf)[4];
1232 minval = 0;
1233 for (i = 0; i < minval_size; i++) {
1234 minval_mask = ((unsigned char *)*buf)[5 + i];
1235 minval_mask <<= i * 8;
1236 minval |= minval_mask;
1237 }
1238
1239 HDassert(minbits <= p.size * 8);
1240 p.minbits = minbits;
1241
1242 /* calculate size of output buffer after decompression */
1243 size_out = d_nelmts * p.size;
1244
1245 /* allocate memory space for decompressed buffer */
1246 if (NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out)))
1247 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
1248 "memory allocation failed for scaleoffset decompression")
1249
1250 /* special case: minbits equal to full precision */
1251 if (minbits == p.size * 8) {
1252 H5MM_memcpy(outbuf, (unsigned char *)(*buf) + buf_offset, size_out);
1253 /* free the original buffer */
1254 H5MM_xfree(*buf);
1255
1256 /* convert to dataset datatype endianness order if needed */
1257 if (need_convert)
1258 H5Z__scaleoffset_convert(outbuf, d_nelmts, p.size);
1259
1260 *buf = outbuf;
1261 outbuf = NULL;
1262 *buf_size = size_out;
1263 ret_value = size_out;
1264 goto done;
1265 }
1266
1267 /* decompress the buffer if minbits not equal to zero */
1268 if (minbits != 0)
1269 H5Z__scaleoffset_decompress(outbuf, d_nelmts, (unsigned char *)(*buf) + buf_offset, p);
1270 else {
1271 /* fill value is not defined and all data elements have the same value */
1272 for (i = 0; i < size_out; i++)
1273 outbuf[i] = 0;
1274 }
1275
1276 /* before postprocess, get memory type */
1277 if ((type = H5Z__scaleoffset_get_type(dtype_class, p.size, dtype_sign)) == 0)
1278 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "cannot use C integer datatype for cast")
1279
1280 /* postprocess after decompression */
1281 if (dtype_class == H5Z_SCALEOFFSET_CLS_INTEGER)
1282 H5Z__scaleoffset_postdecompress_i(outbuf, d_nelmts, type, filavail, cd_values, minbits, minval);
1283
1284 if (dtype_class == H5Z_SCALEOFFSET_CLS_FLOAT)
1285 if (scale_type == 0) { /* variable-minimum-bits method */
1286 if (H5Z__scaleoffset_postdecompress_fd(outbuf, d_nelmts, type, filavail, cd_values, minbits,
1287 minval, D_val) == FAIL)
1288 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "post-decompression failed")
1289 }
1290
1291 /* after postprocess, convert to dataset datatype endianness order if needed */
1292 if (need_convert)
1293 H5Z__scaleoffset_convert(outbuf, d_nelmts, p.size);
1294 }
1295 /* output; compress */
1296 else {
1297 HDassert(nbytes == d_nelmts * p.size);
1298
1299 /* before preprocess, convert to memory endianness order if needed */
1300 if (need_convert)
1301 H5Z__scaleoffset_convert(*buf, d_nelmts, p.size);
1302
1303 /* before preprocess, get memory type */
1304 if ((type = H5Z__scaleoffset_get_type(dtype_class, p.size, dtype_sign)) == 0)
1305 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "cannot use C integer datatype for cast")
1306
1307 /* preprocess before compression */
1308 if (dtype_class == H5Z_SCALEOFFSET_CLS_INTEGER)
1309 H5Z__scaleoffset_precompress_i(*buf, d_nelmts, type, filavail, cd_values, &minbits, &minval);
1310
1311 if (dtype_class == H5Z_SCALEOFFSET_CLS_FLOAT)
1312 if (scale_type == 0) { /* variable-minimum-bits method */
1313 if (H5Z__scaleoffset_precompress_fd(*buf, d_nelmts, type, filavail, cd_values, &minbits,
1314 &minval, D_val) == FAIL)
1315 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "pre-compression failed")
1316 }
1317
1318 HDassert(minbits <= p.size * 8);
1319
1320 /* calculate buffer size after compression
1321 * minbits and minval are stored in the front of the compressed buffer
1322 */
1323 p.minbits = minbits;
1324 size_out = buf_offset + nbytes * p.minbits / (p.size * 8) + 1; /* may be 1 larger */
1325
1326 /* allocate memory space for compressed buffer */
1327 if (NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out)))
1328 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for scaleoffset compression")
1329
1330 /* store minbits and minval in the front of output compressed buffer
1331 * store byte by byte from least significant byte to most significant byte
1332 * constant buffer size (21 bytes) is left for these two parameters
1333 * 4 bytes for minbits, 1 byte for size of minval, 16 bytes for minval
1334 */
1335 for (i = 0; i < 4; i++)
1336 ((unsigned char *)outbuf)[i] = (unsigned char)((minbits & ((uint32_t)0xff << i * 8)) >> i * 8);
1337
1338 ((unsigned char *)outbuf)[4] = sizeof(unsigned long long);
1339
1340 for (i = 0; i < sizeof(unsigned long long); i++)
1341 ((unsigned char *)outbuf)[5 + i] =
1342 (unsigned char)((minval & ((unsigned long long)0xff << i * 8)) >> i * 8);
1343
1344 /* Zero out remaining, unused bytes */
1345 /* (Looks like an error in the original determination of how many
1346 * bytes would be needed for parameters. - QAK, 2010/08/19)
1347 */
1348 HDmemset(outbuf + 13, 0, (size_t)8);
1349
1350 /* special case: minbits equal to full precision */
1351 if (minbits == p.size * 8) {
1352 H5MM_memcpy(outbuf + buf_offset, *buf, nbytes);
1353 /* free the original buffer */
1354 H5MM_xfree(*buf);
1355
1356 *buf = outbuf;
1357 outbuf = NULL;
1358 *buf_size = size_out;
1359 ret_value = buf_offset + nbytes;
1360 goto done;
1361 }
1362
1363 /* compress the buffer if minbits not equal to zero
1364 * minbits equal to zero only when fill value is not defined and
1365 * all data elements have the same value
1366 */
1367 if (minbits != 0)
1368 H5Z__scaleoffset_compress((unsigned char *)*buf, d_nelmts, outbuf + buf_offset,
1369 size_out - buf_offset, p);
1370 }
1371
1372 /* free the input buffer */
1373 H5MM_xfree(*buf);
1374
1375 /* set return values */
1376 *buf = outbuf;
1377 outbuf = NULL;
1378 *buf_size = size_out;
1379 ret_value = size_out;
1380
1381 done:
1382 if (outbuf)
1383 H5MM_xfree(outbuf);
1384 FUNC_LEAVE_NOAPI(ret_value)
1385 }
1386
1387 /* ============ Scaleoffset Algorithm ===============================================
1388 * assume one byte has 8 bit
1389 * assume padding bit is 0
1390 * assume size of unsigned char is one byte
1391 * assume one data item of certain datatype is stored continously in bytes
1392 * atomic datatype is treated on byte basis
1393 */
1394
1395 /* change byte order of input buffer either from little-endian to big-endian
1396 * or from big-endian to little-endian 2/21/2005
1397 */
1398 static void
H5Z__scaleoffset_convert(void * buf,unsigned d_nelmts,unsigned dtype_size)1399 H5Z__scaleoffset_convert(void *buf, unsigned d_nelmts, unsigned dtype_size)
1400 {
1401 if (dtype_size > 1) {
1402 size_t i, j;
1403 unsigned char *buffer, temp;
1404
1405 buffer = (unsigned char *)buf;
1406 for (i = 0; i < d_nelmts * dtype_size; i += dtype_size)
1407 for (j = 0; j < dtype_size / 2; j++) {
1408 /* swap pair of bytes */
1409 temp = buffer[i + j];
1410 buffer[i + j] = buffer[i + dtype_size - 1 - j];
1411 buffer[i + dtype_size - 1 - j] = temp;
1412 } /* end for */
1413 } /* end if */
1414 } /* end H5Z__scaleoffset_convert() */
1415
1416 /* return ceiling of floating-point log2 function
1417 * receive unsigned integer as argument 3/10/2005
1418 */
1419 static unsigned
H5Z__scaleoffset_log2(unsigned long long num)1420 H5Z__scaleoffset_log2(unsigned long long num)
1421 {
1422 unsigned v = 0;
1423 unsigned long long lower_bound = 1; /* is power of 2, largest value <= num */
1424 unsigned long long val = num;
1425
1426 while (val >>= 1) {
1427 v++;
1428 lower_bound <<= 1;
1429 }
1430
1431 if (num == lower_bound)
1432 return v;
1433 else
1434 return v + 1;
1435 }
1436
1437 /* precompress for integer type */
1438 static void
H5Z__scaleoffset_precompress_i(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t * minbits,unsigned long long * minval)1439 H5Z__scaleoffset_precompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type, unsigned filavail,
1440 const unsigned cd_values[], uint32_t *minbits, unsigned long long *minval)
1441 {
1442 if (type == t_uchar)
1443 H5Z_scaleoffset_precompress_1(unsigned char, data, d_nelmts, filavail, cd_values, minbits, minval);
1444 else if (type == t_ushort)
1445 H5Z_scaleoffset_precompress_1(unsigned short, data, d_nelmts, filavail, cd_values, minbits, minval);
1446 else if (type == t_uint)
1447 H5Z_scaleoffset_precompress_1(unsigned int, data, d_nelmts, filavail, cd_values, minbits, minval);
1448 else if (type == t_ulong)
1449 H5Z_scaleoffset_precompress_1(unsigned long, data, d_nelmts, filavail, cd_values, minbits, minval);
1450 else if (type == t_ulong_long)
1451 H5Z_scaleoffset_precompress_1(unsigned long long, data, d_nelmts, filavail, cd_values, minbits,
1452 minval);
1453 else if (type == t_schar) {
1454 signed char * buf = (signed char *)data, min = 0, max = 0, filval = 0;
1455 unsigned char span;
1456 unsigned i;
1457
1458 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */
1459 H5Z_scaleoffset_get_filval_1(signed char, cd_values, filval);
1460 if (*minbits ==
1461 H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */
1462 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max,
1463 min) if ((unsigned char)(max - min) >
1464 (unsigned char)(~(unsigned char)0 - 2))
1465 {
1466 *minbits = sizeof(signed char) * 8;
1467 return;
1468 }
1469 span = (unsigned char)(max - min + 1);
1470 *minbits = H5Z__scaleoffset_log2((unsigned long long)(span + 1));
1471 }
1472 else /* minbits already set, only calculate min */
1473 H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval,
1474 min) if (*minbits !=
1475 sizeof(signed char) *
1476 8) /* change values if minbits != full precision */
1477 for (i = 0; i < d_nelmts; i++) buf[i] =
1478 (signed char)((buf[i] == filval) ? (((unsigned char)1 << *minbits) - 1)
1479 : (buf[i] - min));
1480 }
1481 else { /* fill value undefined */
1482 if (*minbits ==
1483 H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */
1484 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max,
1485 min) if ((unsigned char)(max - min) >
1486 (unsigned char)(~(unsigned char)0 - 2))
1487 {
1488 *minbits = sizeof(signed char) * 8;
1489 *minval = (unsigned long long)min;
1490 return;
1491 }
1492 span = (unsigned char)(max - min + 1);
1493 *minbits = H5Z__scaleoffset_log2((unsigned long long)span);
1494 }
1495 else /* minbits already set, only calculate min */
1496 H5Z_scaleoffset_min_2(i, d_nelmts, buf,
1497 min) if (*minbits !=
1498 sizeof(signed char) *
1499 8) /* change values if minbits != full precision */
1500 for (i = 0; i < d_nelmts; i++) buf[i] = (signed char)(buf[i] - min);
1501 }
1502 *minval = (unsigned long long)min;
1503 }
1504 else if (type == t_short)
1505 H5Z_scaleoffset_precompress_2(short, data, d_nelmts, filavail, cd_values, minbits, minval);
1506 else if (type == t_int)
1507 H5Z_scaleoffset_precompress_2(int, data, d_nelmts, filavail, cd_values, minbits, minval);
1508 else if (type == t_long)
1509 H5Z_scaleoffset_precompress_2(long, data, d_nelmts, filavail, cd_values, minbits, minval);
1510 else if (type == t_long_long)
1511 H5Z_scaleoffset_precompress_2(long long, data, d_nelmts, filavail, cd_values, minbits, minval);
1512 }
1513
1514 /* postdecompress for integer type */
1515 static void
H5Z__scaleoffset_postdecompress_i(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t minbits,unsigned long long minval)1516 H5Z__scaleoffset_postdecompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1517 unsigned filavail, const unsigned cd_values[], uint32_t minbits,
1518 unsigned long long minval)
1519 {
1520 long long sminval = *(long long *)&minval; /* for signed integer types */
1521
1522 if (type == t_uchar)
1523 H5Z_scaleoffset_postdecompress_1(unsigned char, data, d_nelmts, filavail, cd_values, minbits, minval);
1524 else if (type == t_ushort)
1525 H5Z_scaleoffset_postdecompress_1(unsigned short, data, d_nelmts, filavail, cd_values, minbits,
1526 minval);
1527 else if (type == t_uint)
1528 H5Z_scaleoffset_postdecompress_1(unsigned int, data, d_nelmts, filavail, cd_values, minbits, minval);
1529 else if (type == t_ulong)
1530 H5Z_scaleoffset_postdecompress_1(unsigned long, data, d_nelmts, filavail, cd_values, minbits, minval);
1531 else if (type == t_ulong_long)
1532 H5Z_scaleoffset_postdecompress_1(unsigned long long, data, d_nelmts, filavail, cd_values, minbits,
1533 minval);
1534 else if (type == t_schar) {
1535 signed char *buf = (signed char *)data, filval = 0;
1536 unsigned i;
1537
1538 if (filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */
1539 H5Z_scaleoffset_get_filval_1(signed char, cd_values, filval) for (i = 0; i < d_nelmts;
1540 i++) buf[i] =
1541 (signed char)((buf[i] == (((unsigned char)1 << minbits) - 1)) ? filval : (buf[i] + sminval));
1542 }
1543 else /* fill value undefined */
1544 for (i = 0; i < d_nelmts; i++)
1545 buf[i] = (signed char)(buf[i] + sminval);
1546 }
1547 else if (type == t_short)
1548 H5Z_scaleoffset_postdecompress_2(short, data, d_nelmts, filavail, cd_values, minbits, sminval);
1549 else if (type == t_int)
1550 H5Z_scaleoffset_postdecompress_2(int, data, d_nelmts, filavail, cd_values, minbits, sminval);
1551 else if (type == t_long)
1552 H5Z_scaleoffset_postdecompress_2(long, data, d_nelmts, filavail, cd_values, minbits, sminval);
1553 else if (type == t_long_long)
1554 H5Z_scaleoffset_postdecompress_2(long long, data, d_nelmts, filavail, cd_values, minbits, sminval);
1555 }
1556
1557 /* precompress for floating-point type, variable-minimum-bits method
1558 success: non-negative, failure: negative 4/15/05 */
1559 static herr_t
H5Z__scaleoffset_precompress_fd(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t * minbits,unsigned long long * minval,double D_val)1560 H5Z__scaleoffset_precompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type, unsigned filavail,
1561 const unsigned cd_values[], uint32_t *minbits, unsigned long long *minval,
1562 double D_val)
1563 {
1564 herr_t ret_value = SUCCEED; /* Return value */
1565
1566 FUNC_ENTER_STATIC
1567
1568 if (type == t_float)
1569 H5Z_scaleoffset_precompress_3(float, HDpowf, HDfabsf, HDroundf, HDlroundf, HDllroundf, data, d_nelmts,
1570 filavail, cd_values, minbits, minval, D_val);
1571 else if (type == t_double)
1572 H5Z_scaleoffset_precompress_3(double, HDpow, HDfabs, HDround, HDlround, HDllround, data, d_nelmts,
1573 filavail, cd_values, minbits, minval, D_val);
1574
1575 done:
1576 FUNC_LEAVE_NOAPI(ret_value)
1577 }
1578
1579 /* postdecompress for floating-point type, variable-minimum-bits method
1580 success: non-negative, failure: negative 4/15/05 */
1581 static herr_t
H5Z__scaleoffset_postdecompress_fd(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t minbits,unsigned long long minval,double D_val)1582 H5Z__scaleoffset_postdecompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1583 unsigned filavail, const unsigned cd_values[], uint32_t minbits,
1584 unsigned long long minval, double D_val)
1585 {
1586 long long sminval = (long long)minval; /* for signed integer types */
1587 herr_t ret_value = SUCCEED; /* Return value */
1588
1589 FUNC_ENTER_STATIC
1590
1591 if (type == t_float)
1592 H5Z_scaleoffset_postdecompress_3(float, HDpowf, data, d_nelmts, filavail, cd_values, minbits, sminval,
1593 D_val);
1594 else if (type == t_double)
1595 H5Z_scaleoffset_postdecompress_3(double, HDpow, data, d_nelmts, filavail, cd_values, minbits, sminval,
1596 D_val);
1597
1598 done:
1599 FUNC_LEAVE_NOAPI(ret_value)
1600 }
1601
1602 static void
H5Z__scaleoffset_next_byte(size_t * j,unsigned * buf_len)1603 H5Z__scaleoffset_next_byte(size_t *j, unsigned *buf_len)
1604 {
1605 ++(*j);
1606 *buf_len = 8 * sizeof(unsigned char);
1607 }
1608
1609 static void
H5Z__scaleoffset_decompress_one_byte(unsigned char * data,size_t data_offset,unsigned k,unsigned begin_i,const unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p,unsigned dtype_len)1610 H5Z__scaleoffset_decompress_one_byte(unsigned char *data, size_t data_offset, unsigned k, unsigned begin_i,
1611 const unsigned char *buffer, size_t *j, unsigned *buf_len,
1612 parms_atomic p, unsigned dtype_len)
1613 {
1614 unsigned dat_len; /* dat_len is the number of bits to be copied in each data byte */
1615 unsigned char val; /* value to be copied in each data byte */
1616
1617 /* initialize value and bits of unsigned char to be copied */
1618 val = buffer[*j];
1619 if (k == begin_i)
1620 dat_len = 8 - (dtype_len - p.minbits) % 8;
1621 else
1622 dat_len = 8;
1623
1624 if (*buf_len > dat_len) {
1625 data[data_offset + k] =
1626 (unsigned char)((unsigned)(val >> (*buf_len - dat_len)) & (unsigned)(~((unsigned)~0 << dat_len)));
1627 *buf_len -= dat_len;
1628 } /* end if */
1629 else {
1630 data[data_offset + k] =
1631 (unsigned char)((val & ~((unsigned)(~0) << *buf_len)) << (dat_len - *buf_len));
1632 dat_len -= *buf_len;
1633 H5Z__scaleoffset_next_byte(j, buf_len);
1634 if (dat_len == 0)
1635 return;
1636
1637 val = buffer[*j];
1638 data[data_offset + k] |=
1639 (unsigned char)((unsigned)(val >> (*buf_len - dat_len)) & ~((unsigned)(~0) << dat_len));
1640 *buf_len -= dat_len;
1641 } /* end else */
1642 }
1643
1644 static void
H5Z__scaleoffset_decompress_one_atomic(unsigned char * data,size_t data_offset,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p)1645 H5Z__scaleoffset_decompress_one_atomic(unsigned char *data, size_t data_offset, unsigned char *buffer,
1646 size_t *j, unsigned *buf_len, parms_atomic p)
1647 {
1648 /* begin_i: the index of byte having first significant bit */
1649 unsigned begin_i;
1650 unsigned dtype_len;
1651 int k;
1652
1653 HDassert(p.minbits > 0);
1654
1655 dtype_len = p.size * 8;
1656
1657 if (p.mem_order == H5Z_SCALEOFFSET_ORDER_LE) { /* little endian */
1658 begin_i = p.size - 1 - (dtype_len - p.minbits) / 8;
1659
1660 for (k = (int)begin_i; k >= 0; k--)
1661 H5Z__scaleoffset_decompress_one_byte(data, data_offset, (unsigned)k, begin_i, buffer, j, buf_len,
1662 p, dtype_len);
1663 }
1664 else { /* big endian */
1665 HDassert(p.mem_order == H5Z_SCALEOFFSET_ORDER_BE);
1666
1667 begin_i = (dtype_len - p.minbits) / 8;
1668
1669 for (k = (int)begin_i; k <= (int)(p.size - 1); k++)
1670 H5Z__scaleoffset_decompress_one_byte(data, data_offset, (unsigned)k, begin_i, buffer, j, buf_len,
1671 p, dtype_len);
1672 }
1673 }
1674
1675 static void
H5Z__scaleoffset_decompress(unsigned char * data,unsigned d_nelmts,unsigned char * buffer,parms_atomic p)1676 H5Z__scaleoffset_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, parms_atomic p)
1677 {
1678 /* i: index of data, j: index of buffer,
1679 buf_len: number of bits to be filled in current byte */
1680 size_t i, j;
1681 unsigned buf_len;
1682
1683 /* must initialize to zeros */
1684 for (i = 0; i < d_nelmts * p.size; i++)
1685 data[i] = 0;
1686
1687 /* initialization before the loop */
1688 j = 0;
1689 buf_len = sizeof(unsigned char) * 8;
1690
1691 /* decompress */
1692 for (i = 0; i < d_nelmts; i++)
1693 H5Z__scaleoffset_decompress_one_atomic(data, i * p.size, buffer, &j, &buf_len, p);
1694 }
1695
1696 static void
H5Z__scaleoffset_compress_one_byte(const unsigned char * data,size_t data_offset,unsigned k,unsigned begin_i,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p,unsigned dtype_len)1697 H5Z__scaleoffset_compress_one_byte(const unsigned char *data, size_t data_offset, unsigned k,
1698 unsigned begin_i, unsigned char *buffer, size_t *j, unsigned *buf_len,
1699 parms_atomic p, unsigned dtype_len)
1700 {
1701 unsigned dat_len; /* dat_len is the number of bits to be copied in each data byte */
1702 unsigned char val; /* value to be copied in each data byte */
1703
1704 /* initialize value and bits of unsigned char to be copied */
1705 val = data[data_offset + k];
1706 if (k == begin_i)
1707 dat_len = 8 - (dtype_len - p.minbits) % 8;
1708 else
1709 dat_len = 8;
1710
1711 if (*buf_len > dat_len) {
1712 buffer[*j] |= (unsigned char)((val & ~((unsigned)(~0) << dat_len)) << (*buf_len - dat_len));
1713 *buf_len -= dat_len;
1714 }
1715 else {
1716 buffer[*j] |=
1717 (unsigned char)((unsigned)(val >> (dat_len - *buf_len)) & ~((unsigned)(~0) << *buf_len));
1718 dat_len -= *buf_len;
1719 H5Z__scaleoffset_next_byte(j, buf_len);
1720 if (dat_len == 0)
1721 return;
1722
1723 buffer[*j] = (unsigned char)((val & ~((unsigned)(~0) << dat_len)) << (*buf_len - dat_len));
1724 *buf_len -= dat_len;
1725 } /* end else */
1726 }
1727
1728 static void
H5Z__scaleoffset_compress_one_atomic(unsigned char * data,size_t data_offset,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p)1729 H5Z__scaleoffset_compress_one_atomic(unsigned char *data, size_t data_offset, unsigned char *buffer,
1730 size_t *j, unsigned *buf_len, parms_atomic p)
1731 {
1732 /* begin_i: the index of byte having first significant bit */
1733 unsigned begin_i;
1734 unsigned dtype_len;
1735 int k;
1736
1737 HDassert(p.minbits > 0);
1738
1739 dtype_len = p.size * 8;
1740
1741 if (p.mem_order == H5Z_SCALEOFFSET_ORDER_LE) { /* little endian */
1742 begin_i = p.size - 1 - (dtype_len - p.minbits) / 8;
1743
1744 for (k = (int)begin_i; k >= 0; k--)
1745 H5Z__scaleoffset_compress_one_byte(data, data_offset, (unsigned)k, begin_i, buffer, j, buf_len, p,
1746 dtype_len);
1747 }
1748 else { /* big endian */
1749 HDassert(p.mem_order == H5Z_SCALEOFFSET_ORDER_BE);
1750 begin_i = (dtype_len - p.minbits) / 8;
1751
1752 for (k = (int)begin_i; k <= (int)(p.size - 1); k++)
1753 H5Z__scaleoffset_compress_one_byte(data, data_offset, (unsigned)k, begin_i, buffer, j, buf_len, p,
1754 dtype_len);
1755 }
1756 }
1757
1758 static void
H5Z__scaleoffset_compress(unsigned char * data,unsigned d_nelmts,unsigned char * buffer,size_t buffer_size,parms_atomic p)1759 H5Z__scaleoffset_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, size_t buffer_size,
1760 parms_atomic p)
1761 {
1762 /* i: index of data, j: index of buffer,
1763 buf_len: number of bits to be filled in current byte */
1764 size_t i, j;
1765 unsigned buf_len;
1766
1767 /* must initialize buffer to be zeros */
1768 for (j = 0; j < buffer_size; j++)
1769 buffer[j] = 0;
1770
1771 /* initialization before the loop */
1772 j = 0;
1773 buf_len = sizeof(unsigned char) * 8;
1774
1775 /* compress */
1776 for (i = 0; i < d_nelmts; i++)
1777 H5Z__scaleoffset_compress_one_atomic(data, i * p.size, buffer, &j, &buf_len, p);
1778 }
1779