1 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2 * Copyright by The HDF Group. *
3 * Copyright by the Board of Trustees of the University of Illinois. *
4 * All rights reserved. *
5 * *
6 * This file is part of HDF5. The full HDF5 copyright notice, including *
7 * terms governing use, modification, and redistribution, is contained in *
8 * the files COPYING and Copyright.html. COPYING can be found at the root *
9 * of the source code distribution tree; Copyright.html can be found at the *
10 * root level of an installed copy of the electronic HDF5 document set and *
11 * is linked from the top-level documents page. It can also be found at *
12 * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
13 * access to either file, you may request a copy from help@hdfgroup.org. *
14 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
15
16 #define H5Z_PACKAGE /*suppress error about including H5Zpkg */
17
18 #include "H5private.h" /* Generic Functions */
19 #include "H5ACprivate.h" /* Metadata cache */
20 #include "H5Eprivate.h" /* Error handling */
21 #include "H5Iprivate.h" /* IDs */
22 #include "H5MMprivate.h" /* Memory management */
23 #include "H5Pprivate.h" /* Property lists */
24 #include "H5Oprivate.h" /* Object headers */
25 #include "H5Sprivate.h" /* Dataspaces */
26 #include "H5Tprivate.h" /* Datatypes */
27 #include "H5Zpkg.h" /* Data filters */
28
29 #ifdef H5_HAVE_FILTER_SCALEOFFSET
30
31 /* Struct of parameters needed for compressing/decompressing one atomic datatype */
32 typedef struct {
33 size_t size; /* datatype size */
34 uint32_t minbits; /* minimum bits to compress one value of such datatype */
35 unsigned mem_order; /* current memory endianness order */
36 } parms_atomic;
37
38 enum H5Z_scaleoffset_t {t_bad=0, t_uchar=1, t_ushort, t_uint, t_ulong, t_ulong_long,
39 t_schar, t_short, t_int, t_long, t_long_long,
40 t_float, t_double};
41
42 /* Local function prototypes */
43 static double H5Z_scaleoffset_rnd(double val);
44 static htri_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
45 static enum H5Z_scaleoffset_t H5Z_scaleoffset_get_type(unsigned dtype_class,
46 unsigned dtype_size, unsigned dtype_sign);
47 static herr_t H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist,
48 H5T_t *type, enum H5Z_scaleoffset_t scale_type, unsigned cd_values[],
49 int need_convert, hid_t dxpl_id);
50 static herr_t H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id);
51 static size_t H5Z_filter_scaleoffset(unsigned flags, size_t cd_nelmts,
52 const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf);
53 static void H5Z_scaleoffset_convert(void *buf, unsigned d_nelmts, size_t dtype_size);
54 static unsigned H5Z_scaleoffset_log2(unsigned long long num);
55 static void H5Z_scaleoffset_precompress_i(void *data, unsigned d_nelmts,
56 enum H5Z_scaleoffset_t type, unsigned filavail, const unsigned cd_values[],
57 uint32_t *minbits, unsigned long long *minval);
58 static void H5Z_scaleoffset_postdecompress_i(void *data, unsigned d_nelmts,
59 enum H5Z_scaleoffset_t type, unsigned filavail, const unsigned cd_values[],
60 uint32_t minbits, unsigned long long minval);
61 static herr_t H5Z_scaleoffset_precompress_fd(void *data, unsigned d_nelmts,
62 enum H5Z_scaleoffset_t type, unsigned filavail, const unsigned cd_values[],
63 uint32_t *minbits, unsigned long long *minval, double D_val);
64 static herr_t H5Z_scaleoffset_postdecompress_fd(void *data, unsigned d_nelmts,
65 enum H5Z_scaleoffset_t type, unsigned filavail, const unsigned cd_values[],
66 uint32_t minbits, unsigned long long minval, double D_val);
67 static void H5Z_scaleoffset_next_byte(size_t *j, unsigned *buf_len);
68 static void H5Z_scaleoffset_decompress_one_byte(unsigned char *data, size_t data_offset,
69 unsigned k, unsigned begin_i, unsigned char *buffer, size_t *j, unsigned *buf_len,
70 parms_atomic p, unsigned dtype_len);
71 static void H5Z_scaleoffset_compress_one_byte(unsigned char *data, size_t data_offset,
72 unsigned k, unsigned begin_i, unsigned char *buffer, size_t *j, unsigned *buf_len,
73 parms_atomic p, unsigned dtype_len);
74 static void H5Z_scaleoffset_decompress_one_atomic(unsigned char *data, size_t data_offset,
75 unsigned char *buffer, size_t *j, unsigned *buf_len, parms_atomic p);
76 static void H5Z_scaleoffset_compress_one_atomic(unsigned char *data, size_t data_offset,
77 unsigned char *buffer, size_t *j, unsigned *buf_len, parms_atomic p);
78 static void H5Z_scaleoffset_decompress(unsigned char *data, unsigned d_nelmts,
79 unsigned char *buffer, parms_atomic p);
80 static void H5Z_scaleoffset_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
81 size_t buffer_size, parms_atomic p);
82
83 /* This message derives from H5Z */
84 H5Z_class2_t H5Z_SCALEOFFSET[1] = {{
85 H5Z_CLASS_T_VERS, /* H5Z_class_t version */
86 H5Z_FILTER_SCALEOFFSET, /* Filter id number */
87 1, /* Assume encoder present: check before registering */
88 1, /* decoder_present flag (set to true) */
89 "scaleoffset", /* Filter name for debugging */
90 H5Z_can_apply_scaleoffset, /* The "can apply" callback */
91 H5Z_set_local_scaleoffset, /* The "set local" callback */
92 H5Z_filter_scaleoffset, /* The actual filter function */
93 }};
94
95 /* Local macros */
96 #define H5Z_SCALEOFFSET_TOTAL_NPARMS 20 /* Total number of parameters for filter */
97 #define H5Z_SCALEOFFSET_PARM_SCALETYPE 0 /* "User" parameter for scale type */
98 #define H5Z_SCALEOFFSET_PARM_SCALEFACTOR 1 /* "User" parameter for scale factor */
99 #define H5Z_SCALEOFFSET_PARM_NELMTS 2 /* "Local" parameter for number of elements in the chunk */
100 #define H5Z_SCALEOFFSET_PARM_CLASS 3 /* "Local" parameter for datatype class */
101 #define H5Z_SCALEOFFSET_PARM_SIZE 4 /* "Local" parameter for datatype size */
102 #define H5Z_SCALEOFFSET_PARM_SIGN 5 /* "Local" parameter for integer datatype sign */
103 #define H5Z_SCALEOFFSET_PARM_ORDER 6 /* "Local" parameter for datatype byte order */
104 #define H5Z_SCALEOFFSET_PARM_FILAVAIL 7 /* "Local" parameter for dataset fill value existence */
105 #define H5Z_SCALEOFFSET_PARM_FILVAL 8 /* "Local" parameter for start location to store dataset fill value */
106
107 #define H5Z_SCALEOFFSET_CLS_INTEGER 0 /* Integer (datatype class) */
108 #define H5Z_SCALEOFFSET_CLS_FLOAT 1 /* Floatig-point (datatype class) */
109
110 #define H5Z_SCALEOFFSET_SGN_NONE 0 /* Unsigned integer type */
111 #define H5Z_SCALEOFFSET_SGN_2 1 /* Two's complement signed integer type */
112
113 #define H5Z_SCALEOFFSET_ORDER_LE 0 /* Little endian (datatype byte order) */
114 #define H5Z_SCALEOFFSET_ORDER_BE 1 /* Big endian (datatype byte order) */
115
116 #define H5Z_SCALEOFFSET_FILL_UNDEFINED 0 /* Fill value is not defined */
117 #define H5Z_SCALEOFFSET_FILL_DEFINED 1 /* Fill value is defined */
118
119 /* Store fill value in cd_values[] */
120 #define H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
121 { \
122 unsigned _i = H5Z_SCALEOFFSET_PARM_FILVAL; /* index into cd_values */ \
123 uint32_t _cd_value; /* Current cd_value */ \
124 char *_fv_p; /* Pointer to current byte in fill_val */ \
125 size_t _copy_size = 4; /* # of bytes to copy this iteration */ \
126 size_t _size_rem = sizeof(type); /* # of bytes left to copy to cd_values */ \
127 \
128 /* Store the fill value as the last entry in cd_values[] \
129 * Store byte by byte from least significant byte to most significant byte \
130 * Plenty of space left for the fill value (from index 8 to 19) \
131 * H5O_pline_encode will byte-swap each individual cd value, but we still \
132 * need to swap the cd values as a whole if we are on a BE machine. Note \
133 * that we need to make sure to put the data only in the lowest 4 bytes of \
134 * each, if sizeof(unsigned) > 4. \
135 */ \
136 if(H5T_native_order_g == H5T_ORDER_LE) { \
137 _fv_p = (char *)&(fill_val); \
138 /* Copy 4 bytes at a time to each cd value */ \
139 do { \
140 if(_size_rem < 4) { \
141 /* Amount left to copy is smaller than a cd_value, adjust copy \
142 * size and initialize cd_value as it will not be fully \
143 * overwritten */ \
144 _copy_size = _size_rem; \
145 _cd_value = (uint32_t)0; \
146 } /* end if */ \
147 \
148 /* Copy the value */ \
149 HDmemcpy(&_cd_value, _fv_p, _copy_size); \
150 (cd_values)[_i] = (unsigned)_cd_value; \
151 \
152 /* Next field */ \
153 _i++; \
154 _fv_p += _copy_size; \
155 _size_rem -= _copy_size; \
156 } while(_size_rem); \
157 } /* end if */ \
158 else { \
159 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
160 \
161 /* Copy 4 bytes at a time to each cd value, but start at the end \
162 * (highest address) of fill_val */ \
163 _fv_p = ((char *)&(fill_val)) + sizeof(type) - MIN(4, _size_rem); \
164 while(_size_rem >= 4) { \
165 /* Copy the value */ \
166 HDmemcpy(&_cd_value, _fv_p, _copy_size); \
167 (cd_values)[_i] = (unsigned)_cd_value; \
168 \
169 /* Next field */ \
170 _i++; \
171 _size_rem -= 4; \
172 if(_size_rem >= 4) \
173 _fv_p -= 4; \
174 else \
175 _fv_p -= _size_rem; \
176 } /* end while */ \
177 \
178 HDassert(_fv_p == (char *)&(fill_val)); \
179 if(_size_rem) { \
180 /* Amount left to copy is smaller than a cd_value, initialize \
181 * _cd_value as it will not be fully overwritten and copy to the end \
182 * of _cd value as it is BE. */ \
183 _cd_value = (uint32_t)0; \
184 HDmemcpy((char *)&_cd_value + 4 - _size_rem, _fv_p, _size_rem); \
185 (cd_values)[_i] = (unsigned)_cd_value; \
186 } /* end if */ \
187 } /* end else */ \
188 }
189
190 /* Set the fill value parameter in cd_values[] for unsigned integer type */
191 #define H5Z_scaleoffset_set_filval_1(type, dcpl_plist, dt, cd_values, need_convert, dxpl_id)\
192 { \
193 type fill_val; \
194 \
195 /* Get dataset fill value */ \
196 if(H5P_get_fill_value(dcpl_plist, dt, &fill_val, dxpl_id) < 0) \
197 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
198 \
199 if(need_convert) \
200 H5Z_scaleoffset_convert(&fill_val, 1, sizeof(type)); \
201 \
202 H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
203 }
204
205 /* Set the fill value parameter in cd_values[] for signed integer type */
206 #define H5Z_scaleoffset_set_filval_2(type, dcpl_plist, dt, cd_values, need_convert, dxpl_id)\
207 { \
208 type fill_val; \
209 \
210 /* Get dataset fill value */ \
211 if(H5P_get_fill_value(dcpl_plist, dt, &fill_val, dxpl_id) < 0) \
212 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
213 \
214 if(need_convert) \
215 H5Z_scaleoffset_convert(&fill_val, 1, sizeof(type)); \
216 \
217 H5Z_scaleoffset_save_filval(unsigned type, cd_values, fill_val) \
218 }
219
220 /* Set the fill value parameter in cd_values[] for character integer type */
221 #define H5Z_scaleoffset_set_filval_3(type, dcpl_plist, dt, cd_values, need_convert, dxpl_id)\
222 { \
223 type fill_val; \
224 \
225 /* Get dataset fill value */ \
226 if(H5P_get_fill_value(dcpl_plist, dt, &fill_val, dxpl_id) < 0) \
227 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
228 \
229 /* Store the fill value as the last entry in cd_values[] */ \
230 (cd_values)[H5Z_SCALEOFFSET_PARM_FILVAL] = (unsigned)((unsigned char)fill_val); \
231 }
232
233 /* Set the fill value parameter in cd_values[] for floating-point type */
234 #define H5Z_scaleoffset_set_filval_4(type, dcpl_plist, dt, cd_values, need_convert, dxpl_id)\
235 { \
236 type fill_val; \
237 \
238 /* Get dataset fill value */ \
239 if(H5P_get_fill_value(dcpl_plist, dt, &fill_val, dxpl_id) < 0) \
240 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get fill value") \
241 \
242 if(need_convert) \
243 H5Z_scaleoffset_convert(&fill_val, 1, sizeof(type)); \
244 \
245 H5Z_scaleoffset_save_filval(type, cd_values, fill_val) \
246 }
247
248 /* Get the fill value for integer type */
249 #define H5Z_scaleoffset_get_filval_1(type, cd_values, fill_val) \
250 { \
251 unsigned _i = H5Z_SCALEOFFSET_PARM_FILVAL; /* index into cd_values */ \
252 uint32_t _cd_value; /* Current cd_value */ \
253 char *_fv_p; /* Pointer to current byte in fill_val */ \
254 size_t _copy_size = 4; /* # of bytes to copy this iteration */ \
255 size_t _size_rem = sizeof(type); /* # of bytes left to copy to filval */ \
256 \
257 /* Retrieve the fill value from the last entry in cd_values[] \
258 * Store byte by byte from least significant byte to most significant byte \
259 * Plenty of space left for the fill value (from index 8 to 19) \
260 * H5O_pline_encode will byte-swap each individual cd value, but we still \
261 * need to swap the cd values as a whole if we are on a BE machine. Note \
262 * that we need to make sure to put the data only in the lowest 4 bytes of \
263 * each, if sizeof(unsigned) > 4. \
264 */ \
265 if(H5T_native_order_g == H5T_ORDER_LE) { \
266 _fv_p = (char *)&(fill_val); \
267 /* Copy 4 bytes at a time to each cd value */ \
268 do { \
269 if(_size_rem < 4) \
270 /* Amount left to copy is smaller than a cd_value, adjust copy \
271 * size and initialize cd_value as it will not be fully \
272 * overwritten */ \
273 _copy_size = _size_rem; \
274 \
275 /* Copy the value */ \
276 _cd_value = (uint32_t)(cd_values)[_i]; \
277 HDmemcpy(_fv_p, &_cd_value, _copy_size); \
278 \
279 /* Next field */ \
280 _i++; \
281 _fv_p += _copy_size; \
282 _size_rem -= _copy_size; \
283 } while(_size_rem); \
284 } /* end if */ \
285 else { \
286 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
287 \
288 /* Copy 4 bytes at a time to each cd value, but start at the end \
289 * (highest address) of fill_val */ \
290 _fv_p = ((char *)&(fill_val)) + sizeof(type) - MIN(4, _size_rem); \
291 while(_size_rem >= 4) { \
292 /* Copy the value */ \
293 _cd_value = (uint32_t)(cd_values)[_i]; \
294 HDmemcpy(_fv_p, &_cd_value, _copy_size); \
295 \
296 /* Next field */ \
297 _i++; \
298 _size_rem -= 4; \
299 if(_size_rem >=4) \
300 _fv_p -= 4; \
301 else \
302 _fv_p -= _size_rem; \
303 } /* end while */ \
304 \
305 HDassert(_fv_p == (char *)&(fill_val)); \
306 if(_size_rem) { \
307 /* Amount left to copy is smaller than a cd_value, initialize \
308 * _cd_value as it will not be fully overwritten and copy to the end \
309 * of _cd value as it is BE. */ \
310 _cd_value = (uint32_t)(cd_values)[_i]; \
311 HDmemcpy(_fv_p, (char *)&_cd_value + 4 - _size_rem, _size_rem); \
312 } /* end if */ \
313 } /* end else */ \
314 }
315
316 /* Get the fill value for floating-point type */
317 #define H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
318 { \
319 if(sizeof(type) <= sizeof(long long)) \
320 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) \
321 else \
322 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
323 }
324
325 /* Find maximum and minimum values of a buffer with fill value defined for integer type */
326 #define H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min)\
327 { \
328 i = 0; while(i < d_nelmts && buf[i]== filval) i++; \
329 if(i < d_nelmts) min = max = buf[i]; \
330 for(; i < d_nelmts; i++) { \
331 if(buf[i] == filval) continue; /* ignore fill value */ \
332 if(buf[i] > max) max = buf[i]; \
333 if(buf[i] < min) min = buf[i]; \
334 } \
335 }
336
337 /* Find maximum and minimum values of a buffer with fill value undefined */
338 #define H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min)\
339 { \
340 min = max = buf[0]; \
341 for(i = 0; i < d_nelmts; i++) { \
342 if(buf[i] > max) max = buf[i]; \
343 if(buf[i] < min) min = buf[i]; \
344 } \
345 }
346
347 /* Find maximum and minimum values of a buffer with fill value defined for floating-point type */
348 #define H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \
349 { \
350 i = 0; while(i < d_nelmts && HDfabs(buf[i] - filval) < HDpow(10.0f, -D_val)) i++; \
351 if(i < d_nelmts) min = max = buf[i]; \
352 for(; i < d_nelmts; i++) { \
353 if(HDfabs(buf[i] - filval) < HDpow(10.0f, -D_val)) \
354 continue; /* ignore fill value */ \
355 if(buf[i] > max) max = buf[i]; \
356 if(buf[i] < min) min = buf[i]; \
357 } \
358 }
359
360 /* Find minimum value of a buffer with fill value defined for integer type */
361 #define H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval, min) \
362 { \
363 i = 0; while(i < d_nelmts && buf[i]== filval) i++; \
364 if(i < d_nelmts) min = buf[i]; \
365 for(; i < d_nelmts; i++) { \
366 if(buf[i] == filval) continue; /* ignore fill value */ \
367 if(buf[i] < min) min = buf[i]; \
368 } \
369 }
370
371 /* Find minimum value of a buffer with fill value undefined */
372 #define H5Z_scaleoffset_min_2(i, d_nelmts, buf, min)\
373 { \
374 min = buf[0]; \
375 for(i = 0; i < d_nelmts; i++) \
376 if(buf[i] < min) min = buf[i]; \
377 }
378
379 /* Check and handle special situation for unsigned integer type */
380 #define H5Z_scaleoffset_check_1(type, max, min, minbits) \
381 { \
382 if(max - min > (type)(~(type)0 - 2)) \
383 { *minbits = sizeof(type)*8; return; } \
384 }
385
386 /* Check and handle special situation for signed integer type */
387 #define H5Z_scaleoffset_check_2(type, max, min, minbits) \
388 { \
389 if((unsigned type)(max - min) > (unsigned type)(~(unsigned type)0 - 2)) \
390 { *minbits = sizeof(type)*8; return; } \
391 }
392
393 /* Check and handle special situation for floating-point type */
394 #define H5Z_scaleoffset_check_3(i, type, max, min, minbits, D_val) \
395 { \
396 if(sizeof(type)==sizeof(int)) { \
397 if(H5Z_scaleoffset_rnd(max*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)) \
398 > HDpow(2.0f, (double)(sizeof(int)*8 - 1))) { \
399 *minbits = sizeof(int)*8; goto done; \
400 } \
401 } else if(sizeof(type)==sizeof(long)) { \
402 if(H5Z_scaleoffset_rnd(max*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)) \
403 > HDpow(2.0f, (double)(sizeof(long)*8 - 1))) { \
404 *minbits = sizeof(long)*8; goto done; \
405 } \
406 } else if(sizeof(type)==sizeof(long long)) { \
407 if(H5Z_scaleoffset_rnd(max*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)) \
408 > HDpow(2.0f, (double)(sizeof(long long)*8 - 1))) { \
409 *minbits = sizeof(long long)*8; goto done; \
410 } \
411 } else \
412 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype")\
413 }
414
415 /* Precompress for unsigned integer type */
416 #define H5Z_scaleoffset_precompress_1(type, data, d_nelmts, filavail, cd_values, minbits, minval)\
417 { \
418 type *buf = (type *)data, min = 0, max = 0, span, filval = 0; \
419 unsigned i; \
420 \
421 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
422 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) \
423 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */ \
424 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min) \
425 H5Z_scaleoffset_check_1(type, max, min, minbits) \
426 span = (type)(max - min + 1); \
427 *minbits = H5Z_scaleoffset_log2((unsigned long long)(span+1)); \
428 } else /* minbits already set, only calculate min */ \
429 H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval, min) \
430 if(*minbits != sizeof(type)*8) /* change values if minbits != full precision */ \
431 for(i = 0; i < d_nelmts; i++) \
432 buf[i] = (type)((buf[i] == filval) ? (((type)1 << *minbits) - 1) : (buf[i] - min)); \
433 } else { /* fill value undefined */ \
434 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT ) { /* minbits not set yet, calculate max, min, and minbits */ \
435 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
436 H5Z_scaleoffset_check_1(type, max, min, minbits) \
437 span = (type)(max - min + 1); \
438 *minbits = H5Z_scaleoffset_log2((unsigned long long)span); \
439 } else /* minbits already set, only calculate min */ \
440 H5Z_scaleoffset_min_2(i, d_nelmts, buf, min) \
441 if(*minbits != sizeof(type)*8) /* change values if minbits != full precision */ \
442 for(i = 0; i < d_nelmts; i++) \
443 buf[i] = (type)(buf[i] - min); \
444 } \
445 *minval = min; \
446 }
447
448 /* Precompress for signed integer type */
449 #define H5Z_scaleoffset_precompress_2(type, data, d_nelmts, filavail, cd_values, minbits, minval)\
450 { \
451 type *buf = (type *)data, min = 0, max = 0, filval = 0; \
452 unsigned type span; unsigned i; \
453 \
454 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
455 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) \
456 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */ \
457 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min) \
458 H5Z_scaleoffset_check_2(type, max, min, minbits) \
459 span = (unsigned type)(max - min + 1); \
460 *minbits = H5Z_scaleoffset_log2((unsigned long long)(span + 1)); \
461 } else /* minbits already set, only calculate min */ \
462 H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval, min) \
463 if(*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
464 for(i = 0; i < d_nelmts; i++) \
465 buf[i] = (type)((buf[i] == filval) ? (type)(((unsigned type)1 << *minbits) - 1) : (buf[i] - min)); \
466 } else { /* fill value undefined */ \
467 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT ) { /* minbits not set yet, calculate max, min, and minbits */\
468 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
469 H5Z_scaleoffset_check_2(type, max, min, minbits) \
470 span = (unsigned type)(max - min + 1); \
471 *minbits = H5Z_scaleoffset_log2((unsigned long long)span); \
472 } else /* minbits already set, only calculate min */ \
473 H5Z_scaleoffset_min_2(i, d_nelmts, buf, min) \
474 if(*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
475 for(i = 0; i < d_nelmts; i++) \
476 buf[i] = (type)(buf[i] - min); \
477 } \
478 *minval = (unsigned long long)min; \
479 }
480
481 /* Modify values of data in precompression if fill value defined for floating-point type */
482 #define H5Z_scaleoffset_modify_1(i, type, buf, d_nelmts, filval, minbits, min, D_val) \
483 { \
484 if(sizeof(type)==sizeof(int)) \
485 for(i = 0; i < d_nelmts; i++) { \
486 if(HDfabs(buf[i] - filval) < HDpow(10.0f, -D_val)) \
487 *(int *)&buf[i] = (int)(((unsigned int)1 << *minbits) - 1); \
488 else \
489 *(int *)&buf[i] = H5Z_scaleoffset_rnd( \
490 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
491 } \
492 else if(sizeof(type)==sizeof(long)) \
493 for(i = 0; i < d_nelmts; i++) { \
494 if(HDfabs(buf[i] - filval) < HDpow(10.0f, -D_val)) \
495 *(long *)&buf[i] = (long)(((unsigned long)1 << *minbits) - 1); \
496 else \
497 *(long *)&buf[i] = H5Z_scaleoffset_rnd( \
498 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
499 } \
500 else if(sizeof(type)==sizeof(long long)) \
501 for(i = 0; i < d_nelmts; i++) { \
502 if(HDfabs(buf[i] - filval) < HDpow(10.0f, -D_val)) \
503 *(long long *)&buf[i] = (long long)(((unsigned long long)1 << *minbits) - 1); \
504 else \
505 *(long long *)&buf[i] = H5Z_scaleoffset_rnd( \
506 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
507 } \
508 else \
509 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype")\
510 }
511
512 /* Modify values of data in precompression if fill value undefined for floating-point type */
513 #define H5Z_scaleoffset_modify_2(i, type, buf, d_nelmts, min, D_val) \
514 { \
515 if(sizeof(type)==sizeof(int)) \
516 for(i = 0; i < d_nelmts; i++) \
517 *(int *)&buf[i] = H5Z_scaleoffset_rnd( \
518 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
519 else if(sizeof(type)==sizeof(long)) \
520 for(i = 0; i < d_nelmts; i++) \
521 *(long *)&buf[i] = H5Z_scaleoffset_rnd( \
522 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
523 else if(sizeof(type)==sizeof(long long)) \
524 for(i = 0; i < d_nelmts; i++) \
525 *(long long *)&buf[i] = H5Z_scaleoffset_rnd( \
526 buf[i]*HDpow(10.0f, D_val) - min*HDpow(10.0f, D_val)); \
527 else \
528 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype")\
529 }
530
531 /* Save the minimum value for floating-point type */
532 #define H5Z_scaleoffset_save_min(i, type, minval, min) \
533 { \
534 if(sizeof(type) <= sizeof(long long)) \
535 /* Save min value to corresponding position \
536 * byte-order will be swapped as appropriate, but be sure to \
537 * account for offset in BE if sizes differ \
538 */ \
539 if(H5T_native_order_g == H5T_ORDER_LE) \
540 HDmemcpy(minval, &min, sizeof(type)); \
541 else { \
542 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
543 HDmemcpy(((char *)minval) + (sizeof(long long) - sizeof(type)), \
544 &min, sizeof(type)); \
545 } /* end else */ \
546 else \
547 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
548 }
549
550 /* Precompress for floating-point type using variable-minimum-bits method */
551 #define H5Z_scaleoffset_precompress_3(type, data, d_nelmts, filavail, cd_values, \
552 minbits, minval, D_val) \
553 { \
554 type *buf = (type *)data, min = 0, max = 0, filval = 0; \
555 unsigned long long span; \
556 unsigned i; \
557 \
558 *minval = 0; \
559 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
560 H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
561 H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \
562 H5Z_scaleoffset_check_3(i, type, max, min, minbits, D_val) \
563 span = H5Z_scaleoffset_rnd(max * HDpow(10.0f, D_val) - min * HDpow(10.0f, D_val)) + 1; \
564 *minbits = H5Z_scaleoffset_log2((unsigned long long)(span + 1)); \
565 if(*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
566 H5Z_scaleoffset_modify_1(i, type, buf, d_nelmts, filval, minbits, min, D_val) \
567 } else { /* fill value undefined */ \
568 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \
569 H5Z_scaleoffset_check_3(i, type, max, min, minbits, D_val) \
570 span = H5Z_scaleoffset_rnd(max * HDpow(10.0f, D_val) - min * HDpow(10.0f, D_val)) + 1; \
571 *minbits = H5Z_scaleoffset_log2((unsigned long long)span); \
572 if(*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \
573 H5Z_scaleoffset_modify_2(i, type, buf, d_nelmts, min, D_val) \
574 } \
575 H5Z_scaleoffset_save_min(i, type, minval, min) \
576 }
577
578 /* Postdecompress for unsigned integer type */
579 #define H5Z_scaleoffset_postdecompress_1(type, data, d_nelmts, filavail, cd_values, minbits, minval)\
580 { \
581 type *buf = (type *)data, filval = 0; unsigned i; \
582 \
583 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
584 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) \
585 for(i = 0; i < d_nelmts; i++) \
586 buf[i] = (type)((buf[i] == (((type)1 << minbits) - 1)) ? filval : (buf[i] + minval)); \
587 } else /* fill value undefined */ \
588 for(i = 0; i < d_nelmts; i++) buf[i] = (type)(buf[i] + (type)(minval)); \
589 }
590
591 /* Postdecompress for signed integer type */
592 #define H5Z_scaleoffset_postdecompress_2(type, data, d_nelmts, filavail, cd_values, minbits, minval)\
593 { \
594 type *buf = (type *)data, filval = 0; \
595 unsigned i; \
596 \
597 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
598 H5Z_scaleoffset_get_filval_1(type, cd_values, filval) \
599 for(i = 0; i < d_nelmts; i++) \
600 buf[i] = (type)(((unsigned type)buf[i] == (((unsigned type)1 << minbits) - 1)) ? filval : (buf[i] + minval));\
601 } else /* fill value undefined */ \
602 for(i = 0; i < d_nelmts; i++) \
603 buf[i] = (type)(buf[i] + (type)(minval)); \
604 }
605
606 /* Retrive minimum value of floating-point type */
607 #define H5Z_scaleoffset_get_min(type, minval, min) \
608 { \
609 if(sizeof(type) <= sizeof(long long)) \
610 /* retrieve min value from corresponding position \
611 * byte-order has already been swapped as appropriate, but be sure to \
612 * account for offset in BE if sizes differ \
613 */ \
614 if(H5T_native_order_g == H5T_ORDER_LE) \
615 HDmemcpy(&min, &minval, sizeof(type)); \
616 else { \
617 HDassert(H5T_native_order_g == H5T_ORDER_BE); \
618 HDmemcpy(&min, ((char *)&minval) + (sizeof(long long) \
619 - sizeof(type)), sizeof(type)); \
620 } /* end else */ \
621 else \
622 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
623 }
624
625 /* Modify values of data in postdecompression if fill value defined for floating-point type */
626 #define H5Z_scaleoffset_modify_3(i, type, buf, d_nelmts, filval, minbits, min, D_val) \
627 { \
628 if(sizeof(type)==sizeof(int)) \
629 for(i = 0; i < d_nelmts; i++) \
630 buf[i] = (type)((*(int *)&buf[i] == (int)(((unsigned int)1 << minbits) - 1)) ? \
631 filval : (double)(*(int *)&buf[i]) / HDpow(10.0f, D_val) + min); \
632 else if(sizeof(type)==sizeof(long)) \
633 for(i = 0; i < d_nelmts; i++) \
634 buf[i] = (type)((*(long *)&buf[i] == (long)(((unsigned long)1 << minbits) - 1)) ? \
635 filval : (double)(*(long *)&buf[i]) / HDpow(10.0f, D_val) + min); \
636 else if(sizeof(type)==sizeof(long long)) \
637 for(i = 0; i < d_nelmts; i++) \
638 buf[i] = (type)((*(long long *)&buf[i] == (long long)(((unsigned long long)1 << minbits) - 1)) ? \
639 filval : (double)(*(long long *)&buf[i]) / HDpow(10.0f, D_val) + min); \
640 else \
641 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
642 }
643
644 /* Modify values of data in postdecompression if fill value undefined for floating-point type */
645 #define H5Z_scaleoffset_modify_4(i, type, buf, d_nelmts, min, D_val) \
646 { \
647 if(sizeof(type)==sizeof(int)) \
648 for(i = 0; i < d_nelmts; i++) \
649 buf[i] = (type)((double)(*(int *)&buf[i]) / HDpow(10.0f, D_val) + min); \
650 else if(sizeof(type)==sizeof(long)) \
651 for(i = 0; i < d_nelmts; i++) \
652 buf[i] = (type)((double)(*(long *)&buf[i]) / HDpow(10.0f, D_val) + min); \
653 else if(sizeof(type)==sizeof(long long)) \
654 for(i = 0; i < d_nelmts; i++) \
655 buf[i] = (type)((double)(*(long long *)&buf[i]) / HDpow(10.0f, D_val) + min); \
656 else \
657 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \
658 }
659
660 /* Postdecompress for floating-point type using variable-minimum-bits method */
661 #define H5Z_scaleoffset_postdecompress_3(type, data, d_nelmts, filavail, cd_values, \
662 minbits, minval, D_val) \
663 { \
664 type *buf = (type *)data, filval = 0, min = 0; \
665 unsigned i; \
666 \
667 H5Z_scaleoffset_get_min(type, minval, min) \
668 \
669 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */ \
670 H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \
671 H5Z_scaleoffset_modify_3(i, type, buf, d_nelmts, filval, minbits, min, D_val) \
672 } else /* fill value undefined */ \
673 H5Z_scaleoffset_modify_4(i, type, buf, d_nelmts, min, D_val) \
674 }
675
676
677 /*-------------------------------------------------------------------------
678 * Function: H5Z_can_apply_scaleoffset
679 *
680 * Purpose: Check the parameters for scaleoffset compression for
681 * validity and whether they fit a particular dataset.
682 *
683 * Return: Success: Non-negative
684 * Failure: Negative
685 *
686 * Programmer: Xiaowen Wu
687 * Friday, February 4, 2005
688 *
689 * Modifications:
690 *
691 *-------------------------------------------------------------------------
692 */
693 static htri_t
H5Z_can_apply_scaleoffset(hid_t UNUSED dcpl_id,hid_t type_id,hid_t UNUSED space_id)694 H5Z_can_apply_scaleoffset(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id)
695 {
696 const H5T_t *type; /* Datatype */
697 H5T_class_t dtype_class; /* Datatype's class */
698 H5T_order_t dtype_order; /* Datatype's endianness order */
699 htri_t ret_value = TRUE; /* Return value */
700
701 FUNC_ENTER_NOAPI_NOINIT
702
703 /* Get datatype */
704 if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
705 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
706
707 /* Get datatype's class, for checking the "datatype class" */
708 if((dtype_class = H5T_get_class(type, TRUE)) == H5T_NO_CLASS)
709 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype class")
710
711 /* Get datatype's size, for checking the "datatype size" */
712 if(H5T_get_size(type) == 0)
713 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
714
715 if(dtype_class == H5T_INTEGER || dtype_class == H5T_FLOAT) {
716 /* Get datatype's endianness order */
717 if((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
718 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order")
719
720 /* Range check datatype's endianness order */
721 if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE)
722 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "bad datatype endianness order")
723 } else
724 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FALSE, "datatype class not supported by scaleoffset")
725
726 done:
727 FUNC_LEAVE_NOAPI(ret_value)
728 } /* end H5Z_can_apply_scaleoffset() */
729
730
731 /*-------------------------------------------------------------------------
732 * Function: H5Z_scaleoffset_get_type
733 *
734 * Purpose: Get the specific integer type based on datatype size and sign
735 * or floating-point type based on size
736 *
737 * Return: Success: id number of integer type
738 * Failure: 0
739 *
740 * Programmer: Xiaowen Wu
741 * Wednesday, April 13, 2005
742 *
743 * Modifications:
744 *
745 *-------------------------------------------------------------------------
746 */
747 static enum H5Z_scaleoffset_t
H5Z_scaleoffset_get_type(unsigned dtype_class,unsigned dtype_size,unsigned dtype_sign)748 H5Z_scaleoffset_get_type(unsigned dtype_class, unsigned dtype_size, unsigned dtype_sign)
749 {
750 enum H5Z_scaleoffset_t type = t_bad; /* integer type */
751 enum H5Z_scaleoffset_t ret_value; /* return value */
752
753 FUNC_ENTER_NOAPI_NOINIT
754
755 if(dtype_class==H5Z_SCALEOFFSET_CLS_INTEGER) {
756 if(dtype_sign==H5Z_SCALEOFFSET_SGN_NONE) { /* unsigned integer */
757 if (dtype_size == sizeof(unsigned char)) type = t_uchar;
758 else if(dtype_size == sizeof(unsigned short)) type = t_ushort;
759 else if(dtype_size == sizeof(unsigned int)) type = t_uint;
760 else if(dtype_size == sizeof(unsigned long)) type = t_ulong;
761 else if(dtype_size == sizeof(unsigned long long)) type = t_ulong_long;
762 else
763 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
764 }
765
766 if(dtype_sign==H5Z_SCALEOFFSET_SGN_2) { /* signed integer */
767 if (dtype_size == sizeof(signed char)) type = t_schar;
768 else if(dtype_size == sizeof(short)) type = t_short;
769 else if(dtype_size == sizeof(int)) type = t_int;
770 else if(dtype_size == sizeof(long)) type = t_long;
771 else if(dtype_size == sizeof(long long)) type = t_long_long;
772 else
773 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
774 }
775 }
776
777 if(dtype_class==H5Z_SCALEOFFSET_CLS_FLOAT) {
778 if(dtype_size == sizeof(float)) type = t_float;
779 else if(dtype_size == sizeof(double)) type = t_double;
780 else
781 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, t_bad, "cannot find matched memory dataype")
782 }
783
784 /* Set return value */
785 ret_value = type;
786
787 done:
788 FUNC_LEAVE_NOAPI(ret_value)
789 }
790
791
792 /*-------------------------------------------------------------------------
793 * Function: H5Z_scaleoffset_set_parms_fillval
794 *
795 * Purpose: Get the fill value of the dataset and store in cd_values[]
796 *
797 * Return: Success: Non-negative
798 * Failure: Negative
799 *
800 * Programmer: Xiaowen Wu
801 * Monday, March 7, 2005
802 *
803 *-------------------------------------------------------------------------
804 */
805 static herr_t
H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t * dcpl_plist,H5T_t * type,enum H5Z_scaleoffset_t scale_type,unsigned cd_values[],int need_convert,hid_t dxpl_id)806 H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist,
807 H5T_t *type, enum H5Z_scaleoffset_t scale_type,
808 unsigned cd_values[], int need_convert, hid_t dxpl_id)
809 {
810 herr_t ret_value = SUCCEED; /* Return value */
811
812 FUNC_ENTER_NOAPI_NOINIT
813
814 if(scale_type == t_uchar)
815 H5Z_scaleoffset_set_filval_3(unsigned char, dcpl_plist, type, cd_values, need_convert, dxpl_id)
816 else if(scale_type == t_ushort)
817 H5Z_scaleoffset_set_filval_1(unsigned short, dcpl_plist, type, cd_values, need_convert, dxpl_id)
818 else if(scale_type == t_uint)
819 H5Z_scaleoffset_set_filval_1(unsigned int, dcpl_plist, type, cd_values, need_convert, dxpl_id)
820 else if(scale_type == t_ulong)
821 H5Z_scaleoffset_set_filval_1(unsigned long, dcpl_plist, type, cd_values, need_convert, dxpl_id)
822 else if(scale_type == t_ulong_long)
823 H5Z_scaleoffset_set_filval_1(unsigned long long, dcpl_plist, type, cd_values, need_convert, dxpl_id)
824 else if(scale_type == t_schar)
825 H5Z_scaleoffset_set_filval_3(signed char, dcpl_plist, type, cd_values, need_convert, dxpl_id)
826 else if(scale_type == t_short)
827 H5Z_scaleoffset_set_filval_2(short, dcpl_plist, type, cd_values, need_convert, dxpl_id)
828 else if(scale_type == t_int)
829 H5Z_scaleoffset_set_filval_2(int, dcpl_plist, type, cd_values, need_convert, dxpl_id)
830 else if(scale_type == t_long)
831 H5Z_scaleoffset_set_filval_2(long, dcpl_plist, type, cd_values, need_convert, dxpl_id)
832 else if(scale_type == t_long_long)
833 H5Z_scaleoffset_set_filval_2(long long, dcpl_plist, type, cd_values, need_convert, dxpl_id)
834 else if(scale_type == t_float)
835 H5Z_scaleoffset_set_filval_4(float, dcpl_plist, type, cd_values, need_convert, dxpl_id)
836 else if(scale_type == t_double)
837 H5Z_scaleoffset_set_filval_4(double, dcpl_plist, type, cd_values, need_convert, dxpl_id)
838
839 done:
840 FUNC_LEAVE_NOAPI(ret_value)
841 } /* end H5Z_scaleoffset_set_parms_fillval() */
842
843
844 /*-------------------------------------------------------------------------
845 * Function: H5Z_set_local_scaleoffset
846 *
847 * Purpose: Set the "local" dataset parameters for scaleoffset
848 * compression.
849 *
850 * Return: Success: Non-negative
851 * Failure: Negative
852 *
853 * Programmer: Xiaowen Wu
854 * Friday, February 4, 2005
855 *
856 * Modifications:
857 *
858 *-------------------------------------------------------------------------
859 */
860 static herr_t
H5Z_set_local_scaleoffset(hid_t dcpl_id,hid_t type_id,hid_t space_id)861 H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id)
862 {
863 H5P_genplist_t *dcpl_plist; /* Property list pointer */
864 H5T_t *type; /* Datatype */
865 const H5S_t *ds; /* Dataspace */
866 unsigned flags; /* Filter flags */
867 size_t cd_nelmts = H5Z_SCALEOFFSET_USER_NPARMS; /* Number of filter parameters */
868 unsigned cd_values[H5Z_SCALEOFFSET_TOTAL_NPARMS]; /* Filter parameters */
869 hssize_t npoints; /* Number of points in the dataspace */
870 H5T_class_t dtype_class; /* Datatype's class */
871 H5T_order_t dtype_order; /* Datatype's endianness order */
872 size_t dtype_size; /* Datatype's size (in bytes) */
873 H5T_sign_t dtype_sign; /* Datatype's sign */
874 enum H5Z_scaleoffset_t scale_type; /* Specific datatype */
875 H5D_fill_value_t status; /* Status of fill value in property list */
876 herr_t ret_value = SUCCEED; /* Return value */
877
878 FUNC_ENTER_NOAPI_NOINIT
879
880 /* Get the plist structure */
881 if(NULL == (dcpl_plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE)))
882 HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
883
884 /* Get datatype */
885 if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE)))
886 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype")
887
888 #ifdef H5_CLEAR_MEMORY
889 /* Initialize the parameters to a known state */
890 HDmemset(cd_values, 0, sizeof(cd_values));
891 #endif /* H5_CLEAR_MEMORY */
892
893 /* Get the filter's current parameters */
894 if(H5P_get_filter_by_id(dcpl_plist, H5Z_FILTER_SCALEOFFSET, &flags, &cd_nelmts, cd_values, (size_t)0, NULL, NULL) < 0)
895 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get scaleoffset parameters")
896
897 /* Get dataspace */
898 if(NULL == (ds = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE)))
899 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space")
900
901 /* Get total number of elements in the chunk */
902 if((npoints = H5S_GET_EXTENT_NPOINTS(ds)) < 0)
903 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace")
904
905 /* Set "local" parameter for this dataset's number of elements */
906 H5_ASSIGN_OVERFLOW(cd_values[H5Z_SCALEOFFSET_PARM_NELMTS],npoints,hssize_t,unsigned);
907
908 /* Get datatype's class */
909 if((dtype_class = H5T_get_class(type, TRUE)) == H5T_NO_CLASS)
910 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype class")
911
912 /* Set "local" parameter for datatype's class */
913 switch(dtype_class) {
914 case H5T_INTEGER:
915 cd_values[H5Z_SCALEOFFSET_PARM_CLASS] = H5Z_SCALEOFFSET_CLS_INTEGER;
916 break;
917
918 case H5T_FLOAT:
919 cd_values[H5Z_SCALEOFFSET_PARM_CLASS] = H5Z_SCALEOFFSET_CLS_FLOAT;
920 break;
921
922 case H5T_NO_CLASS:
923 case H5T_TIME:
924 case H5T_STRING:
925 case H5T_BITFIELD:
926 case H5T_OPAQUE:
927 case H5T_COMPOUND:
928 case H5T_REFERENCE:
929 case H5T_ENUM:
930 case H5T_VLEN:
931 case H5T_ARRAY:
932 case H5T_NCLASSES:
933 default:
934 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "datatype class not supported by scaleoffset")
935 } /* end switch */
936
937 /* Get datatype's size */
938 if((dtype_size = H5T_get_size(type)) == 0)
939 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size")
940
941 /* Set "local" parameter for datatype size */
942 cd_values[H5Z_SCALEOFFSET_PARM_SIZE] = dtype_size;
943
944 if(dtype_class == H5T_INTEGER) {
945 /* Get datatype's sign */
946 if((dtype_sign = H5T_get_sign(type)) == H5T_SGN_ERROR)
947 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype sign")
948
949 /* Set "local" parameter for integer datatype sign */
950 switch(dtype_sign) {
951 case H5T_SGN_NONE:
952 cd_values[H5Z_SCALEOFFSET_PARM_SIGN] = H5Z_SCALEOFFSET_SGN_NONE;
953 break;
954
955 case H5T_SGN_2:
956 cd_values[H5Z_SCALEOFFSET_PARM_SIGN] = H5Z_SCALEOFFSET_SGN_2;
957 break;
958
959 case H5T_SGN_ERROR:
960 case H5T_NSGN:
961 default:
962 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad integer sign")
963 } /* end switch */
964 } /* end if */
965
966 /* Get datatype's endianness order */
967 if((dtype_order = H5T_get_order(type)) == H5T_ORDER_ERROR)
968 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
969
970 /* Set "local" parameter for datatype endianness */
971 switch(dtype_order) {
972 case H5T_ORDER_LE: /* Little-endian byte order */
973 cd_values[H5Z_SCALEOFFSET_PARM_ORDER] = H5Z_SCALEOFFSET_ORDER_LE;
974 break;
975
976 case H5T_ORDER_BE: /* Big-endian byte order */
977 cd_values[H5Z_SCALEOFFSET_PARM_ORDER] = H5Z_SCALEOFFSET_ORDER_BE;
978 break;
979
980 case H5T_ORDER_ERROR:
981 case H5T_ORDER_VAX:
982 case H5T_ORDER_NONE:
983 default:
984 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order")
985 } /* end switch */
986
987 /* Check whether fill value is defined for dataset */
988 if(H5P_fill_value_defined(dcpl_plist, &status) < 0)
989 HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to determine if fill value is defined")
990
991 /* Set local parameter for availability of fill value */
992 if(status == H5D_FILL_VALUE_UNDEFINED)
993 cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL] = H5Z_SCALEOFFSET_FILL_UNDEFINED;
994 else {
995 int need_convert = FALSE; /* Flag indicating convertion of byte order */
996
997 cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL] = H5Z_SCALEOFFSET_FILL_DEFINED;
998
999 /* Check if memory byte order matches dataset datatype byte order */
1000 if(H5T_native_order_g != dtype_order)
1001 need_convert = TRUE;
1002
1003 /* Before getting fill value, get its type */
1004 if((scale_type = H5Z_scaleoffset_get_type(cd_values[H5Z_SCALEOFFSET_PARM_CLASS],
1005 cd_values[H5Z_SCALEOFFSET_PARM_SIZE], cd_values[H5Z_SCALEOFFSET_PARM_SIGN])) == 0)
1006 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot use C integer datatype for cast")
1007
1008 /* Get dataset fill value and store in cd_values[] */
1009 if(H5Z_scaleoffset_set_parms_fillval(dcpl_plist, type, scale_type, cd_values, need_convert, H5AC_ind_dxpl_id) < 0)
1010 HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "unable to set fill value")
1011 } /* end else */
1012
1013 /* Modify the filter's parameters for this dataset */
1014 if(H5P_modify_filter(dcpl_plist, H5Z_FILTER_SCALEOFFSET, flags, (size_t)H5Z_SCALEOFFSET_TOTAL_NPARMS, cd_values) < 0)
1015 HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local scaleoffset parameters")
1016
1017 done:
1018 FUNC_LEAVE_NOAPI(ret_value)
1019 } /* end H5Z_set_local_scaleoffset() */
1020
1021
1022 /*-------------------------------------------------------------------------
1023 * Function: H5Z_filter_scaleoffset
1024 *
1025 * Purpose: Implement an I/O filter for storing packed integer
1026 * data using scale and offset method.
1027 *
1028 * Return: Success: Size of buffer filtered
1029 * Failure: 0
1030 *
1031 * Programmer: Xiaowen Wu
1032 * Monday, February 7, 2005
1033 *
1034 * Modifications:
1035 *
1036 *-------------------------------------------------------------------------
1037 */
1038 static size_t
H5Z_filter_scaleoffset(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],size_t nbytes,size_t * buf_size,void ** buf)1039 H5Z_filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
1040 size_t nbytes, size_t *buf_size, void **buf)
1041 {
1042 size_t ret_value = 0; /* return value */
1043 size_t size_out = 0; /* size of output buffer */
1044 unsigned d_nelmts = 0; /* number of data elements in the chunk */
1045 unsigned dtype_class; /* datatype class */
1046 unsigned dtype_sign; /* integer datatype sign */
1047 unsigned filavail; /* flag indicating if fill value is defined or not */
1048 H5Z_SO_scale_type_t scale_type = H5Z_SO_FLOAT_DSCALE;/* scale type */
1049 int scale_factor = 0; /* scale factor */
1050 double D_val = 0.0f; /* decimal scale factor */
1051 uint32_t minbits = 0; /* minimum number of bits to store values */
1052 unsigned long long minval= 0; /* minimum value of input buffer */
1053 enum H5Z_scaleoffset_t type; /* memory type corresponding to dataset datatype */
1054 int need_convert = FALSE; /* flag indicating convertion of byte order */
1055 unsigned char *outbuf = NULL; /* pointer to new output buffer */
1056 unsigned buf_offset = 21; /* buffer offset because of parameters stored in file */
1057 unsigned i; /* index */
1058 parms_atomic p; /* parameters needed for compress/decompress functions */
1059
1060 FUNC_ENTER_NOAPI_NOINIT
1061
1062 /* check arguments */
1063 if(cd_nelmts != H5Z_SCALEOFFSET_TOTAL_NPARMS)
1064 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scaleoffset number of parameters")
1065
1066 /* Check if memory byte order matches dataset datatype byte order */
1067 switch(H5T_native_order_g) {
1068 case H5T_ORDER_LE: /* memory is little-endian byte order */
1069 if(cd_values[H5Z_SCALEOFFSET_PARM_ORDER] == H5Z_SCALEOFFSET_ORDER_BE)
1070 need_convert = TRUE;
1071 break;
1072
1073 case H5T_ORDER_BE: /* memory is big-endian byte order */
1074 if(cd_values[H5Z_SCALEOFFSET_PARM_ORDER] == H5Z_SCALEOFFSET_ORDER_LE)
1075 need_convert = TRUE;
1076 break;
1077
1078 case H5T_ORDER_ERROR:
1079 case H5T_ORDER_VAX:
1080 case H5T_ORDER_NONE:
1081 default:
1082 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "bad H5T_NATIVE_INT endianness order")
1083 } /* end switch */
1084
1085 /* copy filter parameters to local variables */
1086 d_nelmts = cd_values[H5Z_SCALEOFFSET_PARM_NELMTS];
1087 dtype_class = cd_values[H5Z_SCALEOFFSET_PARM_CLASS];
1088 dtype_sign = cd_values[H5Z_SCALEOFFSET_PARM_SIGN];
1089 filavail = cd_values[H5Z_SCALEOFFSET_PARM_FILAVAIL];
1090 scale_factor = (int)cd_values[H5Z_SCALEOFFSET_PARM_SCALEFACTOR];
1091 scale_type = (H5Z_SO_scale_type_t)cd_values[H5Z_SCALEOFFSET_PARM_SCALETYPE];
1092
1093 /* check and assign proper values set by user to related parameters
1094 * scale type can be H5Z_SO_FLOAT_DSCALE (0), H5Z_SO_FLOAT_ESCALE (1) or H5Z_SO_INT (other)
1095 * H5Z_SO_FLOAT_DSCALE : floating-point type, variable-minimum-bits method,
1096 * scale factor is decimal scale factor
1097 * H5Z_SO_FLOAT_ESCALE : floating-point type, fixed-minimum-bits method,
1098 * scale factor is the fixed minimum number of bits
1099 * H5Z_SO_INT : integer type, scale_factor is minimum number of bits
1100 */
1101 if(dtype_class==H5Z_SCALEOFFSET_CLS_FLOAT) { /* floating-point type */
1102 if(scale_type!=H5Z_SO_FLOAT_DSCALE && scale_type!=H5Z_SO_FLOAT_ESCALE)
1103 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scale type")
1104 }
1105
1106 if(dtype_class==H5Z_SCALEOFFSET_CLS_INTEGER) { /* integer type */
1107 if(scale_type!=H5Z_SO_INT)
1108 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid scale type")
1109
1110 /* if scale_factor is less than 0 for integer, library will reset it to 0
1111 * in this case, library will calculate the minimum-bits
1112 */
1113 if(scale_factor < 0) scale_factor = 0;
1114 }
1115
1116 /* fixed-minimum-bits method is not implemented and is forbidden */
1117 if(scale_type==H5Z_SO_FLOAT_ESCALE)
1118 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "E-scaling method not supported")
1119
1120 if(scale_type==H5Z_SO_FLOAT_DSCALE) { /* floating-point type, variable-minimum-bits */
1121 D_val = (double)scale_factor;
1122 } else { /* integer type, or floating-point type with fixed-minimum-bits method */
1123 if(scale_factor > (int)(cd_values[H5Z_SCALEOFFSET_PARM_SIZE] * 8))
1124 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "minimum number of bits exceeds maximum")
1125
1126 /* no need to process data */
1127 if(scale_factor == (int)(cd_values[H5Z_SCALEOFFSET_PARM_SIZE] * 8)) {
1128 ret_value = *buf_size;
1129 goto done;
1130 }
1131 minbits = (uint32_t)scale_factor;
1132 }
1133
1134 /* prepare parameters to pass to compress/decompress functions */
1135 p.size = cd_values[H5Z_SCALEOFFSET_PARM_SIZE];
1136 p.mem_order = H5T_native_order_g;
1137
1138 /* input; decompress */
1139 if (flags & H5Z_FLAG_REVERSE) {
1140 /* retrieve values of minbits and minval from input compressed buffer
1141 * retrieve them corresponding to how they are stored during compression
1142 */
1143 uint32_t minbits_mask = 0;
1144 unsigned long long minval_mask = 0;
1145 unsigned minval_size = 0;
1146
1147 minbits = 0;
1148 for(i = 0; i < 4; i++) {
1149 minbits_mask = ((unsigned char *)*buf)[i];
1150 minbits_mask <<= i*8;
1151 minbits |= minbits_mask;
1152 }
1153
1154 /* retrieval of minval takes into consideration situation where sizeof
1155 * unsigned long long (datatype of minval) may change from compression
1156 * to decompression, only smaller size is used
1157 */
1158 minval_size = sizeof(unsigned long long) <= ((unsigned char *)*buf)[4] ?
1159 sizeof(unsigned long long) : ((unsigned char *)*buf)[4];
1160 minval = 0;
1161 for(i = 0; i < minval_size; i++) {
1162 minval_mask = ((unsigned char *)*buf)[5+i];
1163 minval_mask <<= i*8;
1164 minval |= minval_mask;
1165 }
1166
1167 HDassert(minbits <= p.size * 8);
1168 p.minbits = minbits;
1169
1170 /* calculate size of output buffer after decompression */
1171 size_out = d_nelmts * p.size;
1172
1173 /* allocate memory space for decompressed buffer */
1174 if(NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out)))
1175 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for scaleoffset decompression")
1176
1177 /* special case: minbits equal to full precision */
1178 if(minbits == p.size * 8) {
1179 HDmemcpy(outbuf, (unsigned char*)(*buf)+buf_offset, size_out);
1180
1181 /* convert to dataset datatype endianness order if needed */
1182 if(need_convert)
1183 H5Z_scaleoffset_convert(outbuf, d_nelmts, p.size);
1184
1185 *buf = outbuf;
1186 outbuf = NULL;
1187 *buf_size = size_out;
1188 ret_value = size_out;
1189 goto done;
1190 }
1191
1192 /* decompress the buffer if minbits not equal to zero */
1193 if(minbits != 0)
1194 H5Z_scaleoffset_decompress(outbuf, d_nelmts, (unsigned char*)(*buf)+buf_offset, p);
1195 else {
1196 /* fill value is not defined and all data elements have the same value */
1197 for(i = 0; i < size_out; i++) outbuf[i] = 0;
1198 }
1199
1200 /* before postprocess, get memory type */
1201 if((type = H5Z_scaleoffset_get_type(dtype_class, (unsigned)p.size, dtype_sign)) == 0)
1202 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "cannot use C integer datatype for cast")
1203
1204 /* postprocess after decompression */
1205 if(dtype_class==H5Z_SCALEOFFSET_CLS_INTEGER)
1206 H5Z_scaleoffset_postdecompress_i(outbuf, d_nelmts, type, filavail,
1207 cd_values, minbits, minval);
1208
1209 if(dtype_class==H5Z_SCALEOFFSET_CLS_FLOAT)
1210 if(scale_type==0) { /* variable-minimum-bits method */
1211 if(H5Z_scaleoffset_postdecompress_fd(outbuf, d_nelmts, type, filavail,
1212 cd_values, minbits, minval, D_val)==FAIL)
1213 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "post-decompression failed")
1214 }
1215
1216 /* after postprocess, convert to dataset datatype endianness order if needed */
1217 if(need_convert)
1218 H5Z_scaleoffset_convert(outbuf, d_nelmts, p.size);
1219 }
1220 /* output; compress */
1221 else {
1222 HDassert(nbytes == d_nelmts * p.size);
1223
1224 /* before preprocess, convert to memory endianness order if needed */
1225 if(need_convert)
1226 H5Z_scaleoffset_convert(*buf, d_nelmts, p.size);
1227
1228 /* before preprocess, get memory type */
1229 if((type = H5Z_scaleoffset_get_type(dtype_class, (unsigned)p.size, dtype_sign))==0)
1230 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "cannot use C integer datatype for cast")
1231
1232 /* preprocess before compression */
1233 if(dtype_class==H5Z_SCALEOFFSET_CLS_INTEGER)
1234 H5Z_scaleoffset_precompress_i(*buf, d_nelmts, type, filavail,
1235 cd_values, &minbits, &minval);
1236
1237 if(dtype_class==H5Z_SCALEOFFSET_CLS_FLOAT)
1238 if(scale_type==0) { /* variable-minimum-bits method */
1239 if(H5Z_scaleoffset_precompress_fd(*buf, d_nelmts, type, filavail,
1240 cd_values, &minbits, &minval, D_val)==FAIL)
1241 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, 0, "pre-compression failed")
1242 }
1243
1244 HDassert(minbits <= p.size * 8);
1245
1246 /* calculate buffer size after compression
1247 * minbits and minval are stored in the front of the compressed buffer
1248 */
1249 p.minbits = minbits;
1250 size_out = buf_offset + nbytes * p.minbits / (p.size * 8) + 1; /* may be 1 larger */
1251
1252 /* allocate memory space for compressed buffer */
1253 if(NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out)))
1254 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for scaleoffset compression")
1255
1256 /* store minbits and minval in the front of output compressed buffer
1257 * store byte by byte from least significant byte to most significant byte
1258 * constant buffer size (21 bytes) is left for these two parameters
1259 * 4 bytes for minbits, 1 byte for size of minval, 16 bytes for minval
1260 */
1261 for(i = 0; i < 4; i++)
1262 ((unsigned char *)outbuf)[i] = (unsigned char)((minbits & ((uint32_t)0xff << i*8)) >> i*8);
1263
1264 ((unsigned char *)outbuf)[4] = sizeof(unsigned long long);
1265
1266 for(i = 0; i < sizeof(unsigned long long); i++)
1267 ((unsigned char *)outbuf)[5+i] = (unsigned char)((minval & ((unsigned long long)0xff << i*8)) >> i*8);
1268
1269 #ifdef H5_CLEAR_MEMORY
1270 /* Zero out remaining, unused bytes */
1271 /* (Looks like an error in the original determination of how many
1272 * bytes would be needed for parameters. - QAK, 2010/08/19)
1273 */
1274 HDmemset(outbuf + 13, 0, (size_t)8);
1275 #endif /* H5_CLEAR_MEMORY */
1276
1277 /* special case: minbits equal to full precision */
1278 if(minbits == p.size * 8) {
1279 HDmemcpy(outbuf + buf_offset, *buf, nbytes);
1280 *buf = outbuf;
1281 outbuf = NULL;
1282 *buf_size = size_out;
1283 ret_value = buf_offset + nbytes;
1284 goto done;
1285 }
1286
1287 /* compress the buffer if minbits not equal to zero
1288 * minbits equal to zero only when fill value is not defined and
1289 * all data elements have the same value
1290 */
1291 if(minbits != 0)
1292 H5Z_scaleoffset_compress((unsigned char *)*buf, d_nelmts, outbuf + buf_offset, size_out - buf_offset, p);
1293 }
1294
1295 /* free the input buffer */
1296 H5MM_xfree(*buf);
1297
1298 /* set return values */
1299 *buf = outbuf;
1300 outbuf = NULL;
1301 *buf_size = size_out;
1302 ret_value = size_out;
1303
1304 done:
1305 if(outbuf)
1306 H5MM_xfree(outbuf);
1307 FUNC_LEAVE_NOAPI(ret_value)
1308 }
1309
1310 /* ============ Scaleoffset Algorithm ===============================================
1311 * assume one byte has 8 bit
1312 * assume padding bit is 0
1313 * assume size of unsigned char is one byte
1314 * assume one data item of certain datatype is stored continously in bytes
1315 * atomic datatype is treated on byte basis
1316 */
1317
1318
1319 /* change byte order of input buffer either from little-endian to big-endian
1320 * or from big-endian to little-endian 2/21/2005
1321 */
1322 static void
H5Z_scaleoffset_convert(void * buf,unsigned d_nelmts,size_t dtype_size)1323 H5Z_scaleoffset_convert(void *buf, unsigned d_nelmts, size_t dtype_size)
1324 {
1325 if(dtype_size > 1) {
1326 unsigned i, j;
1327 unsigned char *buffer, temp;
1328
1329 buffer = (unsigned char *)buf;
1330 for(i = 0; i < d_nelmts * dtype_size; i += dtype_size)
1331 for(j = 0; j < dtype_size / 2; j++) {
1332 /* swap pair of bytes */
1333 temp = buffer[i + j];
1334 buffer[i + j] = buffer[i + dtype_size - 1 - j];
1335 buffer[i + dtype_size - 1 - j] = temp;
1336 } /* end for */
1337 } /* end if */
1338 } /* end H5Z_scaleoffset_convert() */
1339
1340 /* Round a floating-point value to the nearest integer value 4/19/05 */
1341 /* rounding to the bigger absolute value if val is in the middle,
1342 0.5 -> 1, -0.5 ->-1
1343 5/9/05, KY */
1344 static double
H5Z_scaleoffset_rnd(double val)1345 H5Z_scaleoffset_rnd(double val)
1346 {
1347 double u_val, l_val;
1348
1349 u_val = HDceil(val);
1350 l_val = HDfloor(val);
1351
1352 if(val > 0) {
1353 if((u_val - val) <= (val - l_val))
1354 return u_val;
1355 else
1356 return l_val;
1357 } /* end if */
1358 else {
1359 if((val - l_val) <= (u_val - val))
1360 return l_val;
1361 else
1362 return u_val;
1363 }
1364 } /* H5Z_scaleoffset_rnd() */
1365
1366 /* return ceiling of floating-point log2 function
1367 * receive unsigned integer as argument 3/10/2005
1368 */
1369 static unsigned
H5Z_scaleoffset_log2(unsigned long long num)1370 H5Z_scaleoffset_log2(unsigned long long num)
1371 {
1372 unsigned v = 0;
1373 unsigned long long lower_bound = 1; /* is power of 2, largest value <= num */
1374 unsigned long long val = num;
1375
1376 while(val >>= 1) {
1377 v++;
1378 lower_bound <<= 1;
1379 }
1380
1381 if(num == lower_bound)
1382 return v;
1383 else
1384 return v + 1;
1385 }
1386
1387 /* precompress for integer type */
1388 static void
H5Z_scaleoffset_precompress_i(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t * minbits,unsigned long long * minval)1389 H5Z_scaleoffset_precompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1390 unsigned filavail, const unsigned cd_values[], uint32_t *minbits, unsigned long long *minval)
1391 {
1392 if(type == t_uchar)
1393 H5Z_scaleoffset_precompress_1(unsigned char, data, d_nelmts,
1394 filavail, cd_values, minbits, minval)
1395 else if(type == t_ushort)
1396 H5Z_scaleoffset_precompress_1(unsigned short, data, d_nelmts,
1397 filavail, cd_values, minbits, minval)
1398 else if(type == t_uint)
1399 H5Z_scaleoffset_precompress_1(unsigned int, data, d_nelmts,
1400 filavail, cd_values, minbits, minval)
1401 else if(type == t_ulong)
1402 H5Z_scaleoffset_precompress_1(unsigned long, data, d_nelmts,
1403 filavail, cd_values, minbits, minval)
1404 else if(type == t_ulong_long)
1405 H5Z_scaleoffset_precompress_1(unsigned long long, data, d_nelmts,
1406 filavail, cd_values, minbits, minval)
1407 else if(type == t_schar) {
1408 signed char *buf = (signed char *)data, min = 0, max = 0, filval = 0;
1409 unsigned char span;
1410 unsigned i;
1411
1412 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */
1413 H5Z_scaleoffset_get_filval_1(signed char, cd_values, filval);
1414 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */
1415 H5Z_scaleoffset_max_min_1(i, d_nelmts, buf, filval, max, min)
1416 if((unsigned char)(max - min) > (unsigned char)(~(unsigned char)0 - 2)) {
1417 *minbits = sizeof(signed char)*8;
1418 return;
1419 }
1420 span = (unsigned char)(max - min + 1);
1421 *minbits = H5Z_scaleoffset_log2((unsigned long long)(span+1));
1422 } else /* minbits already set, only calculate min */
1423 H5Z_scaleoffset_min_1(i, d_nelmts, buf, filval, min)
1424 if(*minbits != sizeof(signed char)*8) /* change values if minbits != full precision */
1425 for(i = 0; i < d_nelmts; i++)
1426 buf[i] = (signed char)((buf[i] == filval) ? (((unsigned char)1 << *minbits) - 1) : (buf[i] - min));
1427 } else { /* fill value undefined */
1428 if(*minbits == H5Z_SO_INT_MINBITS_DEFAULT) { /* minbits not set yet, calculate max, min, and minbits */
1429 H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min)
1430 if((unsigned char)(max - min) > (unsigned char)(~(unsigned char)0 - 2)) {
1431 *minbits = sizeof(signed char)*8;
1432 *minval = (unsigned long long)min;
1433 return;
1434 }
1435 span = (unsigned char)(max - min + 1);
1436 *minbits = H5Z_scaleoffset_log2((unsigned long long)span);
1437 } else /* minbits already set, only calculate min */
1438 H5Z_scaleoffset_min_2(i, d_nelmts, buf, min)
1439 if(*minbits != sizeof(signed char) * 8) /* change values if minbits != full precision */
1440 for(i = 0; i < d_nelmts; i++)
1441 buf[i] = (signed char)(buf[i] - min);
1442 }
1443 *minval = (unsigned long long)min;
1444 }
1445 else if(type == t_short)
1446 H5Z_scaleoffset_precompress_2(short, data, d_nelmts,
1447 filavail, cd_values, minbits, minval)
1448 else if(type == t_int)
1449 H5Z_scaleoffset_precompress_2(int, data, d_nelmts,
1450 filavail, cd_values, minbits, minval)
1451 else if(type == t_long)
1452 H5Z_scaleoffset_precompress_2(long, data, d_nelmts,
1453 filavail, cd_values, minbits, minval)
1454 else if(type == t_long_long)
1455 H5Z_scaleoffset_precompress_2(long long, data, d_nelmts,
1456 filavail, cd_values, minbits, minval)
1457 }
1458
1459 /* postdecompress for integer type */
1460 static void
H5Z_scaleoffset_postdecompress_i(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t minbits,unsigned long long minval)1461 H5Z_scaleoffset_postdecompress_i(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1462 unsigned filavail, const unsigned cd_values[], uint32_t minbits, unsigned long long minval)
1463 {
1464 long long sminval = *(long long*)&minval; /* for signed integer types */
1465
1466 if(type == t_uchar)
1467 H5Z_scaleoffset_postdecompress_1(unsigned char, data, d_nelmts, filavail,
1468 cd_values, minbits, minval)
1469 else if(type == t_ushort)
1470 H5Z_scaleoffset_postdecompress_1(unsigned short, data, d_nelmts, filavail,
1471 cd_values, minbits, minval)
1472 else if(type == t_uint)
1473 H5Z_scaleoffset_postdecompress_1(unsigned int, data, d_nelmts, filavail,
1474 cd_values, minbits, minval)
1475 else if(type == t_ulong)
1476 H5Z_scaleoffset_postdecompress_1(unsigned long, data, d_nelmts, filavail,
1477 cd_values, minbits, minval)
1478 else if(type == t_ulong_long)
1479 H5Z_scaleoffset_postdecompress_1(unsigned long long, data, d_nelmts, filavail,
1480 cd_values, minbits, minval)
1481 else if(type == t_schar) {
1482 signed char *buf = (signed char *)data, filval = 0;
1483 unsigned i;
1484
1485 if(filavail == H5Z_SCALEOFFSET_FILL_DEFINED) { /* fill value defined */
1486 H5Z_scaleoffset_get_filval_1(signed char, cd_values, filval)
1487 for(i = 0; i < d_nelmts; i++)
1488 buf[i] = (signed char)((buf[i] == (((unsigned char)1 << minbits) - 1)) ? filval : (buf[i] + sminval));
1489 } else /* fill value undefined */
1490 for(i = 0; i < d_nelmts; i++)
1491 buf[i] = (signed char)(buf[i] + sminval);
1492 }
1493 else if(type == t_short)
1494 H5Z_scaleoffset_postdecompress_2(short, data, d_nelmts, filavail,
1495 cd_values, minbits, sminval)
1496 else if(type == t_int)
1497 H5Z_scaleoffset_postdecompress_2(int, data, d_nelmts, filavail,
1498 cd_values, minbits, sminval)
1499 else if(type == t_long)
1500 H5Z_scaleoffset_postdecompress_2(long, data, d_nelmts, filavail,
1501 cd_values, minbits, sminval)
1502 else if(type == t_long_long)
1503 H5Z_scaleoffset_postdecompress_2(long long, data, d_nelmts, filavail,
1504 cd_values, minbits, sminval)
1505 }
1506
1507 /* precompress for floating-point type, variable-minimum-bits method
1508 success: non-negative, failure: negative 4/15/05 */
1509 static herr_t
H5Z_scaleoffset_precompress_fd(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t * minbits,unsigned long long * minval,double D_val)1510 H5Z_scaleoffset_precompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1511 unsigned filavail, const unsigned cd_values[], uint32_t *minbits,
1512 unsigned long long *minval, double D_val)
1513 {
1514 herr_t ret_value=SUCCEED; /* Return value */
1515
1516 FUNC_ENTER_NOAPI_NOINIT
1517
1518 if(type == t_float)
1519 H5Z_scaleoffset_precompress_3(float, data, d_nelmts,
1520 filavail, cd_values, minbits, minval, D_val)
1521 else if(type == t_double)
1522 H5Z_scaleoffset_precompress_3(double, data, d_nelmts,
1523 filavail, cd_values, minbits, minval, D_val)
1524
1525 done:
1526 FUNC_LEAVE_NOAPI(ret_value)
1527 }
1528
1529 /* postdecompress for floating-point type, variable-minimum-bits method
1530 success: non-negative, failure: negative 4/15/05 */
1531 static herr_t
H5Z_scaleoffset_postdecompress_fd(void * data,unsigned d_nelmts,enum H5Z_scaleoffset_t type,unsigned filavail,const unsigned cd_values[],uint32_t minbits,unsigned long long minval,double D_val)1532 H5Z_scaleoffset_postdecompress_fd(void *data, unsigned d_nelmts, enum H5Z_scaleoffset_t type,
1533 unsigned filavail, const unsigned cd_values[], uint32_t minbits,
1534 unsigned long long minval, double D_val)
1535 {
1536 long long sminval = (long long)minval; /* for signed integer types */
1537 herr_t ret_value=SUCCEED; /* Return value */
1538
1539 FUNC_ENTER_NOAPI_NOINIT
1540
1541 if(type == t_float)
1542 H5Z_scaleoffset_postdecompress_3(float, data, d_nelmts, filavail,
1543 cd_values, minbits, sminval, D_val)
1544 else if(type == t_double)
1545 H5Z_scaleoffset_postdecompress_3(double, data, d_nelmts, filavail,
1546 cd_values, minbits, sminval, D_val)
1547
1548 done:
1549 FUNC_LEAVE_NOAPI(ret_value)
1550 }
1551
1552 static void
H5Z_scaleoffset_next_byte(size_t * j,unsigned * buf_len)1553 H5Z_scaleoffset_next_byte(size_t *j, unsigned *buf_len)
1554 {
1555 ++(*j);
1556 *buf_len = 8 * sizeof(unsigned char);
1557 }
1558
1559 static void
H5Z_scaleoffset_decompress_one_byte(unsigned char * data,size_t data_offset,unsigned k,unsigned begin_i,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p,unsigned dtype_len)1560 H5Z_scaleoffset_decompress_one_byte(unsigned char *data, size_t data_offset,
1561 unsigned k, unsigned begin_i, unsigned char *buffer, size_t *j, unsigned *buf_len,
1562 parms_atomic p, unsigned dtype_len)
1563 {
1564 unsigned dat_len; /* dat_len is the number of bits to be copied in each data byte */
1565 unsigned char val; /* value to be copied in each data byte */
1566
1567 /* initialize value and bits of unsigned char to be copied */
1568 val = buffer[*j];
1569 if(k == begin_i)
1570 dat_len = 8 - (dtype_len - p.minbits) % 8;
1571 else
1572 dat_len = 8;
1573
1574 if(*buf_len > dat_len) {
1575 data[data_offset + k] = (unsigned char)((val >> (*buf_len - dat_len)) & ~(~0 << dat_len));
1576 *buf_len -= dat_len;
1577 } /* end if */
1578 else {
1579 data[data_offset + k] = (unsigned char)((val & ~(~0 << *buf_len)) << (dat_len - *buf_len));
1580 dat_len -= *buf_len;
1581 H5Z_scaleoffset_next_byte(j, buf_len);
1582 if(dat_len == 0)
1583 return;
1584
1585 val = buffer[*j];
1586 data[data_offset + k] |= (unsigned char)((val >> (*buf_len - dat_len)) & ~(~0 << dat_len));
1587 *buf_len -= dat_len;
1588 } /* end else */
1589 }
1590
1591 static void
H5Z_scaleoffset_decompress_one_atomic(unsigned char * data,size_t data_offset,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p)1592 H5Z_scaleoffset_decompress_one_atomic(unsigned char *data, size_t data_offset,
1593 unsigned char *buffer, size_t *j, unsigned *buf_len, parms_atomic p)
1594 {
1595 /* begin_i: the index of byte having first significant bit */
1596 unsigned begin_i;
1597 unsigned dtype_len;
1598 int k;
1599
1600 HDassert(p.minbits > 0);
1601
1602 dtype_len = p.size * 8;
1603
1604 if(p.mem_order == H5Z_SCALEOFFSET_ORDER_LE) { /* little endian */
1605 begin_i = p.size - 1 - (dtype_len - p.minbits) / 8;
1606
1607 for(k = (int)begin_i; k >= 0; k--)
1608 H5Z_scaleoffset_decompress_one_byte(data, data_offset, (unsigned)k, begin_i,
1609 buffer, j, buf_len, p, dtype_len);
1610 }
1611 else { /* big endian */
1612 HDassert(p.mem_order == H5Z_SCALEOFFSET_ORDER_BE);
1613
1614 begin_i = (dtype_len - p.minbits) / 8;
1615
1616 for(k = (int)begin_i; k <= (int)(p.size - 1); k++)
1617 H5Z_scaleoffset_decompress_one_byte(data, data_offset, (unsigned)k, begin_i,
1618 buffer, j, buf_len, p, dtype_len);
1619 }
1620 }
1621
1622 static void
H5Z_scaleoffset_decompress(unsigned char * data,unsigned d_nelmts,unsigned char * buffer,parms_atomic p)1623 H5Z_scaleoffset_decompress(unsigned char *data, unsigned d_nelmts,
1624 unsigned char *buffer, parms_atomic p)
1625 {
1626 /* i: index of data, j: index of buffer,
1627 buf_len: number of bits to be filled in current byte */
1628 size_t i, j;
1629 unsigned buf_len;
1630
1631 /* must initialize to zeros */
1632 for(i = 0; i < d_nelmts*p.size; i++)
1633 data[i] = 0;
1634
1635 /* initialization before the loop */
1636 j = 0;
1637 buf_len = sizeof(unsigned char) * 8;
1638
1639 /* decompress */
1640 for(i = 0; i < d_nelmts; i++)
1641 H5Z_scaleoffset_decompress_one_atomic(data, i * p.size, buffer, &j, &buf_len, p);
1642 }
1643
1644 static void
H5Z_scaleoffset_compress_one_byte(unsigned char * data,size_t data_offset,unsigned k,unsigned begin_i,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p,unsigned dtype_len)1645 H5Z_scaleoffset_compress_one_byte(unsigned char *data, size_t data_offset,
1646 unsigned k, unsigned begin_i, unsigned char *buffer, size_t *j, unsigned *buf_len,
1647 parms_atomic p, unsigned dtype_len)
1648 {
1649 unsigned dat_len; /* dat_len is the number of bits to be copied in each data byte */
1650 unsigned char val; /* value to be copied in each data byte */
1651
1652 /* initialize value and bits of unsigned char to be copied */
1653 val = data[data_offset + k];
1654 if(k == begin_i)
1655 dat_len = 8 - (dtype_len - p.minbits) % 8;
1656 else
1657 dat_len = 8;
1658
1659 if(*buf_len > dat_len) {
1660 buffer[*j] |= (unsigned char)((val & ~(~0 << dat_len)) << (*buf_len - dat_len));
1661 *buf_len -= dat_len;
1662 } else {
1663 buffer[*j] |= (unsigned char)((val >> (dat_len - *buf_len)) & ~(~0 << *buf_len));
1664 dat_len -= *buf_len;
1665 H5Z_scaleoffset_next_byte(j, buf_len);
1666 if(dat_len == 0)
1667 return;
1668
1669 buffer[*j] = (unsigned char)((val & ~(~0 << dat_len)) << (*buf_len - dat_len));
1670 *buf_len -= dat_len;
1671 } /* end else */
1672 }
1673
1674 static void
H5Z_scaleoffset_compress_one_atomic(unsigned char * data,size_t data_offset,unsigned char * buffer,size_t * j,unsigned * buf_len,parms_atomic p)1675 H5Z_scaleoffset_compress_one_atomic(unsigned char *data, size_t data_offset,
1676 unsigned char *buffer, size_t *j, unsigned *buf_len, parms_atomic p)
1677 {
1678 /* begin_i: the index of byte having first significant bit */
1679 unsigned begin_i;
1680 unsigned dtype_len;
1681 int k;
1682
1683 HDassert(p.minbits > 0);
1684
1685 dtype_len = p.size * 8;
1686
1687 if(p.mem_order == H5Z_SCALEOFFSET_ORDER_LE) { /* little endian */
1688 begin_i = p.size - 1 - (dtype_len - p.minbits) / 8;
1689
1690 for(k = (int)begin_i; k >= 0; k--)
1691 H5Z_scaleoffset_compress_one_byte(data, data_offset, (unsigned)k, begin_i,
1692 buffer, j, buf_len, p, dtype_len);
1693 }
1694 else { /* big endian */
1695 HDassert(p.mem_order == H5Z_SCALEOFFSET_ORDER_BE);
1696 begin_i = (dtype_len - p.minbits) / 8;
1697
1698 for(k = (int)begin_i; k <= (int)(p.size - 1); k++)
1699 H5Z_scaleoffset_compress_one_byte(data, data_offset, (unsigned)k, begin_i,
1700 buffer, j, buf_len, p, dtype_len);
1701 }
1702 }
1703
1704 static void
H5Z_scaleoffset_compress(unsigned char * data,unsigned d_nelmts,unsigned char * buffer,size_t buffer_size,parms_atomic p)1705 H5Z_scaleoffset_compress(unsigned char *data, unsigned d_nelmts,
1706 unsigned char *buffer, size_t buffer_size, parms_atomic p)
1707 {
1708 /* i: index of data, j: index of buffer,
1709 buf_len: number of bits to be filled in current byte */
1710 size_t i, j;
1711 unsigned buf_len;
1712
1713 /* must initialize buffer to be zeros */
1714 for(j = 0; j < buffer_size; j++)
1715 buffer[j] = 0;
1716
1717 /* initialization before the loop */
1718 j = 0;
1719 buf_len = sizeof(unsigned char) * 8;
1720
1721 /* compress */
1722 for(i = 0; i < d_nelmts; i++)
1723 H5Z_scaleoffset_compress_one_atomic(data, i * p.size, buffer, &j, &buf_len, p);
1724 }
1725 #endif /* H5_HAVE_FILTER_SCALEOFFSET */
1726
1727