1 /*===========================================================================
2 *
3 * PUBLIC DOMAIN NOTICE
4 * National Center for Biotechnology Information
5 *
6 * This software/database is a "United States Government Work" under the
7 * terms of the United States Copyright Act. It was written as part of
8 * the author's official duties as a United States Government employee and
9 * thus cannot be copyrighted. This software/database is freely available
10 * to the public for use. The National Library of Medicine and the U.S.
11 * Government have not placed any restriction on its use or reproduction.
12 *
13 * Although all reasonable efforts have been taken to ensure the accuracy
14 * and reliability of the software and data, the NLM and the U.S.
15 * Government do not and cannot warrant the performance or results that
16 * may be obtained by using this software or data. The NLM and the U.S.
17 * Government disclaim all warranties, express or implied, including
18 * warranties of performance, merchantability or fitness for any particular
19 * purpose.
20 *
21 * Please cite the author in any work or product based on this material.
22 *
23 * ===========================================================================
24 *
25 */
26 #include <vdb/extern.h>
27 #include <compiler.h>
28 #include <klib/sort.h>
29 #include <klib/defs.h>
30 #include <klib/rc.h>
31 #include <vdb/xform.h>
32 #include <vdb/schema.h>
33 #include <sysalloc.h>
34
35 #include <byteswap.h>
36 #include <os-native.h>
37
38 #include <stdint.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <float.h>
42 #include <limits.h>
43 #include <math.h>
44
45 #include <zlib.h>
46
47 #include <stdio.h>
48 #include <assert.h>
49
50 typedef struct {
51 size_t size;
52 size_t used;
53 void *buf;
54 } szbuf;
55
zlib_compress(szbuf * dst,const void * src,size_t ssize,int32_t strategy,int32_t level)56 static rc_t zlib_compress(szbuf *dst, const void *src, size_t ssize, int32_t strategy, int32_t level) {
57 z_stream s;
58 int zr;
59 rc_t rc = 0;
60
61 memset(&s, 0, sizeof(s));
62 s.next_in = (void *)src;
63 s.avail_in = (uInt)ssize;
64 s.next_out = dst->buf;
65 s.avail_out = (uInt)dst->size;
66
67 dst->used = 0;
68
69 zr = deflateInit2(&s, level, Z_DEFLATED, -15, 9, strategy);
70 switch (zr) {
71 case 0:
72 break;
73 case Z_MEM_ERROR:
74 return RC(rcVDB, rcFunction, rcExecuting, rcMemory, rcExhausted);
75 case Z_STREAM_ERROR:
76 return RC(rcVDB, rcFunction, rcExecuting, rcParam, rcInvalid);
77 default:
78 return RC(rcVDB, rcFunction, rcExecuting, rcSelf, rcUnexpected);
79 }
80 zr = deflate(&s, Z_FINISH);
81 switch (zr) {
82 case Z_STREAM_END:
83 break;
84 case Z_OK:
85 s.total_out = 0;
86 break;
87 default:
88 rc = RC(rcVDB, rcFunction, rcExecuting, rcSelf, rcUnexpected);
89 break;
90 }
91 zr = deflateEnd(&s);
92 if ( zr != Z_OK && s.total_out != 0 )
93 rc = RC(rcVDB, rcFunction, rcExecuting, rcSelf, rcUnexpected);
94 if ( rc == 0 ) {
95 dst->used = (uint32_t)s.total_out;
96 }
97 return rc;
98 }
99
zlib_decompress(void * dst,size_t dsize,size_t * psize,const void * src,size_t ssize)100 static rc_t zlib_decompress(void *dst, size_t dsize, size_t *psize, const void *src, size_t ssize) {
101 z_stream s;
102 int zr;
103 rc_t rc;
104
105 memset(&s, 0, sizeof(s));
106 s.next_in = (void *)src;
107 s.avail_in = (uInt)ssize;
108 s.next_out = dst;
109 s.avail_out = (uInt)dsize;
110
111 zr = inflateInit2(&s, -15);
112 switch (zr) {
113 case 0:
114 break;
115 case Z_MEM_ERROR:
116 return RC(rcVDB, rcFunction, rcExecuting, rcMemory, rcExhausted);
117 default:
118 return RC(rcVDB, rcFunction, rcExecuting, rcNoObj, rcUnexpected);
119 }
120 zr = inflate(&s, Z_FINISH);
121 switch (zr) {
122 case Z_STREAM_END:
123 case Z_OK:
124 *psize = s.total_in;
125 rc = 0;
126 break;
127 case Z_BUF_ERROR:
128 case Z_NEED_DICT:
129 case Z_DATA_ERROR:
130 rc = RC(rcVDB, rcFunction, rcExecuting, rcData, rcCorrupt);
131 break;
132 case Z_MEM_ERROR:
133 rc = RC(rcXF, rcFunction, rcExecuting, rcMemory, rcExhausted);
134 break;
135 default:
136 rc = RC(rcVDB, rcFunction, rcExecuting, rcNoObj, rcUnexpected);
137 break;
138 }
139 if (inflateEnd(&s) == Z_OK) return rc;
140
141 return rc == 0 ? RC(rcVDB, rcFunction, rcExecuting, rcData, rcCorrupt) : rc;
142 }
143
144
145 #define STYPE int8_t
146 #define USTYPE uint8_t
147 #define ENCODE encode_i8
148 #define DECODE decode_i8
149 #include "irzip.impl.h"
150 #undef ENCODE
151 #undef DECODE
152 #undef STYPE
153 #undef USTYPE
154
155 #define STYPE int16_t
156 #define USTYPE uint16_t
157 #define ENCODE encode_i16
158 #define DECODE decode_i16
159 #include "irzip.impl.h"
160 #undef ENCODE
161 #undef DECODE
162 #undef STYPE
163 #undef USTYPE
164
165 #define STYPE int32_t
166 #define USTYPE uint32_t
167 #define ENCODE encode_i32
168 #define DECODE decode_i32
169 #define TRY2SERIES 1
170 #include "irzip.impl.h"
171 #undef TRY2SERIES
172 #undef ENCODE
173 #undef DECODE
174 #undef STYPE
175 #undef USTYPE
176
177 #define STYPE int64_t
178 #define USTYPE uint64_t
179 #define ENCODE encode_i64
180 #define DECODE decode_i64
181 #define TRY2SERIES 1
182 #include "irzip.impl.h"
183 #undef TRY2SERIES
184 #undef ENCODE
185 #undef DECODE
186 #undef STYPE
187 #undef USTYPE
188
189 #define STYPE uint8_t
190 #define USTYPE uint8_t
191 #define ENCODE encode_u8
192 #define DECODE decode_u8
193 #include "irzip.impl.h"
194 #undef ENCODE
195 #undef DECODE
196 #undef STYPE
197 #undef USTYPE
198
199 #define STYPE uint16_t
200 #define USTYPE uint16_t
201 #define ENCODE encode_u16
202 #define DECODE decode_u16
203 #include "irzip.impl.h"
204 #undef ENCODE
205 #undef DECODE
206 #undef STYPE
207 #undef USTYPE
208
209 #define STYPE uint32_t
210 #define USTYPE uint32_t
211 #define ENCODE encode_u32
212 #define DECODE decode_u32
213 #define TRY2SERIES 1
214 #include "irzip.impl.h"
215 #undef TRY2SERIES
216 #undef ENCODE
217 #undef DECODE
218 #undef STYPE
219 #undef USTYPE
220
221 #define STYPE uint64_t
222 #define USTYPE uint64_t
223 #define ENCODE encode_u64
224 #define DECODE decode_u64
225 #define TRY2SERIES 1
226 #include "irzip.impl.h"
227 #undef TRY2SERIES
228 #undef ENCODE
229 #undef DECODE
230 #undef STYPE
231 #undef USTYPE
232
233
234 typedef rc_t (*encode_f)(uint8_t dst[], size_t dsize, size_t *used,
235 int64_t *Min, int64_t *Slope, uint8_t *series_count,uint8_t *planes, const void *Y, unsigned N);
236
237 struct self_t {
238 encode_f f;
239 };
240
241 static
242 struct self_t selfs[8] = {
243 { (encode_f)encode_u8 },
244 { (encode_f)encode_i8 },
245 { (encode_f)encode_u16 },
246 { (encode_f)encode_i16 },
247 { (encode_f)encode_u32 },
248 { (encode_f)encode_i32 },
249 { (encode_f)encode_u64 },
250 { (encode_f)encode_i64 },
251 };
252
253
254
255 static
irzip(void * Self,const VXformInfo * info,VBlobResult * dst,const VBlobData * src,VBlobHeader * hdr)256 rc_t CC irzip(
257 void *Self,
258 const VXformInfo *info,
259 VBlobResult *dst,
260 const VBlobData *src,
261 VBlobHeader *hdr
262 ) {
263 rc_t rc = 0;
264 const struct self_t *self = Self;
265 size_t dsize;
266 int64_t min[2],slope[2];
267 uint8_t planes;
268 uint8_t series_count = 1;
269
270 assert(src->elem_count >> 32 == 0);
271 assert(((dst->elem_count * dst->elem_bits + 7) >> 3) >> 32 == 0);
272 dsize = (uint32_t)((dst->elem_count * dst->elem_bits + 7) >> 3);
273
274
275 rc = self->f(dst->data, dsize, &dsize, min, slope, &series_count, &planes, src->data, (unsigned)src->elem_count);
276 #if 0
277 if(/*src->elem_bits ==64 &&*/ dsize > 0){
278 printf("%2d:irzip_elem_bits=%.8f\telem=%d\tstart=%ld\tslope=%lx\tplanes=%d\n",(int)src->elem_bits,dsize*8./src->elem_count,(int)src->elem_count,min,slope,planes);
279 }
280 #endif
281
282 VBlobHeaderSetVersion(hdr, (series_count > 1)?3:2);
283 if ( rc == 0 ) {
284 rc = VBlobHeaderOpPushTail(hdr, planes);
285 if ( rc == 0) {
286 rc = VBlobHeaderArgPushTail(hdr, min[0]);
287 if ( rc == 0 ) {
288 rc = VBlobHeaderArgPushTail(hdr, slope[0]);
289 if ( rc == 0 ){
290 if(series_count > 1){
291 rc = VBlobHeaderArgPushTail(hdr, min[1]);
292 if(rc == 0) rc = VBlobHeaderArgPushTail(hdr, slope[1]);
293 }
294 dst->byte_order = vboNative;
295 dst->elem_bits = 1;
296 dst->elem_count = dsize << 3;
297 }
298 }
299 }
300 }
301 return rc;
302 }
303
304 /*
305 function izip_fmt izip #2.1 ( izip_set in )
306 */
307 VTRANSFACT_IMPL(vdb_izip, 2, 1, 1) (const void *Self, const VXfactInfo *info,
308 VFuncDesc *rslt, const VFactoryParams *cp,
309 const VFunctionParams *dp )
310 {
311 /*
312 * 2.1.1: bug fix for #VDB-539, bad conversions in encode_u64
313 */
314
315 if (dp->argc != 1) {
316 #if _DEBUGGING
317 fprintf(stderr, "dp->argc = %u != 1\n", dp->argc);
318 #endif
319 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
320 }
321
322 rslt->variant = vftBlob;
323 rslt->u.bf = irzip;
324
325 switch (dp->argv[0].desc.domain) {
326 case vtdInt:
327 switch (dp->argv[0].desc.intrinsic_bits) {
328 case 8:
329 rslt->self = &selfs[1];
330 break;
331 case 16:
332 rslt->self = &selfs[3];
333 break;
334 case 32:
335 rslt->self = &selfs[5];
336 break;
337 case 64:
338 rslt->self = &selfs[7];
339 break;
340 default:
341 #if _DEBUGGING
342 fprintf(stderr, "intrinsic_bits = %u != (8|16|32|64)\n", dp->argv[0].desc.intrinsic_bits);
343 #endif
344 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
345 break;
346 }
347 break;
348 case vtdUint:
349 switch (dp->argv[0].desc.intrinsic_bits) {
350 case 8:
351 rslt->self = &selfs[0];
352 break;
353 case 16:
354 rslt->self = &selfs[2];
355 break;
356 case 32:
357 rslt->self = &selfs[4];
358 break;
359 case 64:
360 rslt->self = &selfs[6];
361 break;
362 default:
363 #if _DEBUGGING
364 fprintf(stderr, "intrinsic_bits = %u != (8|16|32|64)\n", dp->argv[0].desc.intrinsic_bits);
365 #endif
366 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
367 break;
368 }
369 break;
370 default:
371 #if _DEBUGGING
372 fprintf(stderr, "domain != vtdInt or vtdUint\n");
373 #endif
374 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
375 }
376
377 return 0;
378 }
379
380 typedef rc_t (*decode_f)(void *dst, unsigned N, int64_t* min, int64_t* slope, uint8_t series_count,uint8_t planes, const uint8_t src[], size_t ssize);
381
382 static
383 decode_f uselfs[8] = {
384 (decode_f)decode_u8,
385 (decode_f)decode_i8,
386 (decode_f)decode_u16,
387 (decode_f)decode_i16,
388 (decode_f)decode_u32,
389 (decode_f)decode_i32,
390 (decode_f)decode_u64,
391 (decode_f)decode_i64,
392 };
393
394 static
iunzip_func_v1(void * Self,const VXformInfo * info,VBlobResult * dst,const VBlobData * src,int64_t * min,int64_t * slope,uint8_t series_count,uint8_t planes)395 rc_t iunzip_func_v1(
396 void *Self,
397 const VXformInfo *info,
398 VBlobResult *dst,
399 const VBlobData *src,
400 int64_t* min,int64_t* slope,uint8_t series_count,
401 uint8_t planes
402 )
403 {
404 uint32_t ssize;
405 rc_t rc;
406
407 assert(dst->elem_count >> 32 == 0);
408 assert(((src->elem_count * src->elem_bits + 7) >> 3) >> 32 == 0);
409 ssize = (uint32_t)((src->elem_count * src->elem_bits + 7) >> 3);
410
411 dst->byte_order = vboNative;
412
413 rc = uselfs[(uintptr_t)(Self)](dst->data, (unsigned)dst->elem_count, min, slope, series_count, planes,src->data, ssize);
414 return rc;
415 }
416
417 extern rc_t CC iunzip_func_v0(
418 void *Self,
419 const VXformInfo *info,
420 VBlobResult *dst,
421 const VBlobData *src
422 );
423
424 static
iunzip(void * Self,const VXformInfo * info,VBlobResult * dst,const VBlobData * src,VBlobHeader * hdr)425 rc_t CC iunzip(
426 void *Self,
427 const VXformInfo *info,
428 VBlobResult *dst,
429 const VBlobData *src,
430 VBlobHeader *hdr
431 )
432 {
433 switch (VBlobHeaderVersion(hdr)) {
434 case 0:
435 return iunzip_func_v0(Self, info, dst, src);
436 case 1:
437 case 2:
438 case 3:
439 {
440 int64_t min[2],slope[2];
441 uint8_t planes;
442 uint8_t series_count=1;
443 rc_t rc;
444
445 rc = VBlobHeaderOpPopHead(hdr, &planes);
446 if (rc == 0) {
447 rc = VBlobHeaderArgPopHead(hdr, min);
448 if (rc == 0) {
449 rc = VBlobHeaderArgPopHead(hdr, slope);
450 if (rc != 0) slope[0] = 0;
451 else {
452 rc = VBlobHeaderArgPopHead(hdr, min + 1);
453 if(rc==0) rc= VBlobHeaderArgPopHead(hdr, slope+1);
454 if(rc==0) series_count=2;
455 }
456 return iunzip_func_v1(Self, info, dst, src, min,slope, series_count, planes);
457 }
458 }
459 return rc;
460 }
461 default:
462 return RC(rcVDB, rcFunction, rcExecuting, rcParam, rcBadVersion);
463 }
464 }
465
466 /*
467 function izip_set iunzip #2.1 ( izip_fmt in )
468 */
469 VTRANSFACT_IMPL(vdb_iunzip, 2, 1, 0) (const void *Self, const VXfactInfo *info, VFuncDesc *rslt, const VFactoryParams *cp, const VFunctionParams *dp )
470 {
471 rslt->variant = vftBlob;
472 rslt->u.bf = iunzip;
473
474 switch (info->fdesc.desc.domain) {
475 case vtdInt:
476 switch (info->fdesc.desc.intrinsic_bits) {
477 case 8:
478 rslt->self = (void *)1;
479 break;
480 case 16:
481 rslt->self = (void *)3;
482 break;
483 case 32:
484 rslt->self = (void *)5;
485 break;
486 case 64:
487 rslt->self = (void *)7;
488 break;
489 default:
490 #if _DEBUGGING
491 fprintf(stderr, "intrinsic_bits = %u != (8|16|32|64)\n", info->fdesc.desc.intrinsic_bits);
492 #endif
493 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
494 break;
495 }
496 break;
497 case vtdUint:
498 switch (info->fdesc.desc.intrinsic_bits) {
499 case 8:
500 rslt->self = (void *)0;
501 break;
502 case 16:
503 rslt->self = (void *)2;
504 break;
505 case 32:
506 rslt->self = (void *)4;
507 break;
508 case 64:
509 rslt->self = (void *)6;
510 break;
511 default:
512 #if _DEBUGGING
513 fprintf(stderr, "intrinsic_bits = %u != (8|16|32|64)\n", info->fdesc.desc.intrinsic_bits);
514 #endif
515 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
516 break;
517 }
518 break;
519 default:
520 #if _DEBUGGING
521 fprintf(stderr, "domain != vtdInt or vtdUint\n");
522 #endif
523 return RC(rcVDB, rcFunction, rcConstructing, rcParam, rcInvalid);
524 }
525 return 0;
526 }
527
528 #if TESTING
test_encode(const int32_t Y[],unsigned N)529 int test_encode(const int32_t Y[], unsigned N) {
530 uint8_t *dst;
531 size_t dsize;
532 size_t temp;
533 rc_t rc = 0;
534 int64_t min,slope;
535 uint8_t planes;
536
537 temp = N * sizeof(Y[0]);
538 assert(temp >> 32 == 0);
539 dst = malloc(dsize = (unsigned)temp);
540 if (dst == NULL)
541 return RC(rcXF, rcFunction, rcExecuting, rcMemory, rcExhausted);
542
543 rc = encode_i32(dst, dsize, &dsize, &min, &slope, &planes, Y, N);
544
545 {
546 int32_t *X;
547
548 X = malloc(N * sizeof(Y[0]));
549 if (X) {
550 rc = decode_i32( X, N, min, slope, planes, dst, dsize);
551 if (rc == 0) {
552 rc = memcmp(Y, X, N * sizeof(Y[0])) == 0 ? 0 : RC(rcXF, rcFunction, rcExecuting, rcFunction, rcInvalid);
553 }
554 free(X);
555 }
556 }
557
558 free(dst);
559 if (rc)
560 fprintf(stdout, "test failed!\n");
561 return rc;
562 }
563 #endif
564
565