1 #include <stdio.h>
2 #include <string.h>
3 #include "grib_int.h"
4 #include "grib_decode.h"
5 #include "dmemory.h"
6 #include "error.h"
7
8 #if defined (HAVE_CONFIG_H)
9 #include "config.h"
10 #endif
11
12 #if defined (HAVE_LIBSZ)
13 #if defined(__cplusplus)
14 extern "C" {
15 #endif
16 #include <szlib.h>
17 #ifdef __cplusplus
18 }
19 #endif
20
21 #define OPTIONS_MASK (SZ_RAW_OPTION_MASK | SZ_MSB_OPTION_MASK | SZ_NN_OPTION_MASK)
22
23 #define PIXELS_PER_BLOCK (8)
24 #define PIXELS_PER_SCANLINE (PIXELS_PER_BLOCK*128)
25
26 #define MIN_COMPRESS (0.95)
27 #define MIN_SIZE (256)
28 #endif
29
30 #define Z_SZIP 128
31
32 #if defined (HAVE_LIBSZ) || defined (HAVE_LIBAEC)
33 #define SetLen3(var, offset, value) ((var[offset+0] = 0xFF & (value >> 16)), \
34 (var[offset+1] = 0xFF & (value >> 8)), \
35 (var[offset+2] = 0xFF & (value )))
36 #define SetLen4(var, offset, value) ((var[offset+0] = 0xFF & (value >> 24)), \
37 (var[offset+1] = 0xFF & (value >> 16)), \
38 (var[offset+2] = 0xFF & (value >> 8)), \
39 (var[offset+3] = 0xFF & (value )))
40 #endif
41
gribGetZip(size_t recsize,unsigned char * gribbuffer,size_t * urecsize)42 int gribGetZip(size_t recsize, unsigned char *gribbuffer, size_t *urecsize)
43 {
44 int compress = 0;
45 unsigned char *pds = NULL, *gds = NULL, *bms = NULL, *bds = NULL;
46
47 int gribversion = gribVersion(gribbuffer, recsize);
48
49 if ( gribversion == 2 ) return compress;
50
51 long gribrecsize;
52 int nerr = grib1Sections(gribbuffer, (long)recsize, &pds, &gds, &bms, &bds, &gribrecsize);
53 if ( nerr < 0 )
54 {
55 fprintf(stdout, "GRIB message error\n");
56 return compress;
57 }
58
59 if ( nerr > 0 )
60 {
61 fprintf(stdout, "GRIB data corrupted!\n");
62 return compress;
63 }
64
65 /* bds_len = BDS_Len; */
66 /* bds_nbits = BDS_NumBits; */
67 int bds_flag = BDS_Flag;
68 /* lspherc = bds_flag >> 7; */
69 /* lcomplex = (bds_flag >> 6)&1; */
70 int lcompress = (bds_flag >> 4)&1;
71
72 size_t gribsize = 0;
73 if ( lcompress )
74 {
75 compress = BDS_Z;
76 if ( compress == Z_SZIP ) gribsize = (size_t) GET_UINT3(bds[14], bds[15], bds[16]);
77 }
78
79 *urecsize = gribsize;
80
81 return compress;
82 }
83
84
gribZip(unsigned char * dbuf,long dbufsize,unsigned char * sbuf,long sbufsize)85 int gribZip(unsigned char *dbuf, long dbufsize, unsigned char *sbuf, long sbufsize)
86 {
87 #if ! defined(HAVE_LIBSZ)
88 static int libszwarn = 1;
89 #endif
90 unsigned char *pds = NULL, *gds = NULL, *bms = NULL, *bds = NULL;
91 bool llarge = false;
92
93 unsigned gribLen = GET_UINT3(dbuf[4], dbuf[5], dbuf[6]);
94
95 int rec_len = gribLen;
96
97 long gribrecsize;
98 int nerr = grib1Sections(dbuf, dbufsize, &pds, &gds, &bms, &bds, &gribrecsize);
99 if ( nerr < 0 )
100 {
101 fprintf(stdout, "GRIB message error\n");
102 return gribrecsize;
103 }
104
105 if ( nerr > 0 )
106 {
107 fprintf(stdout, "GRIB data corrupted!\n");
108 return gribrecsize;
109 }
110
111 int bds_zoffset = 12;
112
113 int bds_len = BDS_Len;
114 if ( gribLen > JP23SET && bds_len <= 120 )
115 {
116 gribLen &= JP23SET;
117 gribLen *= 120;
118 bds_len = correct_bdslen(bds_len, gribLen, bds-dbuf);
119 llarge = true;
120 bds_zoffset += 2;
121 }
122
123 if ( gribLen > JP24SET || llarge ) return gribLen;
124
125 #if defined(HAVE_LIBSZ)
126 {
127 int bds_zstart = 14;
128 unsigned gribLenOld = 0;
129 int bds_head = 11;
130 int bds_ext = 0;
131 unsigned char *pbuf = NULL;
132
133 int bds_nbits = BDS_NumBits;
134 int bds_flag = BDS_Flag;
135 int bds_ubits = bds_flag & 15;
136 int lspherc = bds_flag >> 7;
137 int lcomplex = (bds_flag >> 6)&1;
138 /* lcompress = (bds_flag >> 4)&1; */
139
140 if ( bds_nbits != 8 && bds_nbits != 16 && bds_nbits != 24 && bds_nbits != 32 )
141 {
142 static bool linfo = true;
143 if ( linfo && bds_nbits != 0 )
144 {
145 linfo = false;
146 fprintf(stderr, "GRIB szip supports only 8, 16, 24 and 32 bit data!\n");
147 }
148 return rec_len;
149 }
150
151 int bits_per_sample = (bds_nbits == 24) ? 8 : bds_nbits;
152
153 SZ_com_t sz_param; /* szip parameter block */
154 sz_param.options_mask = OPTIONS_MASK;
155 sz_param.bits_per_pixel = bits_per_sample;
156 sz_param.pixels_per_block = PIXELS_PER_BLOCK;
157 sz_param.pixels_per_scanline = PIXELS_PER_SCANLINE;
158
159 if ( lspherc )
160 {
161 bds_ext = 4;
162 if ( lcomplex )
163 {
164 int jup = bds[15];
165 int ioff = (jup+1)*(jup+2);
166 bds_ext += 3 + 4*ioff;
167 }
168 }
169
170 size_t datstart = bds_head + bds_ext;
171
172 size_t datsize = ((((bds_len - datstart)*8-bds_ubits)/bds_nbits)*bds_nbits)/8;
173
174 if ( datsize < MIN_SIZE ) return rec_len;
175 /*
176 fprintf(stderr, "%d %d %d %d\n", bds_len, datstart, bds_len - datstart, datsize);
177 */
178 size_t sourceLen = datsize;
179 size_t destLen = sbufsize;
180
181 unsigned char *source = bds + datstart;
182 unsigned char *dest = sbuf;
183
184 if ( bds_nbits == 24 )
185 {
186 long nelem = sourceLen/3;
187 pbuf = (unsigned char*) Malloc(sourceLen);
188 for ( long i = 0; i < nelem; i++ )
189 {
190 pbuf[ i] = source[3*i ];
191 pbuf[ nelem+i] = source[3*i+1];
192 pbuf[2*nelem+i] = source[3*i+2];
193 }
194 source = pbuf;
195 }
196
197 int status = SZ_BufftoBuffCompress(dest, &destLen, source, sourceLen, &sz_param);
198 if ( status != SZ_OK )
199 {
200 if ( status == SZ_NO_ENCODER_ERROR )
201 Warning("SZ_NO_ENCODER_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
202 else if ( status == SZ_PARAM_ERROR )
203 Warning("SZ_PARAM_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
204 else if ( status == SZ_MEM_ERROR )
205 Warning("SZ_MEM_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
206 else if ( status == SZ_OUTBUFF_FULL )
207 /*Warning("SZ_OUTBUFF_FULL code %3d level %3d", PDS_Parameter, PDS_Level2)*/;
208 else
209 Warning("SZ ERROR: %d code %3d level %3d", status, PDS_Parameter, PDS_Level2);
210 }
211
212 if ( pbuf ) Free(pbuf);
213 /*
214 fprintf(stderr, "sourceLen, destLen %d %d\n", sourceLen, destLen);
215 */
216 if ( destLen < MIN_COMPRESS*sourceLen )
217 {
218 source = bds + datstart + bds_zoffset;
219 memcpy(source, dest, destLen);
220
221 /* ----++++ number of unused bits at end of section) */
222
223 BDS_Flag -= bds_ubits;
224
225 gribLenOld = gribLen;
226
227 if ( bds_ext )
228 for ( long i = bds_ext-1; i >= 0; --i )
229 bds[bds_zoffset+bds_head+i] = bds[bds_head+i];
230
231 /*
232 fprintf(stderr, "destLen, datsize, datstart %d %d %d\n", destLen, datsize, datstart);
233 */
234 /* memcpy(bds + datstart + bds_zoffset, source, destLen); */
235 /*
236 fprintf(stderr, "z>>> %d %d %d %d <<<\n", (int) bds[0+datstart+bds_zoffset],
237 (int)bds[1+datstart+bds_zoffset], (int)bds[2+datstart+bds_zoffset], (int)bds[3+datstart+bds_zoffset]);
238 */
239 if ( llarge )
240 {
241 if ( gribLenOld%120 )
242 {
243 fprintf(stderr, "Internal problem, record length not multiple of 120!");
244 while ( gribLenOld%120 ) gribLenOld++;
245 }
246 // gribLenOld = gribLenOld / (-120);
247 // gribLenOld = JP23SET - gribLenOld + 1;
248
249 SetLen3(bds, bds_zstart, gribLenOld);
250 SetLen4(bds, bds_zstart+3, sourceLen);
251 SetLen4(bds, bds_zstart+7, destLen);
252 }
253 else
254 {
255 SetLen3(bds, bds_zstart, gribLenOld);
256 SetLen3(bds, bds_zstart+3, sourceLen);
257 SetLen3(bds, bds_zstart+6, destLen);
258 }
259
260 int bdsLen = datstart + bds_zoffset + destLen;
261
262 bds[11] = 0;
263 bds[12] = 0;
264
265 BDS_Z = Z_SZIP;
266
267 BDS_Flag += 16;
268 if ( (bdsLen%2) == 1 )
269 {
270 BDS_Flag += 8;
271 bds[bdsLen++] = 0;
272 }
273
274 SetLen3(bds, 0, bdsLen);
275
276 gribLen = (bds - dbuf) + bdsLen;
277
278 dbuf[gribLen++] = '7';
279 dbuf[gribLen++] = '7';
280 dbuf[gribLen++] = '7';
281 dbuf[gribLen++] = '7';
282
283 if ( llarge )
284 {
285 long bdslen = gribLen - 4;
286
287 /*
288 If a very large product, the section 4 length field holds
289 the number of bytes in the product after section 4 upto
290 the end of the padding bytes.
291 This is a fixup to get round the restriction on product lengths
292 due to the count being only 24 bits. It is only possible because
293 the (default) rounding for GRIB products is 120 bytes.
294 */
295 while ( gribLen%120 ) dbuf[gribLen++] = 0;
296
297 long itemp = gribLen / (-120);
298 itemp = JP23SET - itemp + 1;
299
300 SetLen3(dbuf, 4, itemp);
301
302 bdslen = gribLen - bdslen;
303
304 SetLen3(bds, 0, bdslen);
305 }
306 else
307 {
308 SetLen3(dbuf, 4, gribLen);
309 }
310 }
311 else
312 {
313 }
314 /*
315 fprintf(stderr, "%3d %3d griblen in %6d out %6d CR %g slen %6d dlen %6d CR %g\n",
316 PDS_Parameter, PDS_Level1, gribLenOld, gribLen,
317 ((double)gribLenOld)/gribLen, sourceLen, destLen,
318 ((double)sourceLen)/destLen);
319 */
320 }
321
322 #else
323
324 UNUSED(sbuf);
325 UNUSED(sbufsize);
326
327 if ( libszwarn )
328 {
329 Warning("Compression disabled, szlib not available!");
330 libszwarn = 0;
331 }
332 #endif
333
334 if ( llarge )
335 while ( gribLen%120 ) dbuf[gribLen++] = 0;
336 else
337 while ( gribLen & 7 ) dbuf[gribLen++] = 0;
338
339 rec_len = gribLen;
340
341 return rec_len;
342 }
343
344
gribUnzip(unsigned char * dbuf,long dbufsize,unsigned char * sbuf,long sbufsize)345 int gribUnzip(unsigned char *dbuf, long dbufsize, unsigned char *sbuf, long sbufsize)
346 {
347 #if ! defined(HAVE_LIBSZ)
348 static int libszwarn = 1;
349 #endif
350 unsigned char *pds = NULL, *gds = NULL, *bms = NULL, *bds = NULL;
351 size_t gribLen = 0;
352 size_t destLen, sourceLen;
353 enum { bds_head = 11 };
354 int bds_ext = 0;
355
356 UNUSED(dbufsize);
357
358 long gribrecsize;
359 int nerr = grib1Sections(sbuf, sbufsize, &pds, &gds, &bms, &bds, &gribrecsize);
360 if ( nerr < 0 )
361 {
362 fprintf(stdout, "GRIB message error\n");
363 return 0;
364 }
365
366 if ( nerr > 0 )
367 {
368 fprintf(stdout, "GRIB data corrupted!\n");
369 return 0;
370 }
371
372 //unsigned bds_len = BDS_Len;
373 bool llarge = false;
374
375 int bds_zoffset = 12;
376 if ( llarge ) bds_zoffset += 2;
377
378 int bds_nbits = BDS_NumBits;
379 int bds_flag = BDS_Flag;
380 int lspherc = bds_flag >> 7;
381 int lcomplex = (bds_flag >> 6)&1;
382 /* lcompress = (bds_flag >> 4)&1; */
383
384 if ( lspherc )
385 {
386 if ( lcomplex )
387 {
388 int jup = bds[bds_zoffset+15];
389 int ioff = (jup+1)*(jup+2);
390 bds_ext = 4 + 3 + 4*ioff;
391 }
392 else
393 {
394 bds_ext = 4;
395 }
396 }
397
398 size_t datstart = bds_head + (size_t)bds_ext;
399
400 unsigned char *source = bds + datstart + bds_zoffset;
401 if ( llarge )
402 sourceLen = ((size_t) ((bds[21]<<24)+(bds[22]<<16)+(bds[23]<<8)+bds[24]));
403 else
404 sourceLen = ((size_t) ((bds[20]<<16)+(bds[21]<<8)+bds[22]));
405
406 nerr = grib1Sections(dbuf, sbufsize, &pds, &gds, &bms, &bds, &gribrecsize);
407 if ( nerr < 0 )
408 {
409 fprintf(stdout, "GRIB message error\n");
410 return 0;
411 }
412
413 if ( nerr > 0 )
414 {
415 fprintf(stdout, "GRIB data corrupted!\n");
416 return 0;
417 }
418
419 unsigned char *dest = bds + datstart;
420 if ( llarge )
421 destLen = ((size_t) ((bds[17]<<24)+(bds[18]<<16)+(bds[19]<<8)+bds[20]));
422 else
423 destLen = ((size_t) ((bds[17]<<16)+(bds[18]<<8)+bds[19]));
424
425 BDS_Flag = (unsigned char)(BDS_Flag - 16);
426
427 size_t bdsLen = datstart + destLen;
428
429 #if defined(HAVE_LIBSZ)
430 {
431 int bds_zstart = 14;
432 unsigned recLen = GET_UINT3(bds[bds_zstart], bds[bds_zstart+1], bds[bds_zstart+2]);
433
434 int bits_per_sample = (bds_nbits == 24) ? 8 : bds_nbits;
435
436 SZ_com_t sz_param; /* szip parameter block */
437 sz_param.options_mask = OPTIONS_MASK;
438 sz_param.bits_per_pixel = bits_per_sample;
439 sz_param.pixels_per_block = PIXELS_PER_BLOCK;
440 sz_param.pixels_per_scanline = PIXELS_PER_SCANLINE;
441
442 if ( bds_ext )
443 for ( long i = 0; i < bds_ext; ++i )
444 bds[bds_head+i] = bds[bds_zoffset+bds_head+i];
445
446 /* fprintf(stderr, "gribUnzip: sourceLen %ld; destLen %ld\n", (long)sourceLen, (long)destLen);
447 fprintf(stderr, "gribUnzip: sourceOff %d; destOff %d\n", bds[12], bds[11]);
448 fprintf(stderr, "gribUnzip: reclen %d; bdslen %d\n", recLen, bdsLen);
449 */
450
451 size_t tmpLen = destLen;
452
453 int status = SZ_BufftoBuffDecompress(dest, &tmpLen, source, sourceLen, &sz_param);
454 if ( status != SZ_OK )
455 {
456 if ( status == SZ_NO_ENCODER_ERROR )
457 Warning("SZ_NO_ENCODER_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
458 else if ( status == SZ_PARAM_ERROR )
459 Warning("SZ_PARAM_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
460 else if ( status == SZ_MEM_ERROR )
461 Warning("SZ_MEM_ERROR code %3d level %3d", PDS_Parameter, PDS_Level2);
462 else if ( status == SZ_OUTBUFF_FULL )
463 Warning("SZ_OUTBUFF_FULL code %3d level %3d", PDS_Parameter, PDS_Level2);
464 else
465 Warning("SZ ERROR: %d code %3d level %3d", status, PDS_Parameter, PDS_Level2);
466 }
467 /*
468 fprintf(stderr, "gribUnzip: sl = %ld dl = %ld tl = %ld\n",
469 (long)sourceLen, (long)destLen,(long) tmpLen);
470 */
471 if ( tmpLen != destLen )
472 Warning("unzip size differ: code %3d level %3d ibuflen %ld ubuflen %ld",
473 PDS_Parameter, PDS_Level2, (long) destLen, (long) tmpLen);
474
475 if ( bds_nbits == 24 )
476 {
477 long nelem = tmpLen/3;
478 unsigned char *pbuf = (unsigned char*) Malloc(tmpLen);
479 for ( long i = 0; i < nelem; i++ )
480 {
481 pbuf[3*i ] = dest[ i];
482 pbuf[3*i+1] = dest[ nelem+i];
483 pbuf[3*i+2] = dest[2*nelem+i];
484 }
485 memcpy(dest, pbuf, tmpLen);
486 Free(pbuf);
487 }
488
489 int bds_ubits = BDS_Flag & 15;
490 BDS_Flag -= bds_ubits;
491
492 if ( (bdsLen%2) == 1 )
493 {
494 BDS_Flag += 8;
495 bds[bdsLen++] = 0;
496 }
497
498 SetLen3(bds, 0, bdsLen);
499
500 gribLen = (bds - dbuf) + bdsLen;
501
502 dbuf[gribLen++] = '7';
503 dbuf[gribLen++] = '7';
504 dbuf[gribLen++] = '7';
505 dbuf[gribLen++] = '7';
506
507 if ( llarge )
508 {
509 long itemp;
510 bdsLen = gribLen - 4;
511 /*
512 If a very large product, the section 4 length field holds
513 the number of bytes in the product after section 4 upto
514 the end of the padding bytes.
515 This is a fixup to get round the restriction on product lengths
516 due to the count being only 24 bits. It is only possible because
517 the (default) rounding for GRIB products is 120 bytes.
518 */
519 while ( gribLen%120 ) dbuf[gribLen++] = 0;
520
521 if ( gribLen != (size_t)recLen )
522 fprintf(stderr, "Internal problem, recLen and gribLen differ!\n");
523
524 itemp = gribLen / (-120);
525 itemp = JP23SET - itemp + 1;
526
527 SetLen3(dbuf, 4, itemp);
528
529 bdsLen = gribLen - bdsLen;
530
531 SetLen3(bds, 0, bdsLen);
532 }
533 else
534 {
535 SetLen3(dbuf, 4, recLen);
536 }
537 /*
538 fprintf(stderr, "recLen, gribLen, bdsLen %d %d %d\n", recLen, gribLen, bdsLen);
539 */
540 if ( llarge )
541 while ( gribLen%120 ) dbuf[gribLen++] = 0;
542 else
543 while ( gribLen & 7 ) dbuf[gribLen++] = 0;
544 /*
545 fprintf(stderr, "recLen, gribLen, bdsLen %d %d %d\n", recLen, gribLen, bdsLen);
546 */
547 }
548 #else
549 UNUSED(bds_nbits);
550 UNUSED(sourceLen);
551 UNUSED(source);
552 UNUSED(bdsLen);
553 UNUSED(dest);
554
555 if ( libszwarn )
556 {
557 Warning("Decompression disabled, szlib not available!");
558 libszwarn = 0;
559 }
560 #endif
561
562 return (int)gribLen;
563 }
564