1 /*
2 * (C) Copyright 2005- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 *
7 * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
8 * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
9 */
10
11 #include "grib_api_internal.h"
12
13 #if GRIB_PTHREADS
14 static pthread_once_t once = PTHREAD_ONCE_INIT;
15 static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
16 static pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER;
init()17 static void init()
18 {
19 pthread_mutexattr_t attr;
20 pthread_mutexattr_init(&attr);
21 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
22 pthread_mutex_init(&mutex1, &attr);
23 pthread_mutex_init(&mutex2, &attr);
24 pthread_mutexattr_destroy(&attr);
25 }
26 #elif GRIB_OMP_THREADS
27 static int once = 0;
28 static omp_nest_lock_t mutex1;
29 static omp_nest_lock_t mutex2;
init()30 static void init()
31 {
32 GRIB_OMP_CRITICAL(lock_grib_io_c)
33 {
34 if (once == 0) {
35 omp_init_nest_lock(&mutex1);
36 omp_init_nest_lock(&mutex2);
37 once = 1;
38 }
39 }
40 }
41 #endif
42
43
44 #define GRIB 0x47524942
45 #define BUDG 0x42554447
46 #define DIAG 0x44494147
47 #define TIDE 0x54494445
48 #define BUFR 0x42554652
49 #define HDF5 0x89484446
50 #define WRAP 0x57524150
51
52 #define ECCODES_READS_BUFR 1
53 #define ECCODES_READS_HDF5 1
54 #define ECCODES_READS_WRAP 1
55
56
57 typedef struct alloc_buffer
58 {
59 size_t size;
60 void* buffer;
61 } alloc_buffer;
62
63 typedef size_t (*readproc)(void*, void*, size_t, int*);
64 typedef int (*seekproc)(void*, off_t);
65 typedef off_t (*tellproc)(void*);
66 typedef void* (*allocproc)(void*, size_t*, int*);
67
68
69 typedef struct reader
70 {
71 void* read_data;
72 readproc read;
73
74 void* alloc_data;
75 allocproc alloc;
76 int headers_only;
77
78 seekproc seek;
79 seekproc seek_from_start;
80 tellproc tell;
81 off_t offset;
82
83 size_t message_size;
84
85 } reader;
86
read_the_rest(reader * r,size_t message_length,unsigned char * tmp,int already_read,int check7777)87 static int read_the_rest(reader* r, size_t message_length, unsigned char* tmp, int already_read, int check7777)
88 {
89 int err = GRIB_SUCCESS;
90 size_t buffer_size;
91 size_t rest;
92 unsigned char* buffer;
93 grib_context* c = grib_context_get_default();
94
95 if (message_length == 0)
96 return GRIB_BUFFER_TOO_SMALL;
97
98 buffer_size = message_length;
99 rest = message_length - already_read;
100 r->message_size = message_length;
101 buffer = (unsigned char*)r->alloc(r->alloc_data, &buffer_size, &err);
102 if (err)
103 return err;
104
105 if (buffer == NULL || (buffer_size < message_length)) {
106 return GRIB_BUFFER_TOO_SMALL;
107 }
108
109 memcpy(buffer, tmp, already_read);
110
111 if ((r->read(r->read_data, buffer + already_read, rest, &err) != rest) || err) {
112 /*fprintf(stderr, "read_the_rest: r->read failed: %s\n", grib_get_error_message(err));*/
113 if (c->debug)
114 fprintf(stderr, "ECCODES DEBUG read_the_rest: Read failed (Coded length=%lu, Already read=%d)\n",
115 message_length, already_read);
116 return err;
117 }
118
119 if (check7777 && !r->headers_only &&
120 (buffer[message_length - 4] != '7' ||
121 buffer[message_length - 3] != '7' ||
122 buffer[message_length - 2] != '7' ||
123 buffer[message_length - 1] != '7'))
124 {
125 if (c->debug)
126 fprintf(stderr, "ECCODES DEBUG read_the_rest: No final 7777 at expected location (Coded length=%lu)\n", message_length);
127 return GRIB_WRONG_LENGTH;
128 }
129
130 return GRIB_SUCCESS;
131 }
132
133 #define CHECK_TMP_SIZE(a) \
134 if (sizeof(tmp) < (a)) { \
135 fprintf(stderr, "%s:%d sizeof(tmp)<%s %d<%d\n", __FILE__, __LINE__, #a, (int)sizeof(tmp), (int)(a)); \
136 return GRIB_INTERNAL_ARRAY_TOO_SMALL; \
137 }
138
139 #define GROW_BUF_IF_REQUIRED(desired_length) \
140 if (buf->length < (desired_length)) { \
141 grib_grow_buffer(c, buf, desired_length); \
142 tmp = buf->data; \
143 }
144
145 #define UINT3(a, b, c) (size_t)((a << 16) + (b << 8) + c);
146
read_GRIB(reader * r)147 static int read_GRIB(reader* r)
148 {
149 unsigned char* tmp = NULL;
150 size_t length = 0;
151 size_t total_length = 0;
152 long edition = 0;
153 int err = 0;
154 int i = 0, j;
155 size_t sec1len = 0;
156 size_t sec2len = 0;
157 size_t sec3len = 0;
158 size_t sec4len = 0;
159 unsigned long flags;
160 size_t buflen = 32768; /* See ECC-515: was 16368 */
161 grib_context* c;
162 grib_buffer* buf;
163
164 /*TODO proper context*/
165 c = grib_context_get_default();
166 tmp = (unsigned char*)malloc(buflen);
167 if (!tmp)
168 return GRIB_OUT_OF_MEMORY;
169 buf = grib_new_buffer(c, tmp, buflen);
170 buf->property = GRIB_MY_BUFFER;
171
172 tmp[i++] = 'G';
173 tmp[i++] = 'R';
174 tmp[i++] = 'I';
175 tmp[i++] = 'B';
176
177 r->offset = r->tell(r->read_data) - 4;
178
179 if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
180 return err;
181
182 length = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
183 i += 3;
184
185 /* Edition number */
186 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
187 return err;
188
189 edition = tmp[i++];
190
191 switch (edition) {
192 case 1:
193 if (r->headers_only) {
194 /* Read section 1 length */
195 if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
196 return err;
197
198 sec1len = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
199 i += 3;
200 /* Read section 1. 3 = length */
201 if ((r->read(r->read_data, tmp + i, sec1len - 3, &err) != sec1len - 3) || err)
202 return err;
203 flags = tmp[15];
204
205 i += sec1len - 3;
206
207 GROW_BUF_IF_REQUIRED(i + 3);
208
209 if (flags & (1 << 7)) {
210 /* Section 2 */
211 if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
212 return err;
213
214 sec2len = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
215 GROW_BUF_IF_REQUIRED(i + sec2len);
216 i += 3;
217 /* Read section 2 */
218 if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
219 return err;
220 i += sec2len - 3;
221 }
222
223
224 if (flags & (1 << 6)) {
225 /* Section 3 */
226 GROW_BUF_IF_REQUIRED(i + 3);
227 for (j = 0; j < 3; j++) {
228 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
229 return err;
230
231 sec3len <<= 8;
232 sec3len |= tmp[i];
233 i++;
234 }
235
236 /* Read section 3 */
237 GROW_BUF_IF_REQUIRED(i + sec3len);
238 if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
239 return err;
240 i += sec3len - 3;
241 }
242
243 GROW_BUF_IF_REQUIRED(i + 11);
244
245 /* Section 4 */
246 for (j = 0; j < 3; j++) {
247 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
248 return err;
249
250 sec4len <<= 8;
251 sec4len |= tmp[i];
252 i++;
253 }
254
255 /* we don't read the data, only headers */
256 if ((r->read(r->read_data, tmp + i, 8, &err) != 8) || err)
257 return err;
258
259 i += 8;
260
261 total_length = length;
262 /* length=8+sec1len + sec2len+sec3len+11; */
263 length = i;
264 err = r->seek(r->read_data, total_length - length - 1);
265 }
266 else if (length & 0x800000) {
267 /* Large GRIBs */
268
269 /* Read section 1 length */
270 for (j = 0; j < 3; j++) {
271 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
272 return err;
273
274 sec1len <<= 8;
275 sec1len |= tmp[i];
276 i++;
277 }
278
279 /* table version */
280 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
281 return err;
282 /* center */
283 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
284 return err;
285 /* process */
286 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
287 return err;
288 /* grid */
289 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
290 return err;
291 /* flags */
292 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
293 return err;
294 flags = tmp[i++];
295
296 /* fprintf(stderr," sec1len=%d i=%d flags=%x\n",sec1len,i,flags); */
297
298 GROW_BUF_IF_REQUIRED(8 + sec1len + 4 + 3);
299
300 /* Read section 1. 3 = length, 5 = table,center,process,grid,flags */
301 if ((r->read(r->read_data, tmp + i, sec1len - 3 - 5, &err) != sec1len - 3 - 5) || err)
302 return err;
303
304 i += sec1len - 3 - 5;
305
306 if (flags & (1 << 7)) {
307 /* Section 2 */
308 for (j = 0; j < 3; j++) {
309 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
310 return err;
311
312 sec2len <<= 8;
313 sec2len |= tmp[i];
314 i++;
315 }
316 /* Read section 2 */
317 GROW_BUF_IF_REQUIRED(i + sec2len);
318 if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
319 return err;
320 i += sec2len - 3;
321 }
322
323 GROW_BUF_IF_REQUIRED(sec1len + sec2len + 4 + 3);
324
325 if (flags & (1 << 6)) {
326 /* Section 3 */
327 for (j = 0; j < 3; j++) {
328 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
329 return err;
330
331 sec3len <<= 8;
332 sec3len |= tmp[i];
333 i++;
334 }
335
336 /* Read section 3 */
337 GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
338 if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
339 return err;
340 i += sec3len - 3;
341 }
342
343 /* fprintf(stderr,"%s sec1len=%d i=%d\n",type,sec1len,i); */
344
345 GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
346
347
348 for (j = 0; j < 3; j++) {
349 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
350 return err;
351
352 sec4len <<= 8;
353 sec4len |= tmp[i];
354 i++;
355 }
356
357 if (sec4len < 120) {
358 /* Special coding */
359 length &= 0x7fffff;
360 length *= 120;
361 length -= sec4len;
362 length += 4;
363 }
364 else {
365 /* length is already set to the right value */
366 }
367 }
368 break;
369
370 case 2:
371 case 3:
372 length = 0;
373
374 if (sizeof(long) >= 8) {
375 for (j = 0; j < 8; j++) {
376 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
377 return err;
378
379 length <<= 8;
380 length |= tmp[i];
381 i++;
382 }
383 }
384 else {
385 /* Check if the length fits in a long */
386 for (j = 0; j < 4; j++) {
387 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
388 return err;
389
390 length <<= 8;
391 length |= tmp[i];
392 i++;
393 }
394
395 if (length)
396 return GRIB_MESSAGE_TOO_LARGE; /* Message too large */
397
398 for (j = 0; j < 4; j++) {
399 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
400 return err;
401
402 length <<= 8;
403 length |= tmp[i];
404 i++;
405 }
406 }
407 break;
408
409 default:
410 r->seek_from_start(r->read_data, r->offset + 4);
411 grib_buffer_delete(c, buf);
412 return GRIB_UNSUPPORTED_EDITION;
413 break;
414 }
415
416 /* Assert(i <= buf->length); */
417 err = read_the_rest(r, length, tmp, i, 1);
418 if (err)
419 r->seek_from_start(r->read_data, r->offset + 4);
420
421 grib_buffer_delete(c, buf);
422
423 return err;
424 }
425
read_PSEUDO(reader * r,const char * type)426 static int read_PSEUDO(reader* r, const char* type)
427 {
428 unsigned char tmp[32]; /* Should be enough */
429 size_t sec1len = 0;
430 size_t sec4len = 0;
431 int err = 0;
432 int i = 0, j = 0;
433
434 Assert(strlen(type) == 4);
435 for (j = 0; j < 4; j++) {
436 tmp[i] = type[i];
437 i++;
438 }
439
440 for (j = 0; j < 3; j++) {
441 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
442 return err;
443
444 sec1len <<= 8;
445 sec1len |= tmp[i];
446 i++;
447 }
448
449 /* fprintf(stderr,"%s sec1len=%d i=%d\n",type,sec1len,i); */
450 CHECK_TMP_SIZE(sec1len + 4 + 3);
451
452 /* Read sectoin1 */
453 if ((r->read(r->read_data, tmp + i, sec1len - 3, &err) != sec1len - 3) || err)
454 return err;
455
456 i += sec1len - 3;
457
458 for (j = 0; j < 3; j++) {
459 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
460 return err;
461
462 sec4len <<= 8;
463 sec4len |= tmp[i];
464 i++;
465 }
466
467 /* fprintf(stderr,"%s sec4len=%d i=%d l=%d\n",type,sec4len,i,4+sec1len+sec4len+4); */
468
469 Assert(i <= sizeof(tmp));
470 return read_the_rest(r, 4 + sec1len + sec4len + 4, tmp, i, 1);
471 }
472
473
read_HDF5_offset(reader * r,int length,unsigned long * v,unsigned char * tmp,int * i)474 static int read_HDF5_offset(reader* r, int length, unsigned long* v, unsigned char* tmp, int* i)
475 {
476 unsigned char buf[8];
477 int j, k;
478 int err = 0;
479
480
481 if ((r->read(r->read_data, buf, length, &err) != length) || err) {
482 return err;
483 }
484
485 k = *i;
486 for (j = 0; j < length; j++) {
487 tmp[k++] = buf[j];
488 }
489 *i = k;
490
491 *v = 0;
492 for (j = length - 1; j >= 0; j--) {
493 *v <<= 8;
494 *v |= buf[j];
495 }
496
497 return 0;
498 }
499
read_HDF5(reader * r)500 static int read_HDF5(reader* r)
501 {
502 /*
503 * See: http://www.hdfgroup.org/HDF5/doc/H5.format.html#Superblock
504 */
505 unsigned char tmp[49]; /* Should be enough */
506 unsigned char buf[4];
507
508 unsigned char version_of_superblock, size_of_offsets, size_of_lengths, consistency_flags;
509 unsigned long base_address, superblock_extension_address, end_of_file_address;
510
511 int i = 0, j;
512 int err = 0;
513 grib_context* c = grib_context_get_default();
514
515 tmp[i++] = 137;
516 tmp[i++] = 'H';
517 tmp[i++] = 'D';
518 tmp[i++] = 'F';
519
520 if ((r->read(r->read_data, buf, 4, &err) != 4) || err) {
521 return err;
522 }
523
524 if (!(buf[0] == '\r' && buf[1] == '\n' && buf[2] == 26 && buf[3] == '\n')) {
525 /* Invalid magic, we should not use grib_context_log without a context */
526 grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid signature");
527 return GRIB_INVALID_MESSAGE;
528 }
529
530 for (j = 0; j < 4; j++) {
531 tmp[i++] = buf[j];
532 }
533
534 if ((r->read(r->read_data, &version_of_superblock, 1, &err) != 1) || err) {
535 return err;
536 }
537
538 tmp[i++] = version_of_superblock;
539
540 if (version_of_superblock == 2 || version_of_superblock == 3) {
541 if ((r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err) {
542 return err;
543 }
544
545 tmp[i++] = size_of_offsets;
546
547 if (size_of_offsets > 8) {
548 grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets);
549 return GRIB_NOT_IMPLEMENTED;
550 }
551
552 if ((r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err) {
553 return err;
554 }
555
556 tmp[i++] = size_of_lengths;
557
558 if ((r->read(r->read_data, &consistency_flags, 1, &err) != 1) || err) {
559 return err;
560 }
561
562 tmp[i++] = consistency_flags;
563
564 err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i);
565 if (err) {
566 return err;
567 }
568
569 err = read_HDF5_offset(r, size_of_offsets, &superblock_extension_address, tmp, &i);
570 if (err) {
571 return err;
572 }
573
574 err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i);
575 if (err) {
576 return err;
577 }
578 }
579 else if (version_of_superblock == 0 || version_of_superblock == 1) {
580 char skip[4];
581 unsigned long file_free_space_info;
582 unsigned char version_of_file_free_space, version_of_root_group_symbol_table, version_number_shared_header, ch;
583
584 if ((r->read(r->read_data, &version_of_file_free_space, 1, &err) != 1) || err)
585 return err;
586 tmp[i++] = version_of_file_free_space;
587
588 if ((r->read(r->read_data, &version_of_root_group_symbol_table, 1, &err) != 1) || err)
589 return err;
590 tmp[i++] = version_of_root_group_symbol_table;
591
592 if ((r->read(r->read_data, &ch, 1, &err) != 1) || err)
593 return err; /* reserved */
594 tmp[i++] = ch;
595
596 if ((r->read(r->read_data, &version_number_shared_header, 1, &err) != 1) || err)
597 return err;
598 tmp[i++] = version_number_shared_header;
599
600 if ((r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err)
601 return err;
602 tmp[i++] = size_of_offsets;
603 if (size_of_offsets > 8) {
604 grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets);
605 return GRIB_NOT_IMPLEMENTED;
606 }
607
608 if ((r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err)
609 return err;
610 tmp[i++] = size_of_lengths;
611
612 if ((r->read(r->read_data, &ch, 1, &err) != 1) || err)
613 return err; /*reserved*/
614 tmp[i++] = ch;
615
616 if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
617 return err; /* Group Leaf/Internal Node K: 4 bytes */
618 tmp[i++] = skip[0];
619 tmp[i++] = skip[1];
620 tmp[i++] = skip[2];
621 tmp[i++] = skip[3];
622
623 if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
624 return err; /* consistency_flags: 4 bytes */
625 tmp[i++] = skip[0];
626 tmp[i++] = skip[1];
627 tmp[i++] = skip[2];
628 tmp[i++] = skip[3];
629
630 if (version_of_superblock == 1) {
631 /* Indexed storage internal node K and reserved: only in version 1 of superblock */
632 if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
633 return err;
634 tmp[i++] = skip[0];
635 tmp[i++] = skip[1];
636 tmp[i++] = skip[2];
637 tmp[i++] = skip[3];
638 }
639
640 err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i);
641 if (err)
642 return err;
643
644 err = read_HDF5_offset(r, size_of_offsets, &file_free_space_info, tmp, &i);
645 if (err)
646 return err;
647
648 err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i);
649 if (err)
650 return err;
651 }
652 else {
653 grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid version of superblock: %ld", (long)version_of_superblock);
654 return GRIB_NOT_IMPLEMENTED;
655 }
656
657 Assert(i <= sizeof(tmp));
658 return read_the_rest(r, end_of_file_address, tmp, i, 0);
659 }
660
read_WRAP(reader * r)661 static int read_WRAP(reader* r)
662 {
663 /*
664 * See: http://www.hdfgroup.org/HDF5/doc/H5.format.html#Superblock
665 */
666 unsigned char tmp[36]; /* Should be enough */
667 unsigned char buf[8];
668
669 unsigned long long length = 0;
670
671 int i = 0, j;
672 int err = 0;
673
674 tmp[i++] = 'W';
675 tmp[i++] = 'R';
676 tmp[i++] = 'A';
677 tmp[i++] = 'P';
678
679 if ((r->read(r->read_data, buf, 8, &err) != 8) || err) {
680 printf("error\n");
681 return err;
682 }
683
684 for (j = 0; j < 8; j++) {
685 length <<= 8;
686 length |= buf[j];
687 tmp[i++] = buf[j];
688 }
689
690 Assert(i <= sizeof(tmp));
691 return read_the_rest(r, length, tmp, i, 1);
692 }
693
read_BUFR(reader * r)694 static int read_BUFR(reader* r)
695 {
696 /* unsigned char tmp[65536];*/ /* Should be enough */
697 size_t length = 0;
698 long edition = 0;
699 int err = 0;
700 int i = 0, j;
701 size_t buflen = 2048;
702 unsigned char* tmp = NULL;
703 grib_context* c = NULL;
704 grib_buffer* buf = NULL;
705
706 /*TODO proper context*/
707 c = grib_context_get_default();
708 tmp = (unsigned char*)malloc(buflen);
709 if (!tmp)
710 return GRIB_OUT_OF_MEMORY;
711 buf = grib_new_buffer(c, tmp, buflen);
712 buf->property = GRIB_MY_BUFFER;
713 r->offset = r->tell(r->read_data) - 4;
714
715 tmp[i++] = 'B';
716 tmp[i++] = 'U';
717 tmp[i++] = 'F';
718 tmp[i++] = 'R';
719
720 for (j = 0; j < 3; j++) {
721 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
722 return err;
723
724 length <<= 8;
725 length |= tmp[i];
726 i++;
727 }
728
729 if (length == 0) {
730 grib_buffer_delete(c, buf);
731 return GRIB_INVALID_MESSAGE;
732 }
733
734 /* Edition number */
735 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
736 return err;
737
738 edition = tmp[i++];
739
740 /* Assert(edition != 1); */
741
742 switch (edition) {
743 case 0:
744 case 1: {
745 int n;
746 size_t sec1len = 0;
747 size_t sec2len = 0;
748 size_t sec3len = 0;
749 size_t sec4len = 0;
750 unsigned long flags;
751
752 sec1len = length;
753
754 /* table version */
755 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
756 return err;
757 /* center */
758 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
759 return err;
760 /* update */
761 if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
762 return err;
763 /* flags */
764 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
765 return err;
766 flags = tmp[i++];
767
768
769 GROW_BUF_IF_REQUIRED(sec1len + 4 + 3);
770
771 /* Read section 1. 3 = length, 5 = table,center,process,flags */
772
773 n = sec1len - 8; /* Just a guess */
774 if ((r->read(r->read_data, tmp + i, n, &err) != n) || err)
775 return err;
776
777 i += n;
778
779 if (flags & (1 << 7)) {
780 /* Section 2 */
781 for (j = 0; j < 3; j++) {
782 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
783 return err;
784
785 sec2len <<= 8;
786 sec2len |= tmp[i];
787 i++;
788 }
789
790 GROW_BUF_IF_REQUIRED(sec1len + sec2len + 4 + 3);
791
792 /* Read section 2 */
793 if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
794 return err;
795 i += sec2len - 3;
796 }
797
798
799 /* Section 3 */
800 for (j = 0; j < 3; j++) {
801 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
802 return err;
803
804 sec3len <<= 8;
805 sec3len |= tmp[i];
806 i++;
807 }
808
809 GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
810
811 /* Read section 3 */
812 if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
813 return err;
814 i += sec3len - 3;
815
816 for (j = 0; j < 3; j++) {
817 if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
818 return err;
819
820 sec4len <<= 8;
821 sec4len |= tmp[i];
822 i++;
823 }
824
825 /* fprintf(stderr," sec1len=%d sec2len=%d sec3len=%d sec4len=%d\n",sec1len, sec2len,sec3len,sec4len); */
826 length = 4 + sec1len + sec2len + sec3len + sec4len + 4;
827 /* fprintf(stderr,"length = %d i = %d\n",length,i); */
828 } break;
829 case 2:
830 case 3:
831 case 4:
832 break;
833 default:
834 r->seek_from_start(r->read_data, r->offset + 4);
835 grib_buffer_delete(c, buf);
836 return GRIB_UNSUPPORTED_EDITION;
837 }
838
839 /* Assert(i <= sizeof(tmp)); */
840 err = read_the_rest(r, length, tmp, i, 1);
841 if (err)
842 r->seek_from_start(r->read_data, r->offset + 4);
843
844 grib_buffer_delete(c, buf);
845
846 return err;
847 }
848
_read_any(reader * r,int grib_ok,int bufr_ok,int hdf5_ok,int wrap_ok)849 static int _read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
850 {
851 unsigned char c;
852 int err = 0;
853 unsigned long magic = 0;
854
855 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
856 magic <<= 8;
857 magic |= c;
858
859 switch (magic & 0xffffffff) {
860 case GRIB:
861 if (grib_ok) {
862 err = read_GRIB(r);
863 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
864 }
865 break;
866
867 case BUFR:
868 if (bufr_ok) {
869 err = read_BUFR(r);
870 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
871 }
872 break;
873
874 case HDF5:
875 if (hdf5_ok) {
876 err = read_HDF5(r);
877 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
878 }
879 break;
880
881 case WRAP:
882 if (wrap_ok) {
883 err = read_WRAP(r);
884 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
885 }
886 break;
887
888 case BUDG:
889 if (grib_ok) {
890 err = read_PSEUDO(r, "BUDG");
891 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
892 }
893 break;
894 case DIAG:
895 if (grib_ok) {
896 err = read_PSEUDO(r, "DIAG");
897 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
898 }
899 break;
900 case TIDE:
901 if (grib_ok) {
902 err = read_PSEUDO(r, "TIDE");
903 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
904 }
905 break;
906 }
907 }
908
909 return err;
910 }
911
read_any(reader * r,int grib_ok,int bufr_ok,int hdf5_ok,int wrap_ok)912 static int read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
913 {
914 int result = 0;
915
916 #ifndef ECCODES_EACH_THREAD_OWN_FILE
917 /* If several threads can open the same file, then we need the locks
918 * so each thread gets its own message. Otherwise if threads are passed
919 * different files, then the lock is not needed
920 */
921 GRIB_MUTEX_INIT_ONCE(&once, &init);
922 GRIB_MUTEX_LOCK(&mutex1);
923 #endif
924
925 result = _read_any(r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
926
927 #ifndef ECCODES_EACH_THREAD_OWN_FILE
928 GRIB_MUTEX_UNLOCK(&mutex1);
929 #endif
930 return result;
931 }
932
read_any_gts(reader * r)933 static int read_any_gts(reader* r)
934 {
935 unsigned char c;
936 int err = 0;
937 unsigned char* buffer = NULL;
938 unsigned long magic = 0;
939 unsigned long start = 0x010d0d0a; /* SOH CR CR LF */
940 unsigned long theEnd = 0x0d0d0a03; /* CR CR LF ETX */
941 unsigned char tmp[1024] = {0,}; /* See ECC-735 */
942 size_t message_size = 0;
943 size_t already_read = 0;
944 int i = 0;
945
946 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
947 magic <<= 8;
948 magic |= c;
949 magic &= 0xffffffff;
950
951 if (magic == start) {
952 tmp[i++] = 0x01;
953 tmp[i++] = 0x0d;
954 tmp[i++] = 0x0d;
955 tmp[i++] = 0x0a;
956
957 r->offset = r->tell(r->read_data) - 4;
958
959 if (r->read(r->read_data, &tmp[i], 6, &err) != 6 || err)
960 return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
961
962 if (tmp[7] != 0x0d || tmp[8] != 0x0d || tmp[9] != 0x0a) {
963 r->seek(r->read_data, -6);
964 continue;
965 }
966 magic = 0;
967 already_read = 10;
968 message_size = already_read;
969 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
970 message_size++;
971 magic <<= 8;
972 magic |= c;
973 magic &= 0xffffffff;
974 if (magic == theEnd) {
975 r->seek(r->read_data, already_read - message_size);
976 buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
977 if (!buffer)
978 return GRIB_OUT_OF_MEMORY;
979 if (err)
980 return err;
981 memcpy(buffer, tmp, already_read);
982 r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
983 r->message_size = message_size;
984 return err;
985 }
986 }
987 }
988 }
989
990 return err;
991 }
992
read_any_taf(reader * r)993 static int read_any_taf(reader* r)
994 {
995 unsigned char c;
996 int err = 0;
997 unsigned char* buffer = NULL;
998 unsigned long magic = 0;
999 unsigned long start = 0x54414620;
1000 unsigned char tmp[1000] = {0,}; /* Should be enough */
1001 size_t message_size = 0;
1002 size_t already_read = 0;
1003 int i = 0;
1004
1005 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
1006 magic <<= 8;
1007 magic |= c;
1008 magic &= 0xffffffff;
1009
1010 if (magic == start) {
1011 tmp[i++] = 0x54;
1012 tmp[i++] = 0x41;
1013 tmp[i++] = 0x46;
1014 tmp[i++] = 0x20;
1015
1016 r->offset = r->tell(r->read_data) - 4;
1017
1018 already_read = 4;
1019 message_size = already_read;
1020 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
1021 message_size++;
1022 if (c == '=') {
1023 r->seek(r->read_data, already_read - message_size);
1024 buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
1025 if (!buffer)
1026 return GRIB_OUT_OF_MEMORY;
1027 if (err)
1028 return err;
1029 memcpy(buffer, tmp, already_read);
1030 r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
1031 r->message_size = message_size;
1032 return err;
1033 }
1034 }
1035 }
1036 }
1037
1038 return err;
1039 }
1040
read_any_metar(reader * r)1041 static int read_any_metar(reader* r)
1042 {
1043 unsigned char c;
1044 int err = 0;
1045 unsigned char* buffer = NULL;
1046 unsigned long magic = 0;
1047 unsigned long start = 0x4d455441;
1048 unsigned char tmp[32] = {0,}; /* Should be enough */
1049 size_t message_size = 0;
1050 size_t already_read = 0;
1051 int i = 0;
1052
1053 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
1054 magic <<= 8;
1055 magic |= c;
1056 magic &= 0xffffffff;
1057
1058 if (magic == start) {
1059 if (r->read(r->read_data, &c, 1, &err) != 1 || err != 0)
1060 break;
1061 if (c == 'R') {
1062 tmp[i++] = 0x4d;
1063 tmp[i++] = 0x45;
1064 tmp[i++] = 0x54;
1065 tmp[i++] = 0x41;
1066 tmp[i++] = 'R';
1067
1068 r->offset = r->tell(r->read_data) - 4;
1069
1070 already_read = 5;
1071 message_size = already_read;
1072 while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
1073 message_size++;
1074 if (c == '=') {
1075 r->seek(r->read_data, already_read - message_size);
1076 buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
1077 if (!buffer)
1078 return GRIB_OUT_OF_MEMORY;
1079 if (err)
1080 return err;
1081 memcpy(buffer, tmp, already_read);
1082 r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
1083 r->message_size = message_size;
1084 return err;
1085 }
1086 }
1087 }
1088 }
1089 }
1090
1091 return err;
1092 }
1093
stdio_tell(void * data)1094 off_t stdio_tell(void* data)
1095 {
1096 FILE* f = (FILE*)data;
1097 return ftello(f);
1098 }
1099
stdio_seek(void * data,off_t len)1100 int stdio_seek(void* data, off_t len)
1101 {
1102 FILE* f = (FILE*)data;
1103 int err = 0;
1104 if (fseeko(f, len, SEEK_CUR))
1105 err = GRIB_IO_PROBLEM;
1106 return err;
1107 }
1108
stdio_seek_from_start(void * data,off_t len)1109 int stdio_seek_from_start(void* data, off_t len)
1110 {
1111 FILE* f = (FILE*)data;
1112 int err = 0;
1113 if (fseeko(f, len, SEEK_SET))
1114 err = GRIB_IO_PROBLEM;
1115 return err;
1116 }
1117
stdio_read(void * data,void * buf,size_t len,int * err)1118 size_t stdio_read(void* data, void* buf, size_t len, int* err)
1119 {
1120 FILE* f = (FILE*)data;
1121 size_t n;
1122 /* char iobuf[1024*1024]; */
1123
1124 if (len == 0)
1125 return 0;
1126
1127 /* setvbuf(f,iobuf,_IOFBF,sizeof(iobuf)); */
1128 n = fread(buf, 1, len, f);
1129 /* fprintf(stderr,"read %d = %x %c\n",1,(int)buf[0],buf[0]); */
1130 if (n != len) {
1131 /* fprintf(stderr,"Failed to read %d, only got %d\n",len,n); */
1132 *err = GRIB_IO_PROBLEM;
1133 if (feof(f))
1134 *err = GRIB_END_OF_FILE;
1135 if (ferror(f))
1136 *err = GRIB_IO_PROBLEM;
1137 }
1138 return n;
1139 }
1140
1141 /*================== */
1142 typedef struct user_buffer
1143 {
1144 void* user_buffer;
1145 size_t buffer_size;
1146 } user_buffer;
1147
user_provider_buffer(void * data,size_t * length,int * err)1148 static void* user_provider_buffer(void* data, size_t* length, int* err)
1149 {
1150 user_buffer* u = (user_buffer*)data;
1151 *length = u->buffer_size;
1152 return u->user_buffer;
1153 }
1154
_wmo_read_any_from_file(FILE * f,void * buffer,size_t * len,int grib_ok,int bufr_ok,int hdf5_ok,int wrap_ok)1155 static int _wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
1156 {
1157 int err;
1158 user_buffer u;
1159 reader r;
1160
1161 u.user_buffer = buffer;
1162 u.buffer_size = *len;
1163
1164 r.message_size = 0;
1165 r.read_data = f;
1166 r.read = &stdio_read;
1167 r.seek = &stdio_seek;
1168 r.seek_from_start = &stdio_seek_from_start;
1169 r.tell = &stdio_tell;
1170 r.alloc_data = &u;
1171 r.alloc = &user_provider_buffer;
1172 r.headers_only = 0;
1173
1174 err = read_any(&r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
1175 *len = r.message_size;
1176
1177 return err;
1178 }
1179
wmo_read_any_from_file(FILE * f,void * buffer,size_t * len)1180 int wmo_read_any_from_file(FILE* f, void* buffer, size_t* len)
1181 {
1182 return _wmo_read_any_from_file(f, buffer, len, 1, 1, 1, 1);
1183 }
1184
wmo_read_grib_from_file(FILE * f,void * buffer,size_t * len)1185 int wmo_read_grib_from_file(FILE* f, void* buffer, size_t* len)
1186 {
1187 return _wmo_read_any_from_file(f, buffer, len, 1, 0, 0, 0);
1188 }
1189
wmo_read_bufr_from_file(FILE * f,void * buffer,size_t * len)1190 int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len)
1191 {
1192 return _wmo_read_any_from_file(f, buffer, len, 0, 1, 0, 0);
1193 }
1194
wmo_read_gts_from_file(FILE * f,void * buffer,size_t * len)1195 int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len)
1196 {
1197 int err;
1198 user_buffer u;
1199 reader r;
1200
1201 u.user_buffer = buffer;
1202 u.buffer_size = *len;
1203
1204 r.message_size = 0;
1205 r.read_data = f;
1206 r.read = &stdio_read;
1207 r.seek = &stdio_seek;
1208 r.seek_from_start = &stdio_seek_from_start;
1209 r.tell = &stdio_tell;
1210 r.alloc_data = &u;
1211 r.alloc = &user_provider_buffer;
1212 r.headers_only = 0;
1213
1214 err = read_any_gts(&r);
1215 *len = r.message_size;
1216
1217 return err;
1218 }
1219
wmo_read_taf_from_file(FILE * f,void * buffer,size_t * len)1220 int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len)
1221 {
1222 int err;
1223 user_buffer u;
1224 reader r;
1225
1226 u.user_buffer = buffer;
1227 u.buffer_size = *len;
1228
1229 r.read_data = f;
1230 r.read = &stdio_read;
1231 r.seek = &stdio_seek;
1232 r.seek_from_start = &stdio_seek_from_start;
1233 r.tell = &stdio_tell;
1234 r.alloc_data = &u;
1235 r.alloc = &user_provider_buffer;
1236 r.headers_only = 0;
1237
1238 err = read_any_taf(&r);
1239 *len = r.message_size;
1240
1241 return err;
1242 }
1243
wmo_read_metar_from_file(FILE * f,void * buffer,size_t * len)1244 int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len)
1245 {
1246 int err;
1247 user_buffer u;
1248 reader r;
1249
1250 u.user_buffer = buffer;
1251 u.buffer_size = *len;
1252
1253 r.read_data = f;
1254 r.read = &stdio_read;
1255 r.seek = &stdio_seek;
1256 r.seek_from_start = &stdio_seek_from_start;
1257 r.tell = &stdio_tell;
1258 r.alloc_data = &u;
1259 r.alloc = &user_provider_buffer;
1260 r.headers_only = 0;
1261
1262 err = read_any_metar(&r);
1263 *len = r.message_size;
1264
1265 return err;
1266 }
1267
1268 /*================== */
1269
1270 typedef struct stream_struct
1271 {
1272 void* stream_data;
1273 long (*stream_proc)(void*, void* buffer, long len);
1274
1275 } stream_struct;
1276
stream_tell(void * data)1277 static off_t stream_tell(void* data)
1278 {
1279 return 0;
1280 }
1281
stream_seek(void * data,off_t len)1282 static int stream_seek(void* data, off_t len)
1283 {
1284 return 0;
1285 }
stream_read(void * data,void * buffer,size_t len,int * err)1286 static size_t stream_read(void* data, void* buffer, size_t len, int* err)
1287 {
1288 stream_struct* s = (stream_struct*)data;
1289 long n = len;
1290
1291 if (n != len) {
1292 /* size_t cannot be coded into long */
1293 *err = GRIB_INTERNAL_ERROR;
1294 return -1;
1295 }
1296
1297 n = s->stream_proc(s->stream_data, buffer, len);
1298 if (n != len) {
1299 *err = GRIB_IO_PROBLEM;
1300 if (n == -1)
1301 *err = GRIB_END_OF_FILE;
1302 }
1303 return n;
1304 }
1305
1306 /*================== */
1307
1308
allocate_buffer(void * data,size_t * length,int * err)1309 static void* allocate_buffer(void* data, size_t* length, int* err)
1310 {
1311 alloc_buffer* u = (alloc_buffer*)data;
1312 u->buffer = malloc(*length);
1313 u->size = *length;
1314 if (u->buffer == NULL)
1315 *err = GRIB_OUT_OF_MEMORY; /* Cannot allocate buffer */
1316 return u->buffer;
1317 }
1318
wmo_read_any_from_stream(void * stream_data,long (* stream_proc)(void *,void * buffer,long len),void * buffer,size_t * len)1319 int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len)
1320 {
1321 int err;
1322 stream_struct s;
1323 user_buffer u;
1324 reader r;
1325
1326 s.stream_data = stream_data;
1327 s.stream_proc = stream_proc;
1328
1329 u.user_buffer = buffer;
1330 u.buffer_size = *len;
1331
1332 r.message_size = 0;
1333 r.offset = 0;
1334 r.read_data = &s;
1335 r.read = &stream_read;
1336 r.seek = &stream_seek;
1337 r.seek_from_start = &stream_seek;
1338 r.tell = &stream_tell;
1339 r.alloc_data = &u;
1340 r.alloc = &user_provider_buffer;
1341 r.headers_only = 0;
1342
1343 err = read_any(&r, 1, 1, 1, 1);
1344 *len = r.message_size;
1345
1346 return err;
1347 }
1348
1349 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_any_from_stream_malloc(void * stream_data,long (* stream_proc)(void *,void * buffer,long len),size_t * size,int * err)1350 void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err)
1351 {
1352 alloc_buffer u;
1353 stream_struct s;
1354 reader r;
1355
1356 u.buffer = NULL;
1357
1358 s.stream_data = stream_data;
1359 s.stream_proc = stream_proc;
1360
1361 r.message_size = 0;
1362 r.offset = 0;
1363 r.read_data = &s;
1364 r.read = &stream_read;
1365 r.seek = &stream_seek;
1366 r.seek_from_start = &stream_seek;
1367 r.tell = &stream_tell;
1368 r.alloc_data = &u;
1369 r.alloc = &allocate_buffer;
1370 r.headers_only = 0;
1371
1372 *err = read_any(&r, 1, 1, 1, 1);
1373 *size = r.message_size;
1374
1375 return u.buffer;
1376 }
1377
1378 /*================== */
1379
1380 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_gts_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1381 void* wmo_read_gts_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1382 {
1383 alloc_buffer u;
1384 reader r;
1385
1386 u.buffer = NULL;
1387 r.offset = 0;
1388
1389 r.message_size = 0;
1390 r.read_data = f;
1391 r.read = &stdio_read;
1392 r.seek = &stdio_seek;
1393 r.seek_from_start = &stdio_seek_from_start;
1394 r.tell = &stdio_tell;
1395 r.alloc_data = &u;
1396 r.alloc = &allocate_buffer;
1397 r.headers_only = headers_only;
1398
1399 *err = read_any_gts(&r);
1400 *size = r.message_size;
1401 *offset = r.offset;
1402
1403 return u.buffer;
1404 }
1405
1406 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_taf_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1407 void* wmo_read_taf_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1408 {
1409 alloc_buffer u;
1410 reader r;
1411
1412 u.buffer = NULL;
1413
1414 r.offset = 0;
1415 r.message_size = 0;
1416 r.read_data = f;
1417 r.read = &stdio_read;
1418 r.seek = &stdio_seek;
1419 r.seek_from_start = &stdio_seek_from_start;
1420 r.tell = &stdio_tell;
1421 r.alloc_data = &u;
1422 r.alloc = &allocate_buffer;
1423 r.headers_only = headers_only;
1424
1425 *err = read_any_taf(&r);
1426 *size = r.message_size;
1427 *offset = r.offset;
1428
1429 return u.buffer;
1430 }
1431
1432 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_metar_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1433 void* wmo_read_metar_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1434 {
1435 alloc_buffer u;
1436 reader r;
1437
1438 u.buffer = NULL;
1439
1440 r.message_size = 0;
1441 r.read_data = f;
1442 r.offset = 0;
1443 r.read = &stdio_read;
1444 r.seek = &stdio_seek;
1445 r.seek_from_start = &stdio_seek_from_start;
1446 r.tell = &stdio_tell;
1447 r.alloc_data = &u;
1448 r.alloc = &allocate_buffer;
1449 r.headers_only = headers_only;
1450
1451 *err = read_any_metar(&r);
1452 *size = r.message_size;
1453 *offset = r.offset;
1454
1455 return u.buffer;
1456 }
1457
1458 /* This function allocates memory for the result so the user is responsible for freeing it */
_wmo_read_any_from_file_malloc(FILE * f,int * err,size_t * size,off_t * offset,int grib_ok,int bufr_ok,int hdf5_ok,int wrap_ok,int headers_only)1459 static void* _wmo_read_any_from_file_malloc(FILE* f, int* err, size_t* size, off_t* offset,
1460 int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok, int headers_only)
1461 {
1462 alloc_buffer u;
1463 reader r;
1464
1465 u.buffer = NULL;
1466 u.size = 0;
1467
1468 r.message_size = 0;
1469 r.read_data = f;
1470 r.read = &stdio_read;
1471 r.seek = &stdio_seek;
1472 r.seek_from_start = &stdio_seek_from_start;
1473 r.tell = &stdio_tell;
1474 r.alloc_data = &u;
1475 r.alloc = &allocate_buffer;
1476 r.headers_only = headers_only;
1477 r.offset = 0;
1478
1479 *err = read_any(&r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
1480
1481 *size = r.message_size;
1482 *offset = r.offset;
1483
1484 return u.buffer;
1485 }
1486
1487 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_any_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1488 void* wmo_read_any_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1489 {
1490 return _wmo_read_any_from_file_malloc(f, err, size, offset, 1, 1, 1, 1, headers_only);
1491 }
1492 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_grib_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1493 void* wmo_read_grib_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1494 {
1495 return _wmo_read_any_from_file_malloc(f, err, size, offset, 1, 0, 0, 0, headers_only);
1496 }
1497 /* This function allocates memory for the result so the user is responsible for freeing it */
wmo_read_bufr_from_file_malloc(FILE * f,int headers_only,size_t * size,off_t * offset,int * err)1498 void* wmo_read_bufr_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
1499 {
1500 return _wmo_read_any_from_file_malloc(f, err, size, offset, 0, 1, 0, 0, headers_only);
1501 }
1502
1503 /* ======================================= */
1504
1505 typedef struct context_alloc_buffer
1506 {
1507 grib_context* ctx;
1508 void* buffer;
1509 size_t length;
1510 } context_alloc_buffer;
1511
context_allocate_buffer(void * data,size_t * length,int * err)1512 static void* context_allocate_buffer(void* data, size_t* length, int* err)
1513 {
1514 context_alloc_buffer* u = (context_alloc_buffer*)data;
1515 u->buffer = grib_context_malloc(u->ctx, *length);
1516 u->length = *length;
1517
1518 if (u->buffer == NULL)
1519 *err = GRIB_OUT_OF_MEMORY; /* Cannot allocate buffer */
1520 return u->buffer;
1521 }
1522
grib_read_any_headers_only_from_file(grib_context * ctx,FILE * f,void * buffer,size_t * len)1523 int grib_read_any_headers_only_from_file(grib_context* ctx, FILE* f, void* buffer, size_t* len)
1524 {
1525 int err;
1526 user_buffer u;
1527 reader r;
1528
1529 u.user_buffer = buffer;
1530 u.buffer_size = *len;
1531
1532 r.message_size = 0;
1533 r.read_data = f;
1534 r.read = &stdio_read;
1535 r.seek = &stdio_seek;
1536 r.seek_from_start = &stdio_seek_from_start;
1537 r.tell = &stdio_tell;
1538 r.alloc_data = &u;
1539 r.alloc = &user_provider_buffer;
1540 r.headers_only = 1;
1541
1542 err = read_any(&r, 1, ECCODES_READS_BUFR, ECCODES_READS_HDF5, ECCODES_READS_WRAP);
1543
1544 *len = r.message_size;
1545
1546 return err;
1547 }
1548
grib_read_any_from_file(grib_context * ctx,FILE * f,void * buffer,size_t * len)1549 int grib_read_any_from_file(grib_context* ctx, FILE* f, void* buffer, size_t* len)
1550 {
1551 int err;
1552 user_buffer u;
1553 reader r;
1554 off_t offset;
1555
1556 u.user_buffer = buffer;
1557 u.buffer_size = *len;
1558
1559 r.message_size = 0;
1560 r.read_data = f;
1561 r.read = &stdio_read;
1562 r.seek = &stdio_seek;
1563 r.seek_from_start = &stdio_seek_from_start;
1564 r.tell = &stdio_tell;
1565 r.alloc_data = &u;
1566 r.alloc = &user_provider_buffer;
1567 r.headers_only = 0;
1568
1569 offset = ftello(f);
1570
1571 err = read_any(&r, 1, ECCODES_READS_BUFR, ECCODES_READS_HDF5, ECCODES_READS_WRAP);
1572
1573 if (err == GRIB_BUFFER_TOO_SMALL) {
1574 if (fseeko(f, offset, SEEK_SET))
1575 err = GRIB_IO_PROBLEM;
1576 }
1577
1578 *len = r.message_size;
1579
1580 return err;
1581 }
1582
1583 /* ======================================= */
1584 typedef struct memory_read_data
1585 {
1586 unsigned char* data;
1587 size_t data_len;
1588 } memory_read_data;
1589
memory_tell(void * data)1590 static off_t memory_tell(void* data)
1591 {
1592 return 0;
1593 }
1594
memory_seek(void * data,off_t len)1595 static int memory_seek(void* data, off_t len)
1596 {
1597 return 0;
1598 }
1599
memory_read(void * data,void * buf,size_t len,int * err)1600 static size_t memory_read(void* data, void* buf, size_t len, int* err)
1601 {
1602 memory_read_data* m = (memory_read_data*)data;
1603
1604 if (len == 0) {
1605 *err = GRIB_END_OF_FILE;
1606 return 0;
1607 }
1608 else {
1609 size_t l = len > m->data_len ? m->data_len : len;
1610 memcpy(buf, m->data, l);
1611 m->data_len -= l;
1612 m->data += l;
1613 return l;
1614 }
1615 }
1616
grib_read_any_from_memory_alloc(grib_context * ctx,unsigned char ** data,size_t * data_length,void ** buffer,size_t * length)1617 int grib_read_any_from_memory_alloc(grib_context* ctx, unsigned char** data, size_t* data_length, void** buffer, size_t* length)
1618 {
1619 int err;
1620 memory_read_data m;
1621 context_alloc_buffer u;
1622 reader r;
1623
1624 m.data = *data;
1625 m.data_len = *data_length;
1626
1627 u.buffer = NULL;
1628 u.length = 0;
1629 u.ctx = ctx ? ctx : grib_context_get_default();
1630
1631 r.message_size = 0;
1632 r.read_data = &m;
1633 r.read = &memory_read;
1634 r.seek = &memory_seek;
1635 r.seek_from_start = &memory_seek;
1636 r.tell = &memory_tell;
1637 r.alloc_data = &u;
1638 r.alloc = &context_allocate_buffer;
1639 r.headers_only = 0;
1640
1641 err = read_any(&r, 1, ECCODES_READS_BUFR, ECCODES_READS_HDF5, ECCODES_READS_WRAP);
1642 *buffer = u.buffer;
1643 *length = u.length;
1644
1645 *data_length = m.data_len;
1646 *data = m.data;
1647
1648 return err;
1649 }
1650
grib_read_any_from_memory(grib_context * ctx,unsigned char ** data,size_t * data_length,void * buffer,size_t * len)1651 int grib_read_any_from_memory(grib_context* ctx, unsigned char** data, size_t* data_length, void* buffer, size_t* len)
1652 {
1653 int err;
1654 memory_read_data m;
1655 user_buffer u;
1656 reader r;
1657
1658 m.data = *data;
1659 m.data_len = *data_length;
1660
1661 u.user_buffer = buffer;
1662 u.buffer_size = *len;
1663
1664 r.message_size = 0;
1665 r.read_data = &m;
1666 r.read = &memory_read;
1667 r.seek = &memory_seek;
1668 r.seek_from_start = &memory_seek;
1669 r.tell = &memory_tell;
1670 r.alloc_data = &u;
1671 r.alloc = &user_provider_buffer;
1672 r.headers_only = 0;
1673
1674 err = read_any(&r, 1, ECCODES_READS_BUFR, ECCODES_READS_HDF5, ECCODES_READS_WRAP);
1675 *len = r.message_size;
1676
1677 *data_length = m.data_len;
1678 *data = m.data;
1679
1680 return err;
1681 }
1682
grib_count_in_file(grib_context * c,FILE * f,int * n)1683 int grib_count_in_file(grib_context* c, FILE* f, int* n)
1684 {
1685 int err = 0;
1686 *n = 0;
1687 if (!c)
1688 c = grib_context_get_default();
1689
1690 if (c->multi_support_on) {
1691 /* GRIB-395 */
1692 grib_handle* h = NULL;
1693 while ((h = grib_handle_new_from_file(c, f, &err)) != NULL) {
1694 grib_handle_delete(h);
1695 (*n)++;
1696 }
1697 }
1698 else {
1699 void* mesg = NULL;
1700 size_t size = 0;
1701 off_t offset = 0;
1702 while ((mesg = wmo_read_any_from_file_malloc(f, 0, &size, &offset, &err)) != NULL && err == GRIB_SUCCESS) {
1703 grib_context_free(c, mesg);
1704 (*n)++;
1705 }
1706 }
1707
1708 rewind(f);
1709
1710 return err == GRIB_END_OF_FILE ? 0 : err;
1711 }
1712
grib_count_in_filename(grib_context * c,const char * filename,int * n)1713 int grib_count_in_filename(grib_context* c, const char* filename, int* n)
1714 {
1715 int err = 0;
1716 FILE* fp = NULL;
1717 if (!c)
1718 c = grib_context_get_default();
1719 fp = fopen(filename, "rb");
1720 if (!fp) {
1721 grib_context_log(c, GRIB_LOG_ERROR, "grib_count_in_filename: Unable to read file \"%s\"", filename);
1722 perror(filename);
1723 return GRIB_IO_PROBLEM;
1724 }
1725 err = grib_count_in_file(c, fp, n);
1726 fclose(fp);
1727 return err;
1728 }
1729
1730 typedef void* (*decoder_proc)(FILE* f, int headers_only, size_t* size, off_t* offset, int* err);
1731
get_reader_for_product(ProductKind product)1732 static decoder_proc get_reader_for_product(ProductKind product)
1733 {
1734 decoder_proc decoder = NULL;
1735 if (product == PRODUCT_GRIB) decoder = &wmo_read_grib_from_file_malloc;
1736 else if (product == PRODUCT_BUFR) decoder = &wmo_read_bufr_from_file_malloc;
1737 else if (product == PRODUCT_GTS) decoder = &wmo_read_gts_from_file_malloc;
1738 else if (product == PRODUCT_ANY) decoder = &wmo_read_any_from_file_malloc;
1739 return decoder;
1740 }
1741
count_product_in_file(grib_context * c,FILE * f,ProductKind product,int * count)1742 static int count_product_in_file(grib_context* c, FILE* f, ProductKind product, int* count)
1743 {
1744 int err = 0;
1745 decoder_proc decoder = NULL;
1746
1747 *count = 0;
1748 if (!c) c = grib_context_get_default();
1749 decoder = get_reader_for_product(product);
1750
1751 if (!decoder) {
1752 grib_context_log(c, GRIB_LOG_ERROR, "count_product_in_file: not supported for given product");
1753 return GRIB_INVALID_ARGUMENT;
1754 }
1755
1756 if (c->multi_support_on && product == PRODUCT_GRIB) {
1757 grib_context_log(c, GRIB_LOG_ERROR, "count_product_in_file: Multi-field GRIBs not supported");
1758 err = GRIB_NOT_IMPLEMENTED;
1759 }
1760 else {
1761 void* mesg = NULL;
1762 size_t size = 0;
1763 off_t offset = 0;
1764 while ((mesg = decoder(f, 0, &size, &offset, &err)) != NULL && err == GRIB_SUCCESS) {
1765 grib_context_free(c, mesg);
1766 (*count)++;
1767 }
1768 rewind(f);
1769 }
1770
1771 return err == GRIB_END_OF_FILE ? 0 : err;
1772 }
1773
codes_extract_offsets_malloc(grib_context * c,const char * filename,ProductKind product,off_t ** offsets,int * length,int strict_mode)1774 int codes_extract_offsets_malloc(grib_context* c, const char* filename, ProductKind product, off_t** offsets, int* length, int strict_mode)
1775 {
1776 int err = 0;
1777 void* mesg = NULL;
1778 size_t size = 0;
1779 off_t offset = 0;
1780 int num_messages = 0, i = 0;
1781 decoder_proc decoder = NULL;
1782 FILE* f = NULL;
1783
1784 decoder = get_reader_for_product(product);
1785 if (!decoder) {
1786 grib_context_log(c, GRIB_LOG_ERROR, "codes_extract_offsets_malloc: not supported for given product");
1787 return GRIB_INVALID_ARGUMENT;
1788 }
1789 if (!c) c = grib_context_get_default();
1790 f = fopen(filename, "rb");
1791 if (!f) {
1792 grib_context_log(c, GRIB_LOG_ERROR, "codes_extract_offsets_malloc: Unable to read file \"%s\"", filename);
1793 perror(filename);
1794 return GRIB_IO_PROBLEM;
1795 }
1796
1797 err = count_product_in_file(c, f, product, &num_messages);
1798 if (err) {
1799 grib_context_log(c, GRIB_LOG_ERROR, "codes_extract_offsets_malloc: Unable to count messages");
1800 fclose(f);
1801 return err;
1802 }
1803 *length = num_messages;
1804 if (num_messages == 0) {
1805 grib_context_log(c, GRIB_LOG_ERROR, "codes_extract_offsets_malloc: No messages in file");
1806 fclose(f);
1807 return GRIB_INVALID_MESSAGE;
1808 }
1809 *offsets = (off_t*)calloc(num_messages, sizeof(off_t));
1810 if (!*offsets) {
1811 fclose(f);
1812 return GRIB_OUT_OF_MEMORY;
1813 }
1814
1815 i = 0;
1816 while (err != GRIB_END_OF_FILE) {
1817 if (i >= num_messages)
1818 break;
1819
1820 mesg = decoder(f, 0, &size, &offset, &err);
1821 if (mesg != NULL && err == 0) {
1822 (*offsets)[i] = offset;
1823 grib_context_free(c, mesg);
1824 }
1825 if (mesg && err) {
1826 if (strict_mode) {
1827 grib_context_free(c, mesg);
1828 fclose(f);
1829 return GRIB_DECODING_ERROR;
1830 }
1831 }
1832 if (!mesg) {
1833 if (err != GRIB_END_OF_FILE && err != GRIB_PREMATURE_END_OF_FILE) {
1834 /* An error occurred */
1835 grib_context_log(c, GRIB_LOG_ERROR, "codes_extract_offsets_malloc: Unable to read message");
1836 if (strict_mode) {
1837 fclose(f);
1838 return GRIB_DECODING_ERROR;
1839 }
1840 }
1841 }
1842 ++i;
1843 }
1844
1845 fclose(f);
1846 return err;
1847 }
1848