1 /****************************************************************************
2 *
3 * Routines to manage files of Mini-SEED.
4 *
5 * Written by Chad Trabant
6 * IRIS Data Management Center
7 *
8 * modified: 2015.108
9 ***************************************************************************/
10
11 #include <errno.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sys/stat.h>
16 #include <sys/types.h>
17 #include <time.h>
18
19 #include "libmseed.h"
20
21 static int ms_fread (char *buf, int size, int num, FILE *stream);
22
23 /* Pack type parameters for the 8 defined types:
24 * [type] : [hdrlen] [sizelen] [chksumlen]
25 */
26 int8_t packtypes[9][3] = {
27 {0, 0, 0},
28 {8, 8, 8},
29 {11, 8, 8},
30 {11, 8, 8},
31 {11, 8, 8},
32 {11, 8, 8},
33 {13, 8, 8},
34 {15, 8, 8},
35 {22, 15, 10}};
36
37 /*********************************************************************
38 * Notes about packed files as read by ms_readmsr_main()
39 *
40 * In general a packed file includes a pack file identifier at the
41 * very beginning, followed by pack header for a data block, followed
42 * by the data block, followed by a chksum for the data block. The
43 * pack header, data block and chksum are then repeated for each data
44 * block in the file:
45 *
46 * ID HDR DATA CHKSUM HDR DATA CHKSUM
47 * |----|-------|--....--|--------|-------|--....--|--------| ...
48 *
49 * |________ repeats ________|
50 *
51 * The HDR section contains fixed width ASCII fields identifying the
52 * data in the next section and it's length in bytes. With this
53 * information the offset of the next CHKSUM and HDR are completely
54 * predictable.
55 *
56 * packtypes[type][0]: length of pack header length
57 * packtypes[type][1]: length of size field in pack header
58 * packtypes[type][2]: chksum length following data blocks, skipped
59 *
60 * Notes from seed_pack.h documenting the PQI and PLS pack types:
61 *
62 * ___________________________________________________________________
63 * There were earlier pack file types numbered 1 through 6. These have been discontinued.
64 * Current file formats can be described as follows:
65 *
66 * Quality-Indexed Pack - Type 7:
67 * _____10_____2__2___3_____8_______mod 256_______8_____2__2___3_____8_______mod 256_______8____ ...
68 * |PQI- |q |lc|chn| size | ...data... | chksum |q |lc|chn| size | ...data... | chksum ...
69 * parsing guide:
70 * 10 | 15 hdr | xx | 8 | 15 hdr | xx
71 * |+0|+2|+4 |+7 |
72 *
73 *
74 * Large-Size Pack - Type 8: (for large channel blocks)
75 * _____10_____2__2___3_____15_______mod 256_______8____2__2__2___3_____15_______mod 256_______8____ ...
76 * |PLS-------|q |lc|chn| size | ...data... | chksum |--|q |lc|chn| size | ...data... | chksum ...
77 * uniform parsing guide:
78 * | 10 | 22 | xx | 10 | 22 | xx |
79 * |+0|+2|+4 |+7 |
80 * (note the use of hyphens after the PLS marker and just after the checksum. this will serve as a visual
81 * aid when scanning between channel blocks and provide consistent 32 byte spacing between data blocks)
82 * ___________________________________________________________________
83 *
84 *********************************************************************/
85
86 /* Initialize the global file reading parameters */
87 MSFileParam gMSFileParam = {NULL, "", NULL, 0, 0, 0, 0, 0, 0, 0};
88
89 /**********************************************************************
90 * ms_readmsr:
91 *
92 * This routine is a simple wrapper for ms_readmsr_main() that uses
93 * the global file reading parameters. This routine is not thread
94 * safe and cannot be used to read more than one file at a time.
95 *
96 * See the comments with ms_readmsr_main() for return values and
97 * further description of arguments.
98 *********************************************************************/
99 int
ms_readmsr(MSRecord ** ppmsr,const char * msfile,int reclen,off_t * fpos,int * last,flag skipnotdata,flag dataflag,flag verbose)100 ms_readmsr (MSRecord **ppmsr, const char *msfile, int reclen, off_t *fpos,
101 int *last, flag skipnotdata, flag dataflag, flag verbose)
102 {
103 MSFileParam *msfp = &gMSFileParam;
104
105 return ms_readmsr_main (&msfp, ppmsr, msfile, reclen, fpos,
106 last, skipnotdata, dataflag, NULL, verbose);
107 } /* End of ms_readmsr() */
108
109 /**********************************************************************
110 * ms_readmsr_r:
111 *
112 * This routine is a simple wrapper for ms_readmsr_main() that uses
113 * the re-entrant capabilities. This routine is thread safe and can
114 * be used to read more than one file at a time as long as separate
115 * MSFileParam structures are used for each file.
116 *
117 * See the comments with ms_readmsr_main() for return values and
118 * further description of arguments.
119 *********************************************************************/
120 int
ms_readmsr_r(MSFileParam ** ppmsfp,MSRecord ** ppmsr,const char * msfile,int reclen,off_t * fpos,int * last,flag skipnotdata,flag dataflag,flag verbose)121 ms_readmsr_r (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile,
122 int reclen, off_t *fpos, int *last, flag skipnotdata,
123 flag dataflag, flag verbose)
124 {
125 return ms_readmsr_main (ppmsfp, ppmsr, msfile, reclen, fpos,
126 last, skipnotdata, dataflag, NULL, verbose);
127 } /* End of ms_readmsr_r() */
128
129 /**********************************************************************
130 * ms_shift_msfp:
131 *
132 * A helper routine to shift (remove bytes from the beginning of) the
133 * file reading buffer for a MSFP. The buffer length, reading offset
134 * and file position indicators are all updated as necessary.
135 *
136 *********************************************************************/
137 static void
ms_shift_msfp(MSFileParam * msfp,int shift)138 ms_shift_msfp (MSFileParam *msfp, int shift)
139 {
140 if (!msfp)
141 return;
142
143 if (shift <= 0 && shift > msfp->readlen)
144 {
145 ms_log (2, "ms_shift_msfp(): Cannot shift buffer, shift: %d, readlen: %d, readoffset: %d\n",
146 shift, msfp->readlen, msfp->readoffset);
147 return;
148 }
149
150 memmove (msfp->rawrec, msfp->rawrec + shift, msfp->readlen - shift);
151 msfp->readlen -= shift;
152
153 if (shift < msfp->readoffset)
154 {
155 msfp->readoffset -= shift;
156 }
157 else
158 {
159 msfp->filepos += (shift - msfp->readoffset);
160 msfp->readoffset = 0;
161 }
162
163 return;
164 } /* End of ms_shift_msfp() */
165
166 /* Macro to calculate length of unprocessed buffer */
167 #define MSFPBUFLEN(MSFP) (MSFP->readlen - MSFP->readoffset)
168
169 /* Macro to return current reading position */
170 #define MSFPREADPTR(MSFP) (MSFP->rawrec + MSFP->readoffset)
171
172 /**********************************************************************
173 * ms_readmsr_main:
174 *
175 * This routine will open and read, with subsequent calls, all
176 * Mini-SEED records in specified file.
177 *
178 * All static file reading parameters are stored in a MSFileParam
179 * struct and returned (via a pointer to a pointer) for the calling
180 * routine to use in subsequent calls. A MSFileParam struct will be
181 * allocated if necessary. This routine is thread safe and can be
182 * used to read multiple files in parallel as long as the file reading
183 * parameters are managed appropriately.
184 *
185 * If reclen is 0 or negative the length of every record is
186 * automatically detected. For auto detection of record length the
187 * record must include a 1000 blockette or be followed by a valid
188 * record header or end of file.
189 *
190 * If *fpos is not NULL it will be updated to reflect the file
191 * position (offset from the beginning in bytes) from where the
192 * returned record was read. As a special case, if *fpos is not NULL
193 * and the value it points to is less than 0 this will be interpreted
194 * as a (positive) starting offset from which to begin reading data;
195 * this feature does not work with packed files.
196 *
197 * If *last is not NULL it will be set to 1 when the last record in
198 * the file is being returned, otherwise it will be 0.
199 *
200 * If the skipnotdata flag is true any data chunks read that do not
201 * have valid data record indicators (D, R, Q, M, etc.) will be skipped.
202 *
203 * dataflag will be passed directly to msr_unpack().
204 *
205 * If a Selections list is supplied it will be used to determine when
206 * a section of data in a packed file may be skipped, packed files are
207 * internal to the IRIS DMC.
208 *
209 * After reading all the records in a file the controlling program
210 * should call it one last time with msfile set to NULL. This will
211 * close the file and free allocated memory.
212 *
213 * Returns MS_NOERROR and populates an MSRecord struct at *ppmsr on
214 * successful read, returns MS_ENDOFFILE on EOF, otherwise returns a
215 * libmseed error code (listed in libmseed.h) and *ppmsr is set to
216 * NULL.
217 *********************************************************************/
218 int
ms_readmsr_main(MSFileParam ** ppmsfp,MSRecord ** ppmsr,const char * msfile,int reclen,off_t * fpos,int * last,flag skipnotdata,flag dataflag,Selections * selections,flag verbose)219 ms_readmsr_main (MSFileParam **ppmsfp, MSRecord **ppmsr, const char *msfile,
220 int reclen, off_t *fpos, int *last, flag skipnotdata,
221 flag dataflag, Selections *selections, flag verbose)
222 {
223 MSFileParam *msfp;
224 off_t packdatasize = 0;
225 int packskipsize;
226 int parseval = 0;
227 int readsize = 0;
228 int readcount = 0;
229 int retcode = MS_NOERROR;
230
231 if (!ppmsr)
232 return MS_GENERROR;
233
234 if (!ppmsfp)
235 return MS_GENERROR;
236
237 msfp = *ppmsfp;
238
239 /* Initialize the file read parameters if needed */
240 if (!msfp)
241 {
242 msfp = (MSFileParam *)malloc (sizeof (MSFileParam));
243
244 if (msfp == NULL)
245 {
246 ms_log (2, "ms_readmsr_main(): Cannot allocate memory for MSFP\n");
247 return MS_GENERROR;
248 }
249
250 /* Redirect the supplied pointer to the allocated params */
251 *ppmsfp = msfp;
252
253 msfp->fp = NULL;
254 msfp->filename[0] = '\0';
255 msfp->rawrec = NULL;
256 msfp->readlen = 0;
257 msfp->readoffset = 0;
258 msfp->packtype = 0;
259 msfp->packhdroffset = 0;
260 msfp->filepos = 0;
261 msfp->filesize = 0;
262 msfp->recordcount = 0;
263 }
264
265 /* When cleanup is requested */
266 if (msfile == NULL)
267 {
268 msr_free (ppmsr);
269
270 if (msfp->fp != NULL)
271 fclose (msfp->fp);
272
273 if (msfp->rawrec != NULL)
274 free (msfp->rawrec);
275
276 /* If the file parameters are the global parameters reset them */
277 if (*ppmsfp == &gMSFileParam)
278 {
279 gMSFileParam.fp = NULL;
280 gMSFileParam.filename[0] = '\0';
281 gMSFileParam.rawrec = NULL;
282 gMSFileParam.readlen = 0;
283 gMSFileParam.readoffset = 0;
284 gMSFileParam.packtype = 0;
285 gMSFileParam.packhdroffset = 0;
286 gMSFileParam.filepos = 0;
287 gMSFileParam.filesize = 0;
288 gMSFileParam.recordcount = 0;
289 }
290 /* Otherwise free the MSFileParam */
291 else
292 {
293 free (*ppmsfp);
294 *ppmsfp = NULL;
295 }
296
297 return MS_NOERROR;
298 }
299
300 /* Allocate reading buffer */
301 if (msfp->rawrec == NULL)
302 {
303 if (!(msfp->rawrec = (char *)malloc (MAXRECLEN)))
304 {
305 ms_log (2, "ms_readmsr_main(): Cannot allocate memory for read buffer\n");
306 return MS_GENERROR;
307 }
308 }
309
310 /* Sanity check: track if we are reading the same file */
311 if (msfp->fp && strncmp (msfile, msfp->filename, sizeof (msfp->filename)))
312 {
313 ms_log (2, "ms_readmsr_main() called with a different file name without being reset\n");
314
315 /* Close previous file and reset needed variables */
316 if (msfp->fp != NULL)
317 fclose (msfp->fp);
318
319 msfp->fp = NULL;
320 msfp->readlen = 0;
321 msfp->readoffset = 0;
322 msfp->packtype = 0;
323 msfp->packhdroffset = 0;
324 msfp->filepos = 0;
325 msfp->filesize = 0;
326 msfp->recordcount = 0;
327 }
328
329 /* Open the file if needed, redirect to stdin if file is "-" */
330 if (msfp->fp == NULL)
331 {
332 /* Store the filename for tracking */
333 strncpy (msfp->filename, msfile, sizeof (msfp->filename) - 1);
334 msfp->filename[sizeof (msfp->filename) - 1] = '\0';
335
336 if (strcmp (msfile, "-") == 0)
337 {
338 msfp->fp = stdin;
339 }
340 else
341 {
342 if ((msfp->fp = fopen (msfile, "rb")) == NULL)
343 {
344 ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
345 msr_free (ppmsr);
346
347 return MS_GENERROR;
348 }
349 else
350 {
351 /* Determine file size */
352 struct stat sbuf;
353
354 if (fstat (fileno (msfp->fp), &sbuf))
355 {
356 ms_log (2, "Cannot open file: %s (%s)\n", msfile, strerror (errno));
357 msr_free (ppmsr);
358
359 return MS_GENERROR;
360 }
361
362 msfp->filesize = sbuf.st_size;
363 }
364 }
365 }
366
367 /* Seek to a specified offset if requested */
368 if (fpos != NULL && *fpos < 0)
369 {
370 /* Only try to seek in real files, not stdin */
371 if (msfp->fp != stdin)
372 {
373 if (lmp_fseeko (msfp->fp, *fpos * -1, SEEK_SET))
374 {
375 ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
376
377 return MS_GENERROR;
378 }
379
380 msfp->filepos = *fpos * -1;
381 msfp->readlen = 0;
382 msfp->readoffset = 0;
383 }
384 }
385
386 /* Zero the last record indicator */
387 if (last)
388 *last = 0;
389
390 /* Read data and search for records */
391 for (;;)
392 {
393 /* Read more data into buffer if not at EOF and buffer has less than MINRECLEN
394 * or more data is needed for the current record detected in buffer. */
395 if (!feof (msfp->fp) && (MSFPBUFLEN (msfp) < MINRECLEN || parseval > 0))
396 {
397 /* Reset offsets if no unprocessed data in buffer */
398 if (MSFPBUFLEN (msfp) <= 0)
399 {
400 msfp->readlen = 0;
401 msfp->readoffset = 0;
402 }
403 /* Otherwise shift existing data to beginning of buffer */
404 else if (msfp->readoffset > 0)
405 {
406 ms_shift_msfp (msfp, msfp->readoffset);
407 }
408
409 /* Determine read size */
410 readsize = (MAXRECLEN - msfp->readlen);
411
412 /* Read data into record buffer */
413 readcount = ms_fread (msfp->rawrec + msfp->readlen, 1, readsize, msfp->fp);
414
415 if (readcount != readsize)
416 {
417 if (!feof (msfp->fp))
418 {
419 ms_log (2, "Short read of %d bytes starting from %" PRId64 "\n",
420 readsize, msfp->filepos);
421 retcode = MS_GENERROR;
422 break;
423 }
424 }
425
426 /* Update read buffer length */
427 msfp->readlen += readcount;
428
429 /* File position corresponding to start of buffer; not strictly necessary */
430 if (msfp->fp != stdin)
431 msfp->filepos = lmp_ftello (msfp->fp) - msfp->readlen;
432 }
433
434 /* Test for packed file signature at the beginning of the file */
435 if (msfp->filepos == 0 && *(MSFPREADPTR (msfp)) == 'P' && MSFPBUFLEN (msfp) >= 48)
436 {
437 msfp->packtype = 0;
438
439 /* Determine pack type, the negative pack type indicates initial header */
440 if (!memcmp ("PED", MSFPREADPTR (msfp), 3))
441 msfp->packtype = -1;
442 else if (!memcmp ("PSD", MSFPREADPTR (msfp), 3))
443 msfp->packtype = -2;
444 else if (!memcmp ("PLC", MSFPREADPTR (msfp), 3))
445 msfp->packtype = -6;
446 else if (!memcmp ("PQI", MSFPREADPTR (msfp), 3))
447 msfp->packtype = -7;
448 else if (!memcmp ("PLS", MSFPREADPTR (msfp), 3))
449 msfp->packtype = -8;
450
451 if (verbose > 0)
452 ms_log (1, "Detected packed file (%3.3s: type %d)\n", MSFPREADPTR (msfp), -msfp->packtype);
453 }
454
455 /* Read pack headers, initial and subsequent headers including (ignored) chksum values */
456 if (msfp->packtype && (msfp->packtype < 0 || msfp->filepos == msfp->packhdroffset) && MSFPBUFLEN (msfp) >= 48)
457 {
458 char hdrstr[30];
459 int64_t datasize;
460
461 /* Determine bytes to skip before header: either initial ID block or type-specific chksum block */
462 packskipsize = (msfp->packtype < 0) ? 10 : packtypes[msfp->packtype][2];
463
464 if (msfp->packtype < 0)
465 msfp->packtype = -msfp->packtype;
466
467 /* Read pack length from pack header accounting for bytes that should be skipped */
468 memset (hdrstr, 0, sizeof (hdrstr));
469 memcpy (hdrstr, MSFPREADPTR (msfp) + (packtypes[msfp->packtype][0] + packskipsize - packtypes[msfp->packtype][1]),
470 packtypes[msfp->packtype][1]);
471 sscanf (hdrstr, " %" SCNd64, &datasize);
472 packdatasize = (off_t)datasize;
473
474 /* Next pack header = File position + skipsize + header size + data size
475 * This offset is actually to the data block chksum which is skipped by the logic above,
476 * the next pack header should directly follow the chksum. */
477 msfp->packhdroffset = msfp->filepos + packskipsize + packtypes[msfp->packtype][0] + packdatasize;
478
479 if (verbose > 1)
480 ms_log (1, "Read packed file header at offset %" PRId64 " (%d bytes follow), chksum offset: %" PRId64 "\n",
481 (msfp->filepos + packskipsize), packdatasize,
482 msfp->packhdroffset);
483
484 /* Shift buffer to new reading offset (aligns records in buffer) */
485 ms_shift_msfp (msfp, msfp->readoffset + (packskipsize + packtypes[msfp->packtype][0]));
486 } /* End of packed header processing */
487
488 /* Check for match if selections are supplied and pack header was read, */
489 /* only when enough data is in buffer and not reading from stdin pipe */
490 if (selections && msfp->packtype && packdatasize && MSFPBUFLEN (msfp) >= 48 && msfp->fp != stdin)
491 {
492 char srcname[100];
493
494 ms_recsrcname (MSFPREADPTR (msfp), srcname, 1);
495
496 if (!ms_matchselect (selections, srcname, HPTERROR, HPTERROR, NULL))
497 {
498 /* Update read position if next section is in buffer */
499 if (MSFPBUFLEN (msfp) >= (msfp->packhdroffset - msfp->filepos))
500 {
501 if (verbose > 1)
502 {
503 ms_log (1, "Skipping (jump) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
504 srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
505 }
506
507 msfp->readoffset += (msfp->packhdroffset - msfp->filepos);
508 msfp->filepos = msfp->packhdroffset;
509 packdatasize = 0;
510 }
511
512 /* Otherwise seek to next pack header and reset reading position */
513 else
514 {
515 if (verbose > 1)
516 {
517 ms_log (1, "Skipping (seek) packed section for %s (%d bytes) starting at offset %" PRId64 "\n",
518 srcname, (msfp->packhdroffset - msfp->filepos), msfp->filepos);
519 }
520
521 if (lmp_fseeko (msfp->fp, msfp->packhdroffset, SEEK_SET))
522 {
523 ms_log (2, "Cannot seek in file: %s (%s)\n", msfile, strerror (errno));
524
525 return MS_GENERROR;
526 break;
527 }
528
529 msfp->filepos = msfp->packhdroffset;
530 msfp->readlen = 0;
531 msfp->readoffset = 0;
532 packdatasize = 0;
533 }
534
535 /* Return to top of loop for proper pack header handling */
536 continue;
537 }
538 } /* End of selection processing */
539
540 /* Attempt to parse record from buffer */
541 if (MSFPBUFLEN (msfp) >= MINRECLEN)
542 {
543 int parselen = MSFPBUFLEN (msfp);
544
545 /* Limit the parse length to offset of pack header if present in the buffer */
546 if (msfp->packhdroffset && msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
547 parselen = msfp->packhdroffset - msfp->filepos;
548
549 parseval = msr_parse (MSFPREADPTR (msfp), parselen, ppmsr, reclen, dataflag, verbose);
550
551 /* Record detected and parsed */
552 if (parseval == 0)
553 {
554 if (verbose > 1)
555 ms_log (1, "Read record length of %d bytes\n", (*ppmsr)->reclen);
556
557 /* Test if this is the last record if file size is known (not pipe) */
558 if (last && msfp->filesize)
559 if ((msfp->filesize - (msfp->filepos + (*ppmsr)->reclen)) < MINRECLEN)
560 *last = 1;
561
562 /* Return file position for this record */
563 if (fpos)
564 *fpos = msfp->filepos;
565
566 /* Update reading offset, file position and record count */
567 msfp->readoffset += (*ppmsr)->reclen;
568 msfp->filepos += (*ppmsr)->reclen;
569 msfp->recordcount++;
570
571 retcode = MS_NOERROR;
572 break;
573 }
574 else if (parseval < 0)
575 {
576 /* Skip non-data if requested */
577 if (skipnotdata)
578 {
579 if (verbose > 1)
580 {
581 if (MS_ISVALIDBLANK ((char *)MSFPREADPTR (msfp)))
582 ms_log (1, "Skipped %d bytes of blank/noise record at byte offset %" PRId64 "\n",
583 MINRECLEN, msfp->filepos);
584 else
585 ms_log (1, "Skipped %d bytes of non-data record at byte offset %" PRId64 "\n",
586 MINRECLEN, msfp->filepos);
587 }
588
589 /* Skip MINRECLEN bytes, update reading offset and file position */
590 msfp->readoffset += MINRECLEN;
591 msfp->filepos += MINRECLEN;
592 }
593 /* Parsing errors */
594 else
595 {
596 ms_log (2, "Cannot detect record at byte offset %" PRId64 ": %s\n",
597 msfp->filepos, msfile);
598
599 /* Print common errors and raw details if verbose */
600 ms_parse_raw (MSFPREADPTR (msfp), MSFPBUFLEN (msfp), verbose, -1);
601
602 retcode = parseval;
603 break;
604 }
605 }
606 else /* parseval > 0 (found record but need more data) */
607 {
608 /* Determine implied record length if needed */
609 int32_t impreclen = reclen;
610
611 /* Check for parse hints that are larger than MAXRECLEN */
612 if ((MSFPBUFLEN (msfp) + parseval) > MAXRECLEN)
613 {
614 if (skipnotdata)
615 {
616 /* Skip MINRECLEN bytes, update reading offset and file position */
617 msfp->readoffset += MINRECLEN;
618 msfp->filepos += MINRECLEN;
619 }
620 else
621 {
622 retcode = MS_OUTOFRANGE;
623 break;
624 }
625 }
626
627 /* Pack header check, if pack header offset is within buffer */
628 else if (impreclen <= 0 && msfp->packhdroffset &&
629 msfp->packhdroffset < (msfp->filepos + MSFPBUFLEN (msfp)))
630 {
631 impreclen = msfp->packhdroffset - msfp->filepos;
632
633 /* Check that record length is within range and a power of 2.
634 * Power of two if (X & (X - 1)) == 0 */
635 if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
636 (impreclen & (impreclen - 1)) == 0)
637 {
638 /* Set the record length implied by the next pack header */
639 reclen = impreclen;
640 }
641 else
642 {
643 ms_log (1, "Implied record length (%d) is invalid\n", impreclen);
644
645 retcode = MS_NOTSEED;
646 break;
647 }
648 }
649
650 /* End of file check */
651 else if (impreclen <= 0 && feof (msfp->fp))
652 {
653 impreclen = msfp->filesize - msfp->filepos;
654
655 /* Check that record length is within range and a power of 2.
656 * Power of two if (X & (X - 1)) == 0 */
657 if (impreclen >= MINRECLEN && impreclen <= MAXRECLEN &&
658 (impreclen & (impreclen - 1)) == 0)
659 {
660 /* Set the record length implied by the end of the file */
661 reclen = impreclen;
662 }
663 /* Otherwise a trucated record */
664 else
665 {
666 if (verbose)
667 {
668 if (msfp->filesize)
669 ms_log (1, "Truncated record at byte offset %" PRId64 ", filesize %d: %s\n",
670 msfp->filepos, msfp->filesize, msfile);
671 else
672 ms_log (1, "Truncated record at byte offset %" PRId64 "\n",
673 msfp->filepos);
674 }
675
676 retcode = MS_ENDOFFILE;
677 break;
678 }
679 }
680 }
681 } /* End of record detection */
682
683 /* Finished when within MINRECLEN from EOF and buffer less than MINRECLEN */
684 if ((msfp->filesize - msfp->filepos) < MINRECLEN && MSFPBUFLEN (msfp) < MINRECLEN)
685 {
686 if (msfp->recordcount == 0 && msfp->packtype == 0)
687 {
688 if (verbose > 0)
689 ms_log (2, "%s: No data records read, not SEED?\n", msfile);
690 retcode = MS_NOTSEED;
691 }
692 else
693 {
694 retcode = MS_ENDOFFILE;
695 }
696
697 break;
698 }
699 } /* End of reading, record detection and parsing loop */
700
701 /* Cleanup target MSRecord if returning an error */
702 if (retcode != MS_NOERROR)
703 {
704 msr_free (ppmsr);
705 }
706
707 return retcode;
708 } /* End of ms_readmsr_main() */
709
710 /*********************************************************************
711 * ms_readtraces:
712 *
713 * This is a simple wrapper for ms_readtraces_selection() that uses no
714 * selections.
715 *
716 * See the comments with ms_readtraces_selection() for return values
717 * and further description of arguments.
718 *********************************************************************/
719 int
ms_readtraces(MSTraceGroup ** ppmstg,const char * msfile,int reclen,double timetol,double sampratetol,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)720 ms_readtraces (MSTraceGroup **ppmstg, const char *msfile, int reclen,
721 double timetol, double sampratetol, flag dataquality,
722 flag skipnotdata, flag dataflag, flag verbose)
723 {
724 return ms_readtraces_selection (ppmstg, msfile, reclen,
725 timetol, sampratetol, NULL,
726 dataquality, skipnotdata,
727 dataflag, verbose);
728 } /* End of ms_readtraces() */
729
730 /*********************************************************************
731 * ms_readtraces_timewin:
732 *
733 * This is a wrapper for ms_readtraces_selection() that creates a
734 * simple selection for a specified time window.
735 *
736 * See the comments with ms_readtraces_selection() for return values
737 * and further description of arguments.
738 *********************************************************************/
739 int
ms_readtraces_timewin(MSTraceGroup ** ppmstg,const char * msfile,int reclen,double timetol,double sampratetol,hptime_t starttime,hptime_t endtime,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)740 ms_readtraces_timewin (MSTraceGroup **ppmstg, const char *msfile, int reclen,
741 double timetol, double sampratetol,
742 hptime_t starttime, hptime_t endtime, flag dataquality,
743 flag skipnotdata, flag dataflag, flag verbose)
744 {
745 Selections selection;
746 SelectTime selecttime;
747
748 selection.srcname[0] = '*';
749 selection.srcname[1] = '\0';
750 selection.timewindows = &selecttime;
751 selection.next = NULL;
752
753 selecttime.starttime = starttime;
754 selecttime.endtime = endtime;
755 selecttime.next = NULL;
756
757 return ms_readtraces_selection (ppmstg, msfile, reclen,
758 timetol, sampratetol, &selection,
759 dataquality, skipnotdata,
760 dataflag, verbose);
761 } /* End of ms_readtraces_timewin() */
762
763 /*********************************************************************
764 * ms_readtraces_selection:
765 *
766 * This routine will open and read all Mini-SEED records in specified
767 * file and populate a trace group. This routine is thread safe.
768 *
769 * If reclen is <= 0 the length of every record is automatically
770 * detected.
771 *
772 * If a Selections list is supplied it will be used to limit which
773 * records are added to the trace group.
774 *
775 * Returns MS_NOERROR and populates an MSTraceGroup struct at *ppmstg
776 * on successful read, otherwise returns a libmseed error code (listed
777 * in libmseed.h).
778 *********************************************************************/
779 int
ms_readtraces_selection(MSTraceGroup ** ppmstg,const char * msfile,int reclen,double timetol,double sampratetol,Selections * selections,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)780 ms_readtraces_selection (MSTraceGroup **ppmstg, const char *msfile,
781 int reclen, double timetol, double sampratetol,
782 Selections *selections, flag dataquality,
783 flag skipnotdata, flag dataflag, flag verbose)
784 {
785 MSRecord *msr = 0;
786 MSFileParam *msfp = 0;
787 int retcode;
788
789 if (!ppmstg)
790 return MS_GENERROR;
791
792 /* Initialize MSTraceGroup if needed */
793 if (!*ppmstg)
794 {
795 *ppmstg = mst_initgroup (*ppmstg);
796
797 if (!*ppmstg)
798 return MS_GENERROR;
799 }
800
801 /* Loop over the input file */
802 while ((retcode = ms_readmsr_main (&msfp, &msr, msfile, reclen, NULL, NULL,
803 skipnotdata, dataflag, NULL, verbose)) == MS_NOERROR)
804 {
805 /* Test against selections if supplied */
806 if (selections)
807 {
808 char srcname[50];
809 hptime_t endtime;
810
811 msr_srcname (msr, srcname, 1);
812 endtime = msr_endtime (msr);
813
814 if (ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL)
815 {
816 continue;
817 }
818 }
819
820 /* Add to trace group */
821 mst_addmsrtogroup (*ppmstg, msr, dataquality, timetol, sampratetol);
822 }
823
824 /* Reset return code to MS_NOERROR on successful read by ms_readmsr() */
825 if (retcode == MS_ENDOFFILE)
826 retcode = MS_NOERROR;
827
828 ms_readmsr_main (&msfp, &msr, NULL, 0, NULL, NULL, 0, 0, NULL, 0);
829
830 return retcode;
831 } /* End of ms_readtraces_selection() */
832
833 /*********************************************************************
834 * ms_readtracelist:
835 *
836 * This is a simple wrapper for ms_readtracelist_selection() that uses
837 * no selections.
838 *
839 * See the comments with ms_readtracelist_selection() for return
840 * values and further description of arguments.
841 *********************************************************************/
842 int
ms_readtracelist(MSTraceList ** ppmstl,const char * msfile,int reclen,double timetol,double sampratetol,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)843 ms_readtracelist (MSTraceList **ppmstl, const char *msfile, int reclen,
844 double timetol, double sampratetol, flag dataquality,
845 flag skipnotdata, flag dataflag, flag verbose)
846 {
847 return ms_readtracelist_selection (ppmstl, msfile, reclen,
848 timetol, sampratetol, NULL,
849 dataquality, skipnotdata,
850 dataflag, verbose);
851 } /* End of ms_readtracelist() */
852
853 /*********************************************************************
854 * ms_readtracelist_timewin:
855 *
856 * This is a wrapper for ms_readtraces_selection() that creates a
857 * simple selection for a specified time window.
858 *
859 * See the comments with ms_readtraces_selection() for return values
860 * and further description of arguments.
861 *********************************************************************/
862 int
ms_readtracelist_timewin(MSTraceList ** ppmstl,const char * msfile,int reclen,double timetol,double sampratetol,hptime_t starttime,hptime_t endtime,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)863 ms_readtracelist_timewin (MSTraceList **ppmstl, const char *msfile,
864 int reclen, double timetol, double sampratetol,
865 hptime_t starttime, hptime_t endtime, flag dataquality,
866 flag skipnotdata, flag dataflag, flag verbose)
867 {
868 Selections selection;
869 SelectTime selecttime;
870
871 selection.srcname[0] = '*';
872 selection.srcname[1] = '\0';
873 selection.timewindows = &selecttime;
874 selection.next = NULL;
875
876 selecttime.starttime = starttime;
877 selecttime.endtime = endtime;
878 selecttime.next = NULL;
879
880 return ms_readtracelist_selection (ppmstl, msfile, reclen,
881 timetol, sampratetol, &selection,
882 dataquality, skipnotdata,
883 dataflag, verbose);
884 } /* End of ms_readtracelist_timewin() */
885
886 /*********************************************************************
887 * ms_readtracelist_selection:
888 *
889 * This routine will open and read all Mini-SEED records in specified
890 * file and populate a trace list. This routine is thread safe.
891 *
892 * If reclen is <= 0 the length of every record is automatically
893 * detected.
894 *
895 * If a Selections list is supplied it will be used to limit which
896 * records are added to the trace list.
897 *
898 * Returns MS_NOERROR and populates an MSTraceList struct at *ppmstl
899 * on successful read, otherwise returns a libmseed error code (listed
900 * in libmseed.h).
901 *********************************************************************/
902 int
ms_readtracelist_selection(MSTraceList ** ppmstl,const char * msfile,int reclen,double timetol,double sampratetol,Selections * selections,flag dataquality,flag skipnotdata,flag dataflag,flag verbose)903 ms_readtracelist_selection (MSTraceList **ppmstl, const char *msfile,
904 int reclen, double timetol, double sampratetol,
905 Selections *selections, flag dataquality,
906 flag skipnotdata, flag dataflag, flag verbose)
907 {
908 MSRecord *msr = 0;
909 MSFileParam *msfp = 0;
910 int retcode;
911
912 if (!ppmstl)
913 return MS_GENERROR;
914
915 /* Initialize MSTraceList if needed */
916 if (!*ppmstl)
917 {
918 *ppmstl = mstl_init (*ppmstl);
919
920 if (!*ppmstl)
921 return MS_GENERROR;
922 }
923
924 /* Loop over the input file */
925 while ((retcode = ms_readmsr_main (&msfp, &msr, msfile, reclen, NULL, NULL,
926 skipnotdata, dataflag, NULL, verbose)) == MS_NOERROR)
927 {
928 /* Test against selections if supplied */
929 if (selections)
930 {
931 char srcname[50];
932 hptime_t endtime;
933
934 msr_srcname (msr, srcname, 1);
935 endtime = msr_endtime (msr);
936
937 if (ms_matchselect (selections, srcname, msr->starttime, endtime, NULL) == NULL)
938 {
939 continue;
940 }
941 }
942
943 /* Add to trace list */
944 mstl_addmsr (*ppmstl, msr, dataquality, 1, timetol, sampratetol);
945 }
946
947 /* Reset return code to MS_NOERROR on successful read by ms_readmsr() */
948 if (retcode == MS_ENDOFFILE)
949 retcode = MS_NOERROR;
950
951 ms_readmsr_main (&msfp, &msr, NULL, 0, NULL, NULL, 0, 0, NULL, 0);
952
953 return retcode;
954 } /* End of ms_readtracelist_selection() */
955
956 /*********************************************************************
957 * ms_fread:
958 *
959 * A wrapper for fread that handles EOF and error conditions.
960 *
961 * Returns the return value from fread.
962 *********************************************************************/
963 static int
ms_fread(char * buf,int size,int num,FILE * stream)964 ms_fread (char *buf, int size, int num, FILE *stream)
965 {
966 int read = 0;
967
968 read = (int)fread (buf, size, num, stream);
969
970 if (read <= 0 && size && num)
971 {
972 if (ferror (stream))
973 ms_log (2, "ms_fread(): Cannot read input file\n");
974
975 else if (!feof (stream))
976 ms_log (2, "ms_fread(): Unknown return from fread()\n");
977 }
978
979 return read;
980 } /* End of ms_fread() */
981
982 /***************************************************************************
983 * ms_record_handler_int:
984 *
985 * Internal record handler. The handler data should be a pointer to
986 * an open file descriptor to which records will be written.
987 *
988 ***************************************************************************/
989 static void
ms_record_handler_int(char * record,int reclen,void * ofp)990 ms_record_handler_int (char *record, int reclen, void *ofp)
991 {
992 if (fwrite (record, reclen, 1, (FILE *)ofp) != 1)
993 {
994 ms_log (2, "Error writing to output file\n");
995 }
996 } /* End of ms_record_handler_int() */
997
998 /***************************************************************************
999 * msr_writemseed:
1000 *
1001 * Pack MSRecord data into Mini-SEED record(s) by calling msr_pack() and
1002 * write to a specified file.
1003 *
1004 * Returns the number of records written on success and -1 on error.
1005 ***************************************************************************/
1006 int
msr_writemseed(MSRecord * msr,const char * msfile,flag overwrite,int reclen,flag encoding,flag byteorder,flag verbose)1007 msr_writemseed (MSRecord *msr, const char *msfile, flag overwrite,
1008 int reclen, flag encoding, flag byteorder, flag verbose)
1009 {
1010 FILE *ofp;
1011 char srcname[50];
1012 char *perms = (overwrite) ? "wb" : "ab";
1013 int packedrecords = 0;
1014
1015 if (!msr || !msfile)
1016 return -1;
1017
1018 /* Open output file or use stdout */
1019 if (strcmp (msfile, "-") == 0)
1020 {
1021 ofp = stdout;
1022 }
1023 else if ((ofp = fopen (msfile, perms)) == NULL)
1024 {
1025 ms_log (1, "Cannot open output file %s: %s\n", msfile, strerror (errno));
1026
1027 return -1;
1028 }
1029
1030 /* Pack the MSRecord */
1031 if (msr->numsamples > 0)
1032 {
1033 msr->encoding = encoding;
1034 msr->reclen = reclen;
1035 msr->byteorder = byteorder;
1036
1037 packedrecords = msr_pack (msr, &ms_record_handler_int, ofp, NULL, 1, verbose - 1);
1038
1039 if (packedrecords < 0)
1040 {
1041 msr_srcname (msr, srcname, 1);
1042 ms_log (1, "Cannot write Mini-SEED for %s\n", srcname);
1043 }
1044 }
1045
1046 /* Close file and return record count */
1047 fclose (ofp);
1048
1049 return (packedrecords >= 0) ? packedrecords : -1;
1050 } /* End of msr_writemseed() */
1051
1052 /***************************************************************************
1053 * mst_writemseed:
1054 *
1055 * Pack MSTrace data into Mini-SEED records by calling mst_pack() and
1056 * write to a specified file.
1057 *
1058 * Returns the number of records written on success and -1 on error.
1059 ***************************************************************************/
1060 int
mst_writemseed(MSTrace * mst,const char * msfile,flag overwrite,int reclen,flag encoding,flag byteorder,flag verbose)1061 mst_writemseed (MSTrace *mst, const char *msfile, flag overwrite,
1062 int reclen, flag encoding, flag byteorder, flag verbose)
1063 {
1064 FILE *ofp;
1065 char srcname[50];
1066 char *perms = (overwrite) ? "wb" : "ab";
1067 int packedrecords = 0;
1068
1069 if (!mst || !msfile)
1070 return -1;
1071
1072 /* Open output file or use stdout */
1073 if (strcmp (msfile, "-") == 0)
1074 {
1075 ofp = stdout;
1076 }
1077 else if ((ofp = fopen (msfile, perms)) == NULL)
1078 {
1079 ms_log (1, "Cannot open output file %s: %s\n", msfile, strerror (errno));
1080
1081 return -1;
1082 }
1083
1084 /* Pack the MSTrace */
1085 if (mst->numsamples > 0)
1086 {
1087 packedrecords = mst_pack (mst, &ms_record_handler_int, ofp, reclen, encoding,
1088 byteorder, NULL, 1, verbose - 1, NULL);
1089
1090 if (packedrecords < 0)
1091 {
1092 mst_srcname (mst, srcname, 1);
1093 ms_log (1, "Cannot write Mini-SEED for %s\n", srcname);
1094 }
1095 }
1096
1097 /* Close file and return record count */
1098 fclose (ofp);
1099
1100 return (packedrecords >= 0) ? packedrecords : -1;
1101 } /* End of mst_writemseed() */
1102
1103 /***************************************************************************
1104 * mst_writemseedgroup:
1105 *
1106 * Pack MSTraceGroup data into Mini-SEED records by calling mst_pack()
1107 * for each MSTrace in the group and write to a specified file.
1108 *
1109 * Returns the number of records written on success and -1 on error.
1110 ***************************************************************************/
1111 int
mst_writemseedgroup(MSTraceGroup * mstg,const char * msfile,flag overwrite,int reclen,flag encoding,flag byteorder,flag verbose)1112 mst_writemseedgroup (MSTraceGroup *mstg, const char *msfile, flag overwrite,
1113 int reclen, flag encoding, flag byteorder, flag verbose)
1114 {
1115 MSTrace *mst;
1116 FILE *ofp;
1117 char srcname[50];
1118 char *perms = (overwrite) ? "wb" : "ab";
1119 int trpackedrecords;
1120 int packedrecords = 0;
1121
1122 if (!mstg || !msfile)
1123 return -1;
1124
1125 /* Open output file or use stdout */
1126 if (strcmp (msfile, "-") == 0)
1127 {
1128 ofp = stdout;
1129 }
1130 else if ((ofp = fopen (msfile, perms)) == NULL)
1131 {
1132 ms_log (1, "Cannot open output file %s: %s\n", msfile, strerror (errno));
1133
1134 return -1;
1135 }
1136
1137 /* Pack each MSTrace in the group */
1138 mst = mstg->traces;
1139 while (mst)
1140 {
1141 if (mst->numsamples <= 0)
1142 {
1143 mst = mst->next;
1144 continue;
1145 }
1146
1147 trpackedrecords = mst_pack (mst, &ms_record_handler_int, ofp, reclen, encoding,
1148 byteorder, NULL, 1, verbose - 1, NULL);
1149
1150 if (trpackedrecords < 0)
1151 {
1152 mst_srcname (mst, srcname, 1);
1153 ms_log (1, "Cannot write Mini-SEED for %s\n", srcname);
1154 }
1155 else
1156 {
1157 packedrecords += trpackedrecords;
1158 }
1159
1160 mst = mst->next;
1161 }
1162
1163 /* Close file and return record count */
1164 fclose (ofp);
1165
1166 return packedrecords;
1167 } /* End of mst_writemseedgroup() */
1168