1 /* This file, drvrmem.c, contains driver routines for memory files. */
2
3 /* The FITSIO software was written by William Pence at the High Energy */
4 /* Astrophysic Science Archive Research Center (HEASARC) at the NASA */
5 /* Goddard Space Flight Center. */
6
7 #include <string.h>
8 #include <stdlib.h>
9 #include <stddef.h> /* apparently needed to define size_t */
10 #include "fitsio2.h"
11
12 #if HAVE_BZIP2
13 #include "bzlib.h"
14 #endif
15
16 /* prototype for .Z file uncompression function in zuncompress.c */
17 int zuncompress2mem(char *filename,
18 FILE *diskfile,
19 char **buffptr,
20 size_t *buffsize,
21 void *(*mem_realloc)(void *p, size_t newsize),
22 size_t *filesize,
23 int *status);
24
25 #if HAVE_BZIP2
26 /* prototype for .bz2 uncompression function (in this file) */
27 void bzip2uncompress2mem(char *filename, FILE *diskfile, int hdl,
28 size_t* filesize, int* status);
29 #endif
30
31
32 #define RECBUFLEN 1000
33
34 static char stdin_outfile[FLEN_FILENAME];
35
36 typedef struct /* structure containing mem file structure */
37 {
38 char **memaddrptr; /* Pointer to memory address pointer; */
39 /* This may or may not point to memaddr. */
40 char *memaddr; /* Pointer to starting memory address; may */
41 /* not always be used, so use *memaddrptr instead */
42 size_t *memsizeptr; /* Pointer to the size of the memory allocation. */
43 /* This may or may not point to memsize. */
44 size_t memsize; /* Size of the memory allocation; this may not */
45 /* always be used, so use *memsizeptr instead. */
46 size_t deltasize; /* Suggested increment for reallocating memory */
47 void *(*mem_realloc)(void *p, size_t newsize); /* realloc function */
48 LONGLONG currentpos; /* current file position, relative to start */
49 LONGLONG fitsfilesize; /* size of the FITS file (always <= *memsizeptr) */
50 FILE *fileptr; /* pointer to compressed output disk file */
51 } memdriver;
52
53 static memdriver memTable[NMAXFILES]; /* allocate mem file handle tables */
54
55 /*--------------------------------------------------------------------------*/
mem_init(void)56 int mem_init(void)
57 {
58 int ii;
59
60 for (ii = 0; ii < NMAXFILES; ii++) /* initialize all empty slots in table */
61 {
62 memTable[ii].memaddrptr = 0;
63 memTable[ii].memaddr = 0;
64 }
65 return(0);
66 }
67 /*--------------------------------------------------------------------------*/
mem_setoptions(int options)68 int mem_setoptions(int options)
69 {
70 /* do something with the options argument, to stop compiler warning */
71 options = 0;
72 return(options);
73 }
74 /*--------------------------------------------------------------------------*/
mem_getoptions(int * options)75 int mem_getoptions(int *options)
76 {
77 *options = 0;
78 return(0);
79 }
80 /*--------------------------------------------------------------------------*/
mem_getversion(int * version)81 int mem_getversion(int *version)
82 {
83 *version = 10;
84 return(0);
85 }
86 /*--------------------------------------------------------------------------*/
mem_shutdown(void)87 int mem_shutdown(void)
88 {
89 return(0);
90 }
91 /*--------------------------------------------------------------------------*/
mem_create(char * filename,int * handle)92 int mem_create(char *filename, int *handle)
93 /*
94 Create a new empty memory file for subsequent writes.
95 The file name is ignored in this case.
96 */
97 {
98 int status;
99
100 /* initially allocate 1 FITS block = 2880 bytes */
101 status = mem_createmem(2880L, handle);
102
103 if (status)
104 {
105 ffpmsg("failed to create empty memory file (mem_create)");
106 return(status);
107 }
108
109 return(0);
110 }
111 /*--------------------------------------------------------------------------*/
mem_create_comp(char * filename,int * handle)112 int mem_create_comp(char *filename, int *handle)
113 /*
114 Create a new empty memory file for subsequent writes.
115 Also create an empty compressed .gz file. The memory file
116 will be compressed and written to the disk file when the file is closed.
117 */
118 {
119 FILE *diskfile;
120 char mode[4];
121 int status;
122
123 /* first, create disk file for the compressed output */
124
125
126 if ( !strcmp(filename, "-.gz") || !strcmp(filename, "stdout.gz") ||
127 !strcmp(filename, "STDOUT.gz") )
128 {
129 /* special case: create uncompressed FITS file in memory, then
130 compress it an write it out to 'stdout' when it is closed. */
131
132 diskfile = stdout;
133 }
134 else
135 {
136 /* normal case: create disk file for the compressed output */
137
138 strcpy(mode, "w+b"); /* create file with read-write */
139
140 diskfile = fopen(filename, "r"); /* does file already exist? */
141
142 if (diskfile)
143 {
144 fclose(diskfile); /* close file and exit with error */
145 return(FILE_NOT_CREATED);
146 }
147
148 #if MACHINE == ALPHAVMS || MACHINE == VAXVMS
149 /* specify VMS record structure: fixed format, 2880 byte records */
150 /* but force stream mode access to enable random I/O access */
151 diskfile = fopen(filename, mode, "rfm=fix", "mrs=2880", "ctx=stm");
152 #else
153 diskfile = fopen(filename, mode);
154 #endif
155
156 if (!(diskfile)) /* couldn't create file */
157 {
158 return(FILE_NOT_CREATED);
159 }
160 }
161
162 /* now create temporary memory file */
163
164 /* initially allocate 1 FITS block = 2880 bytes */
165 status = mem_createmem(2880L, handle);
166
167 if (status)
168 {
169 ffpmsg("failed to create empty memory file (mem_create_comp)");
170 return(status);
171 }
172
173 memTable[*handle].fileptr = diskfile;
174
175 return(0);
176 }
177 /*--------------------------------------------------------------------------*/
mem_openmem(void ** buffptr,size_t * buffsize,size_t deltasize,void * (* memrealloc)(void * p,size_t newsize),int * handle)178 int mem_openmem(void **buffptr, /* I - address of memory pointer */
179 size_t *buffsize, /* I - size of buffer, in bytes */
180 size_t deltasize, /* I - increment for future realloc's */
181 void *(*memrealloc)(void *p, size_t newsize), /* function */
182 int *handle)
183 /*
184 lowest level routine to open a pre-existing memory file.
185 */
186 {
187 int ii;
188
189 *handle = -1;
190 for (ii = 0; ii < NMAXFILES; ii++) /* find empty slot in handle table */
191 {
192 if (memTable[ii].memaddrptr == 0)
193 {
194 *handle = ii;
195 break;
196 }
197 }
198 if (*handle == -1)
199 return(TOO_MANY_FILES); /* too many files opened */
200
201 memTable[ii].memaddrptr = (char **) buffptr; /* pointer to start addres */
202 memTable[ii].memsizeptr = buffsize; /* allocated size of memory */
203 memTable[ii].deltasize = deltasize; /* suggested realloc increment */
204 memTable[ii].fitsfilesize = *buffsize; /* size of FITS file (upper limit) */
205 memTable[ii].currentpos = 0; /* at beginning of the file */
206 memTable[ii].mem_realloc = memrealloc; /* memory realloc function */
207 return(0);
208 }
209 /*--------------------------------------------------------------------------*/
mem_createmem(size_t msize,int * handle)210 int mem_createmem(size_t msize, int *handle)
211 /*
212 lowest level routine to allocate a memory file.
213 */
214 {
215 int ii;
216
217 *handle = -1;
218 for (ii = 0; ii < NMAXFILES; ii++) /* find empty slot in handle table */
219 {
220 if (memTable[ii].memaddrptr == 0)
221 {
222 *handle = ii;
223 break;
224 }
225 }
226 if (*handle == -1)
227 return(TOO_MANY_FILES); /* too many files opened */
228
229 /* use the internally allocated memaddr and memsize variables */
230 memTable[ii].memaddrptr = &memTable[ii].memaddr;
231 memTable[ii].memsizeptr = &memTable[ii].memsize;
232
233 /* allocate initial block of memory for the file */
234 if (msize > 0)
235 {
236 memTable[ii].memaddr = (char *) malloc(msize);
237 if ( !(memTable[ii].memaddr) )
238 {
239 ffpmsg("malloc of initial memory failed (mem_createmem)");
240 return(FILE_NOT_OPENED);
241 }
242 }
243
244 /* set initial state of the file */
245 memTable[ii].memsize = msize;
246 memTable[ii].deltasize = 2880;
247 memTable[ii].fitsfilesize = 0;
248 memTable[ii].currentpos = 0;
249 memTable[ii].mem_realloc = realloc;
250 return(0);
251 }
252 /*--------------------------------------------------------------------------*/
mem_truncate(int handle,LONGLONG filesize)253 int mem_truncate(int handle, LONGLONG filesize)
254 /*
255 truncate the file to a new size
256 */
257 {
258 char *ptr;
259
260 /* call the memory reallocation function, if defined */
261 if ( memTable[handle].mem_realloc )
262 { /* explicit LONGLONG->size_t cast */
263 ptr = (memTable[handle].mem_realloc)(
264 *(memTable[handle].memaddrptr),
265 (size_t) filesize);
266 if (!ptr)
267 {
268 ffpmsg("Failed to reallocate memory (mem_truncate)");
269 return(MEMORY_ALLOCATION);
270 }
271
272 /* if allocated more memory, initialize it to zero */
273 if ( filesize > *(memTable[handle].memsizeptr) )
274 {
275 memset(ptr + *(memTable[handle].memsizeptr),
276 0,
277 ((size_t) filesize) - *(memTable[handle].memsizeptr) );
278 }
279
280 *(memTable[handle].memaddrptr) = ptr;
281 *(memTable[handle].memsizeptr) = (size_t) (filesize);
282 }
283
284 memTable[handle].currentpos = filesize;
285 memTable[handle].fitsfilesize = filesize;
286 return(0);
287 }
288 /*--------------------------------------------------------------------------*/
stdin_checkfile(char * urltype,char * infile,char * outfile)289 int stdin_checkfile(char *urltype, char *infile, char *outfile)
290 /*
291 do any special case checking when opening a file on the stdin stream
292 */
293 {
294 if (strlen(outfile))
295 {
296 stdin_outfile[0] = '\0';
297 strncat(stdin_outfile,outfile,FLEN_FILENAME-1); /* an output file is specified */
298 strcpy(urltype,"stdinfile://");
299 }
300 else
301 *stdin_outfile = '\0'; /* no output file was specified */
302
303 return(0);
304 }
305 /*--------------------------------------------------------------------------*/
stdin_open(char * filename,int rwmode,int * handle)306 int stdin_open(char *filename, int rwmode, int *handle)
307 /*
308 open a FITS file from the stdin file stream by copying it into memory
309 The file name is ignored in this case.
310 */
311 {
312 int status;
313 char cbuff;
314
315 if (*stdin_outfile)
316 {
317 /* copy the stdin stream to the specified disk file then open the file */
318
319 /* Create the output file */
320 status = file_create(stdin_outfile,handle);
321
322 if (status)
323 {
324 ffpmsg("Unable to create output file to copy stdin (stdin_open):");
325 ffpmsg(stdin_outfile);
326 return(status);
327 }
328
329 /* copy the whole stdin stream to the file */
330 status = stdin2file(*handle);
331 file_close(*handle);
332
333 if (status)
334 {
335 ffpmsg("failed to copy stdin to file (stdin_open)");
336 ffpmsg(stdin_outfile);
337 return(status);
338 }
339
340 /* reopen file with proper rwmode attribute */
341 status = file_open(stdin_outfile, rwmode, handle);
342 }
343 else
344 {
345
346 /* get the first character, then put it back */
347 cbuff = fgetc(stdin);
348 ungetc(cbuff, stdin);
349
350 /* compressed files begin with 037 or 'P' */
351 if (cbuff == 31 || cbuff == 75)
352 {
353 /* looks like the input stream is compressed */
354 status = mem_compress_stdin_open(filename, rwmode, handle);
355
356 }
357 else
358 {
359 /* copy the stdin stream into memory then open file in memory */
360
361 if (rwmode != READONLY)
362 {
363 ffpmsg("cannot open stdin with WRITE access");
364 return(READONLY_FILE);
365 }
366
367 status = mem_createmem(2880L, handle);
368
369 if (status)
370 {
371 ffpmsg("failed to create empty memory file (stdin_open)");
372 return(status);
373 }
374
375 /* copy the whole stdin stream into memory */
376 status = stdin2mem(*handle);
377
378 if (status)
379 {
380 ffpmsg("failed to copy stdin into memory (stdin_open)");
381 free(memTable[*handle].memaddr);
382 }
383 }
384 }
385
386 return(status);
387 }
388 /*--------------------------------------------------------------------------*/
stdin2mem(int hd)389 int stdin2mem(int hd) /* handle number */
390 /*
391 Copy the stdin stream into memory. Fill whatever amount of memory
392 has already been allocated, then realloc more memory if necessary.
393 */
394 {
395 size_t nread, memsize, delta;
396 LONGLONG filesize;
397 char *memptr;
398 char simple[] = "SIMPLE";
399 int c, ii, jj;
400
401 memptr = *memTable[hd].memaddrptr;
402 memsize = *memTable[hd].memsizeptr;
403 delta = memTable[hd].deltasize;
404
405 filesize = 0;
406 ii = 0;
407
408 for(jj = 0; (c = fgetc(stdin)) != EOF && jj < 2000; jj++)
409 {
410 /* Skip over any garbage at the beginning of the stdin stream by */
411 /* reading 1 char at a time, looking for 'S', 'I', 'M', 'P', 'L', 'E' */
412 /* Give up if not found in the first 2000 characters */
413
414 if (c == simple[ii])
415 {
416 ii++;
417 if (ii == 6) /* found the complete string? */
418 {
419 memcpy(memptr, simple, 6); /* copy "SIMPLE" to buffer */
420 filesize = 6;
421 break;
422 }
423 }
424 else
425 ii = 0; /* reset search to beginning of the string */
426 }
427
428 if (filesize == 0)
429 {
430 ffpmsg("Couldn't find the string 'SIMPLE' in the stdin stream.");
431 ffpmsg("This does not look like a FITS file.");
432 return(FILE_NOT_OPENED);
433 }
434
435 /* fill up the remainder of the initial memory allocation */
436 nread = fread(memptr + 6, 1, memsize - 6, stdin);
437 nread += 6; /* add in the 6 characters in 'SIMPLE' */
438
439 if (nread < memsize) /* reached the end? */
440 {
441 memTable[hd].fitsfilesize = nread;
442 return(0);
443 }
444
445 filesize = nread;
446
447 while (1)
448 {
449 /* allocate memory for another FITS block */
450 memptr = realloc(memptr, memsize + delta);
451
452 if (!memptr)
453 {
454 ffpmsg("realloc failed while copying stdin (stdin2mem)");
455 return(MEMORY_ALLOCATION);
456 }
457 memsize += delta;
458
459 /* read another FITS block */
460 nread = fread(memptr + filesize, 1, delta, stdin);
461
462 filesize += nread;
463
464 if (nread < delta) /* reached the end? */
465 break;
466 }
467
468 memTable[hd].fitsfilesize = filesize;
469 *memTable[hd].memaddrptr = memptr;
470 *memTable[hd].memsizeptr = memsize;
471
472 return(0);
473 }
474 /*--------------------------------------------------------------------------*/
stdin2file(int handle)475 int stdin2file(int handle) /* handle number */
476 /*
477 Copy the stdin stream to a file. .
478 */
479 {
480 size_t nread;
481 char simple[] = "SIMPLE";
482 int c, ii, jj, status;
483 char recbuf[RECBUFLEN];
484
485 ii = 0;
486 for(jj = 0; (c = fgetc(stdin)) != EOF && jj < 2000; jj++)
487 {
488 /* Skip over any garbage at the beginning of the stdin stream by */
489 /* reading 1 char at a time, looking for 'S', 'I', 'M', 'P', 'L', 'E' */
490 /* Give up if not found in the first 2000 characters */
491
492 if (c == simple[ii])
493 {
494 ii++;
495 if (ii == 6) /* found the complete string? */
496 {
497 memcpy(recbuf, simple, 6); /* copy "SIMPLE" to buffer */
498 break;
499 }
500 }
501 else
502 ii = 0; /* reset search to beginning of the string */
503 }
504
505 if (ii != 6)
506 {
507 ffpmsg("Couldn't find the string 'SIMPLE' in the stdin stream");
508 return(FILE_NOT_OPENED);
509 }
510
511 /* fill up the remainder of the buffer */
512 nread = fread(recbuf + 6, 1, RECBUFLEN - 6, stdin);
513 nread += 6; /* add in the 6 characters in 'SIMPLE' */
514
515 status = file_write(handle, recbuf, nread);
516 if (status)
517 return(status);
518
519 /* copy the rest of stdin stream */
520 while(0 != (nread = fread(recbuf,1,RECBUFLEN, stdin)))
521 {
522 status = file_write(handle, recbuf, nread);
523 if (status)
524 return(status);
525 }
526
527 return(status);
528 }
529 /*--------------------------------------------------------------------------*/
stdout_close(int handle)530 int stdout_close(int handle)
531 /*
532 copy the memory file to stdout, then free the memory
533 */
534 {
535 int status = 0;
536
537 /* copy from memory to standard out. explicit LONGLONG->size_t cast */
538 if(fwrite(memTable[handle].memaddr, 1,
539 ((size_t) memTable[handle].fitsfilesize), stdout) !=
540 (size_t) memTable[handle].fitsfilesize )
541 {
542 ffpmsg("failed to copy memory file to stdout (stdout_close)");
543 status = WRITE_ERROR;
544 }
545
546 free( memTable[handle].memaddr ); /* free the memory */
547 memTable[handle].memaddrptr = 0;
548 memTable[handle].memaddr = 0;
549 return(status);
550 }
551 /*--------------------------------------------------------------------------*/
mem_compress_openrw(char * filename,int rwmode,int * hdl)552 int mem_compress_openrw(char *filename, int rwmode, int *hdl)
553 /*
554 This routine opens the compressed diskfile and creates an empty memory
555 buffer with an appropriate size, then calls mem_uncompress2mem. It allows
556 the memory 'file' to be opened with READWRITE access.
557 */
558 {
559 return(mem_compress_open(filename, READONLY, hdl));
560 }
561 /*--------------------------------------------------------------------------*/
mem_compress_open(char * filename,int rwmode,int * hdl)562 int mem_compress_open(char *filename, int rwmode, int *hdl)
563 /*
564 This routine opens the compressed diskfile and creates an empty memory
565 buffer with an appropriate size, then calls mem_uncompress2mem.
566 */
567 {
568 FILE *diskfile;
569 int status, estimated = 1;
570 unsigned char buffer[4];
571 size_t finalsize, filesize;
572 LONGLONG llsize = 0;
573 unsigned int modulosize;
574 char *ptr;
575
576 if (rwmode != READONLY)
577 {
578 ffpmsg(
579 "cannot open compressed file with WRITE access (mem_compress_open)");
580 ffpmsg(filename);
581 return(READONLY_FILE);
582 }
583
584 /* open the compressed disk file */
585 status = file_openfile(filename, READONLY, &diskfile);
586 if (status)
587 {
588 ffpmsg("failed to open compressed disk file (compress_open)");
589 ffpmsg(filename);
590 return(status);
591 }
592
593 if (fread(buffer, 1, 2, diskfile) != 2) /* read 2 bytes */
594 {
595 fclose(diskfile);
596 return(READ_ERROR);
597 }
598
599 if (memcmp(buffer, "\037\213", 2) == 0) /* GZIP */
600 {
601 /* the uncompressed file size is give at the end */
602 /* of the file in the ISIZE field (modulo 2^32) */
603
604 fseek(diskfile, 0, 2); /* move to end of file */
605 filesize = ftell(diskfile); /* position = size of file */
606 fseek(diskfile, -4L, 1); /* move back 4 bytes */
607 fread(buffer, 1, 4L, diskfile); /* read 4 bytes */
608
609 /* have to worry about integer byte order */
610 modulosize = buffer[0];
611 modulosize |= buffer[1] << 8;
612 modulosize |= buffer[2] << 16;
613 modulosize |= buffer[3] << 24;
614
615 /*
616 the field ISIZE in the gzipped file header only stores 4 bytes and contains
617 the uncompressed file size modulo 2^32. If the uncompressed file size
618 is less than the compressed file size (filesize), then one probably needs to
619 add 2^32 = 4294967296 to the uncompressed file size, assuming that the gzip
620 produces a compressed file that is smaller than the original file.
621
622 But one must allow for the case of very small files, where the
623 gzipped file may actually be larger then the original uncompressed file.
624 Therefore, only perform the modulo 2^32 correction test if the compressed
625 file is greater than 10,000 bytes in size. (Note: this threhold would
626 fail only if the original file was greater than 2^32 bytes in size AND gzip
627 was able to compress it by more than a factor of 400,000 (!) which seems
628 highly unlikely.)
629
630 Also, obviously, this 2^32 modulo correction cannot be performed if the
631 finalsize variable is only 32-bits long. Typically, the 'size_t' integer
632 type must be 8 bytes or larger in size to support data files that are
633 greater than 2 GB (2^31 bytes) in size.
634 */
635 finalsize = modulosize;
636
637 if (sizeof(size_t) > 4 && filesize > 10000) {
638 llsize = (LONGLONG) finalsize;
639 /* use LONGLONG variable to suppress compiler warning */
640 while (llsize < (LONGLONG) filesize) llsize += 4294967296;
641
642 finalsize = (size_t) llsize;
643 }
644
645 estimated = 0; /* file size is known, not estimated */
646 }
647 else if (memcmp(buffer, "\120\113", 2) == 0) /* PKZIP */
648 {
649 /* the uncompressed file size is give at byte 22 the file */
650
651 fseek(diskfile, 22L, 0); /* move to byte 22 */
652 fread(buffer, 1, 4L, diskfile); /* read 4 bytes */
653
654 /* have to worry about integer byte order */
655 modulosize = buffer[0];
656 modulosize |= buffer[1] << 8;
657 modulosize |= buffer[2] << 16;
658 modulosize |= buffer[3] << 24;
659 finalsize = modulosize;
660
661 estimated = 0; /* file size is known, not estimated */
662 }
663 else if (memcmp(buffer, "\037\036", 2) == 0) /* PACK */
664 finalsize = 0; /* for most methods we can't determine final size */
665 else if (memcmp(buffer, "\037\235", 2) == 0) /* LZW */
666 finalsize = 0; /* for most methods we can't determine final size */
667 else if (memcmp(buffer, "\037\240", 2) == 0) /* LZH */
668 finalsize = 0; /* for most methods we can't determine final size */
669 #if HAVE_BZIP2
670 else if (memcmp(buffer, "BZ", 2) == 0) /* BZip2 */
671 finalsize = 0; /* for most methods we can't determine final size */
672 #endif
673 else
674 {
675 /* not a compressed file; this should never happen */
676 fclose(diskfile);
677 return(1);
678 }
679
680 if (finalsize == 0) /* estimate uncompressed file size */
681 {
682 fseek(diskfile, 0, 2); /* move to end of the compressed file */
683 finalsize = ftell(diskfile); /* position = size of file */
684 finalsize = finalsize * 3; /* assume factor of 3 compression */
685 }
686
687 fseek(diskfile, 0, 0); /* move back to beginning of file */
688
689 /* create a memory file big enough (hopefully) for the uncompressed file */
690 status = mem_createmem(finalsize, hdl);
691
692 if (status && estimated)
693 {
694 /* memory allocation failed, so try a smaller estimated size */
695 finalsize = finalsize / 3;
696 status = mem_createmem(finalsize, hdl);
697 }
698
699 if (status)
700 {
701 fclose(diskfile);
702 ffpmsg("failed to create empty memory file (compress_open)");
703 return(status);
704 }
705
706 /* uncompress file into memory */
707 status = mem_uncompress2mem(filename, diskfile, *hdl);
708
709 fclose(diskfile);
710
711 if (status)
712 {
713 mem_close_free(*hdl); /* free up the memory */
714 ffpmsg("failed to uncompress file into memory (compress_open)");
715 return(status);
716 }
717
718 /* if we allocated too much memory initially, then free it */
719 if (*(memTable[*hdl].memsizeptr) >
720 (( (size_t) memTable[*hdl].fitsfilesize) + 256L) )
721 {
722 ptr = realloc(*(memTable[*hdl].memaddrptr),
723 ((size_t) memTable[*hdl].fitsfilesize) );
724 if (!ptr)
725 {
726 ffpmsg("Failed to reduce size of allocated memory (compress_open)");
727 return(MEMORY_ALLOCATION);
728 }
729
730 *(memTable[*hdl].memaddrptr) = ptr;
731 *(memTable[*hdl].memsizeptr) = (size_t) (memTable[*hdl].fitsfilesize);
732 }
733
734 return(0);
735 }
736 /*--------------------------------------------------------------------------*/
mem_compress_stdin_open(char * filename,int rwmode,int * hdl)737 int mem_compress_stdin_open(char *filename, int rwmode, int *hdl)
738 /*
739 This routine reads the compressed input stream and creates an empty memory
740 buffer, then calls mem_uncompress2mem.
741 */
742 {
743 int status;
744 char *ptr;
745
746 if (rwmode != READONLY)
747 {
748 ffpmsg(
749 "cannot open compressed input stream with WRITE access (mem_compress_stdin_open)");
750 return(READONLY_FILE);
751 }
752
753 /* create a memory file for the uncompressed file */
754 status = mem_createmem(28800, hdl);
755
756 if (status)
757 {
758 ffpmsg("failed to create empty memory file (compress_stdin_open)");
759 return(status);
760 }
761
762 /* uncompress file into memory */
763 status = mem_uncompress2mem(filename, stdin, *hdl);
764
765 if (status)
766 {
767 mem_close_free(*hdl); /* free up the memory */
768 ffpmsg("failed to uncompress stdin into memory (compress_stdin_open)");
769 return(status);
770 }
771
772 /* if we allocated too much memory initially, then free it */
773 if (*(memTable[*hdl].memsizeptr) >
774 (( (size_t) memTable[*hdl].fitsfilesize) + 256L) )
775 {
776 ptr = realloc(*(memTable[*hdl].memaddrptr),
777 ((size_t) memTable[*hdl].fitsfilesize) );
778 if (!ptr)
779 {
780 ffpmsg("Failed to reduce size of allocated memory (compress_stdin_open)");
781 return(MEMORY_ALLOCATION);
782 }
783
784 *(memTable[*hdl].memaddrptr) = ptr;
785 *(memTable[*hdl].memsizeptr) = (size_t) (memTable[*hdl].fitsfilesize);
786 }
787
788 return(0);
789 }
790 /*--------------------------------------------------------------------------*/
mem_iraf_open(char * filename,int rwmode,int * hdl)791 int mem_iraf_open(char *filename, int rwmode, int *hdl)
792 /*
793 This routine creates an empty memory buffer, then calls iraf2mem to
794 open the IRAF disk file and convert it to a FITS file in memeory.
795 */
796 {
797 int status;
798 size_t filesize = 0;
799
800 /* create a memory file with size = 0 for the FITS converted IRAF file */
801 status = mem_createmem(filesize, hdl);
802 if (status)
803 {
804 ffpmsg("failed to create empty memory file (mem_iraf_open)");
805 return(status);
806 }
807
808 /* convert the iraf file into a FITS file in memory */
809 status = iraf2mem(filename, memTable[*hdl].memaddrptr,
810 memTable[*hdl].memsizeptr, &filesize, &status);
811
812 if (status)
813 {
814 mem_close_free(*hdl); /* free up the memory */
815 ffpmsg("failed to convert IRAF file into memory (mem_iraf_open)");
816 return(status);
817 }
818
819 memTable[*hdl].currentpos = 0; /* save starting position */
820 memTable[*hdl].fitsfilesize=filesize; /* and initial file size */
821
822 return(0);
823 }
824 /*--------------------------------------------------------------------------*/
mem_rawfile_open(char * filename,int rwmode,int * hdl)825 int mem_rawfile_open(char *filename, int rwmode, int *hdl)
826 /*
827 This routine creates an empty memory buffer, writes a minimal
828 image header, then copies the image data from the raw file into
829 memory. It will byteswap the pixel values if the raw array
830 is in little endian byte order.
831 */
832 {
833 FILE *diskfile;
834 fitsfile *fptr;
835 short *sptr;
836 int status, endian, datatype, bytePerPix, naxis;
837 long dim[5] = {1,1,1,1,1}, ii, nvals, offset = 0;
838 size_t filesize = 0, datasize;
839 char rootfile[FLEN_FILENAME], *cptr = 0, *cptr2 = 0;
840 void *ptr;
841
842 if (rwmode != READONLY)
843 {
844 ffpmsg(
845 "cannot open raw binary file with WRITE access (mem_rawfile_open)");
846 ffpmsg(filename);
847 return(READONLY_FILE);
848 }
849
850 cptr = strchr(filename, '['); /* search for opening bracket [ */
851
852 if (!cptr)
853 {
854 ffpmsg("binary file name missing '[' character (mem_rawfile_open)");
855 ffpmsg(filename);
856 return(URL_PARSE_ERROR);
857 }
858
859 *rootfile = '\0';
860 strncat(rootfile, filename, cptr - filename); /* store the rootname */
861
862 cptr++;
863
864 while (*cptr == ' ')
865 cptr++; /* skip leading blanks */
866
867 /* Get the Data Type of the Image */
868
869 if (*cptr == 'b' || *cptr == 'B')
870 {
871 datatype = BYTE_IMG;
872 bytePerPix = 1;
873 }
874 else if (*cptr == 'i' || *cptr == 'I')
875 {
876 datatype = SHORT_IMG;
877 bytePerPix = 2;
878 }
879 else if (*cptr == 'u' || *cptr == 'U')
880 {
881 datatype = USHORT_IMG;
882 bytePerPix = 2;
883
884 }
885 else if (*cptr == 'j' || *cptr == 'J')
886 {
887 datatype = LONG_IMG;
888 bytePerPix = 4;
889 }
890 else if (*cptr == 'r' || *cptr == 'R' || *cptr == 'f' || *cptr == 'F')
891 {
892 datatype = FLOAT_IMG;
893 bytePerPix = 4;
894 }
895 else if (*cptr == 'd' || *cptr == 'D')
896 {
897 datatype = DOUBLE_IMG;
898 bytePerPix = 8;
899 }
900 else
901 {
902 ffpmsg("error in raw binary file datatype (mem_rawfile_open)");
903 ffpmsg(filename);
904 return(URL_PARSE_ERROR);
905 }
906
907 cptr++;
908
909 /* get Endian: Big or Little; default is same as the local machine */
910
911 if (*cptr == 'b' || *cptr == 'B')
912 {
913 endian = 0;
914 cptr++;
915 }
916 else if (*cptr == 'l' || *cptr == 'L')
917 {
918 endian = 1;
919 cptr++;
920 }
921 else
922 endian = BYTESWAPPED; /* byteswapped machines are little endian */
923
924 /* read each dimension (up to 5) */
925
926 naxis = 1;
927 dim[0] = strtol(cptr, &cptr2, 10);
928
929 if (cptr2 && *cptr2 == ',')
930 {
931 naxis = 2;
932 dim[1] = strtol(cptr2+1, &cptr, 10);
933
934 if (cptr && *cptr == ',')
935 {
936 naxis = 3;
937 dim[2] = strtol(cptr+1, &cptr2, 10);
938
939 if (cptr2 && *cptr2 == ',')
940 {
941 naxis = 4;
942 dim[3] = strtol(cptr2+1, &cptr, 10);
943
944 if (cptr && *cptr == ',')
945 naxis = 5;
946 dim[4] = strtol(cptr+1, &cptr2, 10);
947 }
948 }
949 }
950
951 cptr = maxvalue(cptr, cptr2);
952
953 if (*cptr == ':') /* read starting offset value */
954 offset = strtol(cptr+1, 0, 10);
955
956 nvals = dim[0] * dim[1] * dim[2] * dim[3] * dim[4];
957 datasize = nvals * bytePerPix;
958 filesize = nvals * bytePerPix + 2880;
959 filesize = ((filesize - 1) / 2880 + 1) * 2880;
960
961 /* open the raw binary disk file */
962 status = file_openfile(rootfile, READONLY, &diskfile);
963 if (status)
964 {
965 ffpmsg("failed to open raw binary file (mem_rawfile_open)");
966 ffpmsg(rootfile);
967 return(status);
968 }
969
970 /* create a memory file with corrct size for the FITS converted raw file */
971 status = mem_createmem(filesize, hdl);
972 if (status)
973 {
974 ffpmsg("failed to create memory file (mem_rawfile_open)");
975 fclose(diskfile);
976 return(status);
977 }
978
979 /* open this piece of memory as a new FITS file */
980 ffimem(&fptr, (void **) memTable[*hdl].memaddrptr, &filesize, 0, 0, &status);
981
982 /* write the required header keywords */
983 ffcrim(fptr, datatype, naxis, dim, &status);
984
985 /* close the FITS file, but keep the memory allocated */
986 ffclos(fptr, &status);
987
988 if (status > 0)
989 {
990 ffpmsg("failed to write basic image header (mem_rawfile_open)");
991 fclose(diskfile);
992 mem_close_free(*hdl); /* free up the memory */
993 return(status);
994 }
995
996 if (offset > 0)
997 fseek(diskfile, offset, 0); /* offset to start of the data */
998
999 /* read the raw data into memory */
1000 ptr = *memTable[*hdl].memaddrptr + 2880;
1001
1002 if (fread((char *) ptr, 1, datasize, diskfile) != datasize)
1003 status = READ_ERROR;
1004
1005 fclose(diskfile); /* close the raw binary disk file */
1006
1007 if (status)
1008 {
1009 mem_close_free(*hdl); /* free up the memory */
1010 ffpmsg("failed to copy raw file data into memory (mem_rawfile_open)");
1011 return(status);
1012 }
1013
1014 if (datatype == USHORT_IMG) /* have to subtract 32768 from each unsigned */
1015 { /* value to conform to FITS convention. More */
1016 /* efficient way to do this is to just flip */
1017 /* the most significant bit. */
1018
1019 sptr = (short *) ptr;
1020
1021 if (endian == BYTESWAPPED) /* working with native format */
1022 {
1023 for (ii = 0; ii < nvals; ii++, sptr++)
1024 {
1025 *sptr = ( *sptr ) ^ 0x8000;
1026 }
1027 }
1028 else /* pixels are byteswapped WRT the native format */
1029 {
1030 for (ii = 0; ii < nvals; ii++, sptr++)
1031 {
1032 *sptr = ( *sptr ) ^ 0x80;
1033 }
1034 }
1035 }
1036
1037 if (endian) /* swap the bytes if array is in little endian byte order */
1038 {
1039 if (datatype == SHORT_IMG || datatype == USHORT_IMG)
1040 {
1041 ffswap2( (short *) ptr, nvals);
1042 }
1043 else if (datatype == LONG_IMG || datatype == FLOAT_IMG)
1044 {
1045 ffswap4( (INT32BIT *) ptr, nvals);
1046 }
1047
1048 else if (datatype == DOUBLE_IMG)
1049 {
1050 ffswap8( (double *) ptr, nvals);
1051 }
1052 }
1053
1054 memTable[*hdl].currentpos = 0; /* save starting position */
1055 memTable[*hdl].fitsfilesize=filesize; /* and initial file size */
1056
1057 return(0);
1058 }
1059 /*--------------------------------------------------------------------------*/
mem_uncompress2mem(char * filename,FILE * diskfile,int hdl)1060 int mem_uncompress2mem(char *filename, FILE *diskfile, int hdl)
1061 {
1062 /*
1063 lower level routine to uncompress a file into memory. The file
1064 has already been opened and the memory buffer has been allocated.
1065 */
1066
1067 size_t finalsize;
1068 int status;
1069 /* uncompress file into memory */
1070 status = 0;
1071
1072 if (strstr(filename, ".Z")) {
1073 zuncompress2mem(filename, diskfile,
1074 memTable[hdl].memaddrptr, /* pointer to memory address */
1075 memTable[hdl].memsizeptr, /* pointer to size of memory */
1076 realloc, /* reallocation function */
1077 &finalsize, &status); /* returned file size nd status*/
1078 #if HAVE_BZIP2
1079 } else if (strstr(filename, ".bz2")) {
1080 bzip2uncompress2mem(filename, diskfile, hdl, &finalsize, &status);
1081 #endif
1082 } else {
1083 uncompress2mem(filename, diskfile,
1084 memTable[hdl].memaddrptr, /* pointer to memory address */
1085 memTable[hdl].memsizeptr, /* pointer to size of memory */
1086 realloc, /* reallocation function */
1087 &finalsize, &status); /* returned file size nd status*/
1088 }
1089
1090 memTable[hdl].currentpos = 0; /* save starting position */
1091 memTable[hdl].fitsfilesize=finalsize; /* and initial file size */
1092 return status;
1093 }
1094 /*--------------------------------------------------------------------------*/
mem_size(int handle,LONGLONG * filesize)1095 int mem_size(int handle, LONGLONG *filesize)
1096 /*
1097 return the size of the file; only called when the file is first opened
1098 */
1099 {
1100 *filesize = memTable[handle].fitsfilesize;
1101 return(0);
1102 }
1103 /*--------------------------------------------------------------------------*/
mem_close_free(int handle)1104 int mem_close_free(int handle)
1105 /*
1106 close the file and free the memory.
1107 */
1108 {
1109 free( *(memTable[handle].memaddrptr) );
1110
1111 memTable[handle].memaddrptr = 0;
1112 memTable[handle].memaddr = 0;
1113 return(0);
1114 }
1115 /*--------------------------------------------------------------------------*/
mem_close_keep(int handle)1116 int mem_close_keep(int handle)
1117 /*
1118 close the memory file but do not free the memory.
1119 */
1120 {
1121 memTable[handle].memaddrptr = 0;
1122 memTable[handle].memaddr = 0;
1123 return(0);
1124 }
1125 /*--------------------------------------------------------------------------*/
mem_close_comp(int handle)1126 int mem_close_comp(int handle)
1127 /*
1128 compress the memory file, writing it out to the fileptr (which might
1129 be stdout)
1130 */
1131 {
1132 int status = 0;
1133 size_t compsize;
1134
1135 /* compress file in memory to a .gz disk file */
1136
1137 if(compress2file_from_mem(memTable[handle].memaddr,
1138 (size_t) (memTable[handle].fitsfilesize),
1139 memTable[handle].fileptr,
1140 &compsize, &status ) )
1141 {
1142 ffpmsg("failed to copy memory file to file (mem_close_comp)");
1143 status = WRITE_ERROR;
1144 }
1145
1146 free( memTable[handle].memaddr ); /* free the memory */
1147 memTable[handle].memaddrptr = 0;
1148 memTable[handle].memaddr = 0;
1149
1150 /* close the compressed disk file (except if it is 'stdout' */
1151 if (memTable[handle].fileptr != stdout)
1152 fclose(memTable[handle].fileptr);
1153
1154 return(status);
1155 }
1156 /*--------------------------------------------------------------------------*/
mem_seek(int handle,LONGLONG offset)1157 int mem_seek(int handle, LONGLONG offset)
1158 /*
1159 seek to position relative to start of the file.
1160 */
1161 {
1162 if (offset > memTable[handle].fitsfilesize )
1163 return(END_OF_FILE);
1164
1165 memTable[handle].currentpos = offset;
1166 return(0);
1167 }
1168 /*--------------------------------------------------------------------------*/
mem_read(int hdl,void * buffer,long nbytes)1169 int mem_read(int hdl, void *buffer, long nbytes)
1170 /*
1171 read bytes from the current position in the file
1172 */
1173 {
1174 if (memTable[hdl].currentpos + nbytes > memTable[hdl].fitsfilesize)
1175 return(END_OF_FILE);
1176
1177 memcpy(buffer,
1178 *(memTable[hdl].memaddrptr) + memTable[hdl].currentpos,
1179 nbytes);
1180
1181 memTable[hdl].currentpos += nbytes;
1182 return(0);
1183 }
1184 /*--------------------------------------------------------------------------*/
mem_write(int hdl,void * buffer,long nbytes)1185 int mem_write(int hdl, void *buffer, long nbytes)
1186 /*
1187 write bytes at the current position in the file
1188 */
1189 {
1190 size_t newsize;
1191 char *ptr;
1192
1193 if ((size_t) (memTable[hdl].currentpos + nbytes) >
1194 *(memTable[hdl].memsizeptr) )
1195 {
1196
1197 if (!(memTable[hdl].mem_realloc))
1198 {
1199 ffpmsg("realloc function not defined (mem_write)");
1200 return(WRITE_ERROR);
1201 }
1202
1203 /*
1204 Attempt to reallocate additional memory:
1205 the memory buffer size is incremented by the larger of:
1206 1 FITS block (2880 bytes) or
1207 the defined 'deltasize' parameter
1208 */
1209
1210 newsize = maxvalue( (size_t)
1211 (((memTable[hdl].currentpos + nbytes - 1) / 2880) + 1) * 2880,
1212 *(memTable[hdl].memsizeptr) + memTable[hdl].deltasize);
1213
1214 /* call the realloc function */
1215 ptr = (memTable[hdl].mem_realloc)(
1216 *(memTable[hdl].memaddrptr),
1217 newsize);
1218 if (!ptr)
1219 {
1220 ffpmsg("Failed to reallocate memory (mem_write)");
1221 return(MEMORY_ALLOCATION);
1222 }
1223
1224 *(memTable[hdl].memaddrptr) = ptr;
1225 *(memTable[hdl].memsizeptr) = newsize;
1226 }
1227
1228 /* now copy the bytes from the buffer into memory */
1229 memcpy( *(memTable[hdl].memaddrptr) + memTable[hdl].currentpos,
1230 buffer,
1231 nbytes);
1232
1233 memTable[hdl].currentpos += nbytes;
1234 memTable[hdl].fitsfilesize =
1235 maxvalue(memTable[hdl].fitsfilesize,
1236 memTable[hdl].currentpos);
1237 return(0);
1238 }
1239
1240
1241 #if HAVE_BZIP2
bzip2uncompress2mem(char * filename,FILE * diskfile,int hdl,size_t * filesize,int * status)1242 void bzip2uncompress2mem(char *filename, FILE *diskfile, int hdl,
1243 size_t* filesize, int* status) {
1244 BZFILE* b;
1245 int bzerror;
1246 char buf[8192];
1247 size_t total_read = 0;
1248 char* errormsg = NULL;
1249
1250 *filesize = 0;
1251 *status = 0;
1252 b = BZ2_bzReadOpen(&bzerror, diskfile, 0, 0, NULL, 0);
1253 if (bzerror != BZ_OK) {
1254 BZ2_bzReadClose(&bzerror, b);
1255 if (bzerror == BZ_MEM_ERROR)
1256 ffpmsg("failed to open a bzip2 file: out of memory\n");
1257 else if (bzerror == BZ_CONFIG_ERROR)
1258 ffpmsg("failed to open a bzip2 file: miscompiled bzip2 library\n");
1259 else if (bzerror == BZ_IO_ERROR)
1260 ffpmsg("failed to open a bzip2 file: I/O error");
1261 else
1262 ffpmsg("failed to open a bzip2 file");
1263 *status = READ_ERROR;
1264 return;
1265 }
1266 bzerror = BZ_OK;
1267 while (bzerror == BZ_OK) {
1268 int nread;
1269 nread = BZ2_bzRead(&bzerror, b, buf, sizeof(buf));
1270 if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) {
1271 *status = mem_write(hdl, buf, nread);
1272 if (*status) {
1273 BZ2_bzReadClose(&bzerror, b);
1274 if (*status == MEMORY_ALLOCATION)
1275 ffpmsg("Failed to reallocate memory while uncompressing bzip2 file");
1276 return;
1277 }
1278 total_read += nread;
1279 } else {
1280 if (bzerror == BZ_IO_ERROR)
1281 errormsg = "failed to read bzip2 file: I/O error";
1282 else if (bzerror == BZ_UNEXPECTED_EOF)
1283 errormsg = "failed to read bzip2 file: unexpected end-of-file";
1284 else if (bzerror == BZ_DATA_ERROR)
1285 errormsg = "failed to read bzip2 file: data integrity error";
1286 else if (bzerror == BZ_MEM_ERROR)
1287 errormsg = "failed to read bzip2 file: insufficient memory";
1288 }
1289 }
1290 BZ2_bzReadClose(&bzerror, b);
1291 if (bzerror != BZ_OK) {
1292 if (errormsg)
1293 ffpmsg(errormsg);
1294 else
1295 ffpmsg("failure closing bzip2 file after reading\n");
1296 *status = READ_ERROR;
1297 return;
1298 }
1299 *filesize = total_read;
1300 }
1301 #endif
1302