1 /*****************************************************************************
2 Major portions of this software are copyrighted by the Medical College
3 of Wisconsin, 1994-2000, and are released under the Gnu General Public
4 License, Version 2. See the file README.Copyright for details.
5 ******************************************************************************/
6
7 #include "thd_iochan.h"
8 #include "Amalloc.h"
9 #include <sys/stat.h>
10
11 static int debug = 0 ;
12 #define FAILED if(debug)fprintf(stderr," **FAILED\n")
13 #define DMESS(s,t) if(debug)fprintf(stderr,s,t)
14
15 extern long long THD_filesize( char * pathname ) ;
16 extern char * THD_find_executable( char * ) ;
17
18 /*---------------------------------------------------------------------*/
19 static int use_http_ver = 0 ; /* defaults to HTTP 0.9 */
20 static char *http_user_agent = "read_URL" ;
21
set_HTTP_10(int n)22 void set_HTTP_10( int n ){ /* 24 Mar 2005; ZSS Apr. 2011*/
23 if (n) use_http_ver = 10;
24 else use_http_ver = 0;
25 }
26
set_HTTP_11(int n)27 void set_HTTP_11( int n ){ /* 24 Mar 2005; ZSS Apr. 2011*/
28 if (n) use_http_ver = 11;
29 else use_http_ver = 0;
30 }
31
32 /*---------------------------------------------------------------------*/
33
set_HTTP_user_agent(char * ua)34 extern void set_HTTP_user_agent( char *ua )
35 {
36 if( ua == NULL || *ua == '\0' ) http_user_agent = "read_URL" ;
37 else http_user_agent = strdup(ua) ;
38 }
39
40 /*---------------------------------------------------------------------*/
41 static char tmpdir[256] = "\0" ;
42
setup_tmpdir(void)43 static void setup_tmpdir(void) /* 02 Apr 1999 */
44 {
45 char * td ;
46
47 if( tmpdir[0] != '\0' ) return ;
48
49 td = getenv("TMPDIR") ; /* try two possibilities */
50 if( td == NULL ) td = getenv("TEMPDIR") ;
51
52 if( td == NULL || td[0] == '\0' || strlen(td) > 222 ){
53 strcpy(tmpdir,"/tmp/") ;
54 } else {
55 int ltd = strlen(td) ;
56 strcpy(tmpdir,td) ;
57 if( tmpdir[ltd-1] != '/' ) strcat(tmpdir,"/") ;
58 }
59
60 return ;
61 }
62
63 /*---------------------------------------------------------------------
64 Open an "http://" URL in host, port, and filename pieces.
65 Wait up to msec milliseconds for network functions to occur.
66 If an error occurs, return NULL, otherwise the caller can read
67 from this IOCHAN (using iochan_recv, etc.).
68 -----------------------------------------------------------------------*/
69
open_URL_hpf(char * host,int port,char * file,int msec)70 IOCHAN * open_URL_hpf( char * host , int port , char * file , int msec )
71 {
72 IOCHAN * ioc ;
73 char str[512] ;
74 int ii ;
75
76 if( host == NULL || port <= 0 || file == NULL ) return NULL ;
77
78 sprintf(str,"tcp:%s:%d",host,port) ;
79 DMESS(" ++Opening %s",str);
80 ioc = iochan_init( str , "create" ) ;
81 if( ioc == NULL ){ FAILED; return NULL; }
82 if( debug )fprintf(stderr,".");
83 iochan_set_cutoff( ioc ) ;
84 if( debug )fprintf(stderr,".");
85 ii = iochan_writecheck( ioc , msec ) ;
86 if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc) ; return NULL ; }
87
88 DMESS(" ++GET %s",file);
89 if( use_http_ver == 11)
90 sprintf(str,"GET %s HTTP/1.1\r\n"
91 "Host: %s\r\n"
92 "User-Agent: %s\r\n"
93 "\r\n", file , host, http_user_agent ) ; /* HTTP 1.1 */
94 else if( use_http_ver == 10)
95 sprintf(str,"GET %s HTTP/1.0\r\n"
96 "User-Agent: %s\r\n"
97 "\r\n", file , http_user_agent ) ; /* HTTP 1.0 */
98 else
99 sprintf(str,"GET %s\r\n",file) ; /* HTTP 0.9 */
100 ii = iochan_sendall( ioc , str , strlen(str) ) ;
101 if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc); return NULL; }
102
103 ii = iochan_readcheck( ioc , msec ) ;
104 if( ii <= 0 ){ FAILED; IOCHAN_CLOSE(ioc) ; return NULL ; }
105 DMESS("%s"," **OPENED");
106 return ioc ;
107 }
108
109 /*----------------------------------------------------------------------
110 Open an "http://" URL and prepare to read it (but the caller must
111 actually do the reading). If NULL is returned, an error occurred.
112 ------------------------------------------------------------------------*/
113
114 #define HTTP "http://"
115 #define HTTPLEN 7
116
117 #define HTTPS "https://"
118 #define HTTPSLEN 8
119
120 #define FTP "ftp://"
121 #define FTPLEN 6
122
open_URL_http(char * url,int msec)123 IOCHAN * open_URL_http( char * url , int msec )
124 {
125 char * s, * h , * file ;
126 char hostname[256] ;
127 int port;
128 IOCHAN * ioc ;
129
130 /* check inputs */
131
132 if( url == NULL || strstr(url,HTTP) != url ) return NULL ;
133
134 /* parse hostname */
135
136 for( s=url+HTTPLEN , h=hostname ;
137 (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
138
139 *h = '\0' ; if( hostname[0] == '\0' ) return NULL ;
140
141 /* parse port number if present */
142
143 port = 0 ;
144 if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
145 if( port <= 0 ) port = 80 ;
146
147 /* get the file name (keep leading "/") */
148
149 file = (*s == '/') ? s : "/" ;
150
151 /* do the actual work */
152
153 ioc = open_URL_hpf( hostname , port , file , msec ) ;
154 return ioc ;
155 }
156
157 /*--------------------------------------------------------------*/
158
159 static int prog=0 ;
set_URL_progress(int p)160 void set_URL_progress( int p ){ prog=p; } /* 20 Mar 2003 */
161
162 /*---------------------------------------------------------------------*/
163 /*!
164 strnstr is not standard
165 */
af_strnstr(char * s1,char * s2,size_t n)166 char *af_strnstr(char *s1, char *s2, size_t n)
167 {
168 int n1=0, n2=0;
169 char c1 = '\0', c2 = '\0', *cout=NULL;
170
171 if (s1 && (n1 = strlen(s1)) > n) {c1=s1[n]; s1[n]='\0'; }
172 if (s2 && (n2 = strlen(s2)) > n) {c2=s2[n]; s2[n]='\0'; }
173
174 cout = strstr(s1, s2);
175
176 if (n1 > n) s1[n] = c1;
177 if (n2 > n) s2[n] = c2;
178
179 return(cout);
180 }
181
182 /*---------------------------------------------------------------------*/
183 /*
184 A very simple parsing of HTTP1.1 header fields.
185 Assumes buf and hname are all upper case.
186 Function does not know to avoid searching beyond
187 end of headers ZSS Mar. 2011
188 */
189
HTTP_header_val(char * head,char * hname,size_t max_head)190 char *HTTP_header_val(char *head, char *hname, size_t max_head)
191 {
192 int n_hname = 0;
193 char *cpt = NULL;
194
195 if (!hname || !head) return(NULL);
196
197 if (!af_strnstr(head,"HTTP/1.1", 36)) return(NULL);
198 if (max_head <= 0) {
199 if (strlen(head)<1024) max_head = strlen(head);
200 else max_head = 1024;
201 }
202 n_hname = strlen(hname);
203 cpt = af_strnstr(head,hname, max_head);
204
205 if (cpt) return(cpt+n_hname);
206
207 return(NULL);
208 }
209
210 /*---------------------------------------------------------------------*/
211
HTTP_header_long_val(char * head,char * hname,size_t max_head,long errval)212 long HTTP_header_long_val(char *head, char *hname, size_t max_head, long errval)
213 {
214 char *cpt = NULL;
215 long val=errval;
216
217 if ((cpt = HTTP_header_val(head, hname, max_head))) {
218 val = strtol(cpt, NULL, 10);
219 }
220 return(val);
221 }
222
223 /*---------------------------------------------------------------
224 Read an "http://" URL, with network waits of up to msec
225 milliseconds allowed. Returns number of bytes read -- if this
226 is > 0, then *data will be a pointer to malloc-ed bytes holding
227 the contents of the file.
228
229 If the file is gzip-ed, then it will be un-gzip-ed before being
230 loaded into memory. This uses temporary files in $TMPDIR or
231 /tmp, which must have space to hold the compressed and
232 uncompressed file. If the file is not compressed, then input
233 is directly to memory and no temporary files are used.
234
235 read_URL_http11 is a new version that is meant to handle
236 HTTP1.1 header and return as soon as entity body is read
237 -----------------------------------------------------------------*/
238
239 #define QBUF 4096
240 typedef struct {
241 char *page; /* the whole page */
242 size_t N_head; /* size of header */
243 int head_complete; /* header reading complete */
244 size_t N_page; /* size of page */
245 size_t N_cont; /* size of content */
246 size_t N_alloc; /* allocated size for page */
247 float http_ver;
248 int status;
249 int N_chunks;
250 int cflag;
251 char *data;
252 } URL_PAGE;
253
254 /*---------------------------------------------------------------------*/
255
page_append(char * buf,int n_buf,URL_PAGE * up,int null_term)256 int page_append(char *buf, int n_buf, URL_PAGE *up, int null_term)
257 {
258 if (up->N_page+n_buf > up->N_alloc) {
259 do { up->N_alloc += QBUF; } while (up->N_alloc <= up->N_page+n_buf);
260 up->page = AFREALL(up->page, char, up->N_alloc+1) ;
261 }
262 memcpy( up->page+up->N_page, buf, n_buf);
263 up->N_page += n_buf;
264 if (null_term) {
265 /* make sure we've got a plug */
266 if (up->page[up->N_page-1] != '\0') up->page[up->N_page]='\0';
267 }
268
269 ++up->N_chunks;
270 return(1);
271 }
272
273 /*---------------------------------------------------------------------*/
274
page_parse_status(URL_PAGE * up)275 int page_parse_status(URL_PAGE *up) {
276 char *ttt=NULL, *cpt=NULL;
277 int i, j;
278
279 if (up->status > 0) return(1); /* done */
280 if (!up->page || up->N_page < 1) return(0);
281 i = 0;
282 while (i < up->N_page && up->page[i] != '\r' && up->page[i] != '\n') ++i;
283 /* upper casing */
284 ttt = (char *)calloc(i+1, sizeof(char));
285 for (j=0; j< i; ++j) ttt[j] = toupper(up->page[j]);
286 ttt[j] = '\0';
287
288 /* parse status */
289 up->http_ver = 0.0; up->status = 0;
290 if ((cpt = strstr(ttt,"HTTP/"))) {
291 up->http_ver = (float)strtod(cpt+5, NULL);
292 /* a more proper parsing should be as 1*DIGIT "." 1*DIGIT */
293 j = 0;
294 while (!isblank(cpt[j])) ++j;
295 up->status = (int)strtol(cpt+j, NULL, 10);
296 } else { /* older stuff */
297 up->http_ver = 0.9;
298 /* search more than 1st line for NOT FOUND */
299 ttt = (char *)realloc(ttt, (up->N_page+1)*sizeof(char));
300 for (j=0; j< up->N_page; ++j) ttt[j] = toupper(up->page[j]);
301 ttt[j] = '\0';
302 if (af_strnstr(ttt,"NOT FOUND", 255)) {
303 up->status = 404;
304 }
305 up->status = 200; /* fake it */
306 }
307
308 free(ttt); ttt=NULL;
309 return(1);
310 }
311
312 /*---------------------------------------------------------------------*/
313
page_not_found(URL_PAGE * up)314 int page_not_found(URL_PAGE *up) {
315 return(up->status >= 400 ? 1:0);
316 }
317
318 /*---------------------------------------------------------------------*/
319
page_scan_head(URL_PAGE * up)320 int page_scan_head(URL_PAGE *up) {
321 int i=0, nl=0;
322
323 if (up->head_complete) return(1);
324
325 /* start a couple of characters before the last stop */
326 i = up->N_head-5; if (i<1) i = 1;
327
328 /* search for sequential new lines*/
329 nl = 0;
330 while (i<up->N_page && nl<2) {
331 if( up->page[i] == '\r' ) ++nl;
332 else if( up->page[i] != '\n' ) nl = 0; /* not blank */
333 ++i;
334 }
335 if (nl == 2) {
336 up->head_complete = 1;
337 }
338 up->N_head += i;
339
340 /* make header all upper case */
341 for (i=0; i<up->N_head; ++i) up->page[i] = toupper(up->page[i]);
342
343 /* move till next non new line */
344 while (up->page[up->N_head] == '\n' || up->page[up->N_head] == '\r')
345 ++up->N_head;
346 return(1);
347 }
348
349 /*---------------------------------------------------------------------*/
350 /* return a copy of the header.
351 Caller must free pointer */
page_header_copy(URL_PAGE * up)352 char *page_header_copy(URL_PAGE *up)
353 {
354 char *hcp=NULL;
355 if (!up->page || !up->head_complete) return(NULL);
356 hcp = (char *)calloc(up->N_head+1, sizeof(char));
357 memcpy(hcp, up->page, (up->N_head+1)*sizeof(char));
358 hcp[up->N_head] = '\0';
359 return(hcp);
360 }
361
362 /*---------------------------------------------------------------------*/
363 /* return a pointer to the beginning of the
364 content. Do not free this pointer */
page_content(URL_PAGE * up)365 char *page_content(URL_PAGE *up)
366 {
367 if (up->http_ver < 1.1) return(up->page);
368 if (!up->page || !up->head_complete) return(NULL);
369 return(up->page+up->N_head);
370 }
371
372 /*---------------------------------------------------------------------*/
373
page_init(URL_PAGE * up,char * url)374 int page_init(URL_PAGE *up, char *url)
375 {
376 int ii;
377 char *cpt=NULL, qname[256] ;
378
379 memset(up, 0, sizeof(URL_PAGE));
380 if (!url) return(0);
381
382 ii = strlen(url) ;
383 if( ii > 3 ){
384 cpt = url + (ii-3) ; up->cflag = (strcmp(cpt,".gz") == 0) ;
385 } else {
386 up->cflag = 0 ;
387 }
388
389 return(1);
390 }
391
392 /*---------------------------------------------------------------------*/
393
page_dump(URL_PAGE * up,FILE * out,char * head)394 int page_dump(URL_PAGE *up, FILE *out, char *head)
395 {
396 char cct={'\0'};
397 if (out==NULL) out = stderr;
398
399 if (head) fprintf(out,"%s",head);
400 fprintf(out,"<page:%zu>%s<\\page:%zu>\n",
401 up->N_page, up->page ? up->page:"NULL", up->N_page);
402 if (up->page && up->N_head) {
403 cct = up->page[up->N_head]; up->page[up->N_head] = '\0';
404 }
405 fprintf(out,"<head:%zu-%s>%s<\\head:%zu-%s>\n",
406 up->N_head, up->head_complete ? "complete":"incomplete",
407 up->page ? up->page:"NULL",
408 up->N_head, up->head_complete ? "complete":"incomplete");
409 if (up->page && up->N_head) up->page[up->N_head] = cct;
410 fprintf(out,"<ver>%f<\\ver><status>%d<\\status>\n"
411 "<n_chunks>%d<\\n_chunks>\n"
412 "<cont_len>%zu<\\cont_len>\n"
413 "<cflag>%d<\\cflag>\n"
414 "<data>%s<\\data>\n",
415 up->http_ver, up->status, up->N_chunks, up->N_cont,
416 up->cflag,
417 up->data ? up->data:"NULL");
418 return(1);
419 }
420
421 /*---------------------------------------------------------------------*/
422
page_delete(URL_PAGE * up)423 int page_delete(URL_PAGE *up)
424 {
425 if (up->page) free(up->page);
426 if (up->data) free(up->data);
427 memset(up, 0, sizeof(URL_PAGE));
428
429 return(1);
430 }
431
432 /*---------------------------------------------------------------------*/
433
page_set_data(URL_PAGE * up)434 int page_set_data(URL_PAGE *up)
435 {
436 char qname[256], sbuf[512];
437 int ii, nuse=0;
438 FILE *cfile=NULL;
439
440 if (up->data) return(1);
441
442 if( up->cflag ){ /* uncompress via temp file */
443 setup_tmpdir() ;
444 strcpy(qname,tmpdir) ; strcat(qname,"gosiaXXXXXX") ;
445 mkstemp(qname) ; /* from mktemp 22 Sep, 2014 [rickr] */
446 if( qname[0] != '\0' ){
447 strcat(qname,".gz") ; cfile = fopen( qname , "wb" ) ;
448 if( cfile == NULL ) up->cflag = 0 ;
449 } else {
450 up->cflag = 0 ;
451 }
452
453 if( up->cflag == 0 ){
454 DMESS(" **Temp file %s FAILS\n",qname);
455 up->cflag = -1;
456 return(-1);
457 }
458 DMESS(" ++Temp file=%s",qname);
459
460 /* dump to file */
461 if( fwrite( up->page+up->N_head , 1 , up->N_cont , cfile )
462 != up->N_page-up->N_head ){ /* write failed? */
463 DMESS("\n** Write to temp file %s FAILED!\n",qname);
464 page_delete(up);
465 return( -1 );
466 }
467 fclose(cfile); cfile = NULL;
468
469 /* uncompress and bring back */
470 sprintf( sbuf , "gzip -dq %s" , qname ) ; /* execute gzip */
471 ii = system(sbuf) ;
472 if( ii != 0 ){ DMESS("%s"," **gzip failed!\n");
473 unlink(qname) ; return( -1 ); } /* gzip failed */
474 ii = strlen(qname) ; qname[ii-3] = '\0' ; /* fix filename */
475 nuse = THD_filesize( qname ) ; /* find how big */
476 if( nuse <= 0 ){ DMESS("%s"," **gzip failed!\n");
477 unlink(qname) ; return( -1 ); }
478
479 cfile = fopen( qname , "rb" ) ;
480 if( cfile == NULL ){ DMESS("%s"," **gzip failed!\n");
481 unlink(qname) ; return( -1 ); }
482 up->data = AFMALL(char, nuse) ;
483 fread( up->data , 1 , nuse , cfile ) ; /* read file in */
484 fclose(cfile) ; unlink(qname) ;
485 } else {
486 up->data = AFMALL(char, up->N_page - up->N_head+1);
487 memcpy(up->data, up->page+up->N_head,
488 sizeof(char)*(up->N_page-up->N_head));
489 up->data[up->N_page-up->N_head] = '\0';
490 nuse = up->N_page-up->N_head;
491 }
492 return(nuse);
493 }
494
495 /*---------------------------------------------------------------------*/
496
page_received(URL_PAGE * up)497 int page_received(URL_PAGE *up) {
498 if (up->http_ver < 1.1) return(0);
499 if (up->head_complete) {
500 /* get content length */
501 up->N_cont = HTTP_header_long_val(up->page, "CONTENT-LENGTH:",
502 up->N_head, -1);
503 if (up->N_cont >=0 && up->N_page >= up->N_head+up->N_cont) return(1);
504 }
505 return(0);
506 }
507
508 /*---------------------------------------------------------------------*/
509 /*!
510 Read an HTTP/1.1 url
511 url: The URL
512 msec: Number of msec to wait before abandoning all hope
513 data: A non null pointer to a null char *pointer that will hold the
514 content of the response. At call time, data != NULL and *data = NULL
515 head: If not null, *head will contain all the headers in the response
516
517 returns -1 in failure, total number of characters in data otherwise
518 */
read_URL_http11(char * url,int msec,char ** data,char ** head)519 int read_URL_http11( char * url , int msec , char ** data, char **head )
520 {
521 IOCHAN * ioc ;
522 char * cpt , qbuf[QBUF] ;
523 int ii,jj , nuse , nget=0, nmeg=0 ;
524 size_t con_len = -1;
525 URL_PAGE up;
526
527 /* sanity check */
528
529 if( url == NULL || data == NULL || *data || (head && *head) || msec < 0 )
530 return( -1 );
531
532 /* open http channel to get url */
533
534 ioc = open_URL_http( url , msec ) ;
535 if( ioc == NULL ){ DMESS("%s","\n"); return( -1 ); }
536
537 /* check if url will be returned gzip-ed */
538 page_init(&up, url);
539
540 /* read all of url */
541 nuse = 0 ;
542 do{
543 if(debug)fprintf(stderr,".");
544 ii = iochan_readcheck( ioc , msec ) ; /* wait for data to be ready */
545 if( ii <= 0 ) break ; /* quit if no data */
546 ii = iochan_recv( ioc , qbuf , QBUF ) ;
547 if( ii <= 0 ) break ; /* quit if no data */
548
549 if( prog ){
550 nget += ii ; jj = nget/(1024*1024) ;
551 if( jj > nmeg ){ nmeg=jj; if(debug)fprintf(stderr,"."); }
552 }
553
554 page_append(qbuf, ii, &up, 1); /* append bufer to structure */
555 page_parse_status(&up); /* sets version and status if not set */
556 if (page_not_found(&up)) { /* NOT FOUND? */
557 page_delete(&up);
558 DMESS("%s"," **NOT FOUND\n");
559 IOCHAN_CLOSE(ioc) ; return( -1 );
560 }
561 page_scan_head(&up); /* scan for header, if needed */
562
563 if (debug) page_dump(&up, NULL, NULL );
564
565 nuse += ii ; /* how many bytes so far */
566
567 } while(!page_received(&up)) ;
568
569 IOCHAN_CLOSE(ioc) ;
570
571 if( prog && nmeg > 0 ) fprintf(stderr,"!\n") ;
572
573 /* didn't get anything? */
574
575 if( nuse <= 0 ){
576 page_delete(&up);
577 FAILED; return(-1);
578 }
579 if(debug) fprintf(stderr,"!\n");
580
581 /* Set data */
582 nuse = page_set_data(&up);
583 DMESS("%s","\n"); *data = up.data ; up.data=NULL;
584
585 /* want header ? */
586 if (head) {
587 *head = (char *)realloc(up.page, (up.N_head+1)*sizeof(char));
588 *head[up.N_head] = '\0';
589 up.page = NULL;
590 }
591 page_delete(&up);
592 return(nuse);
593 }
594
595 /*---------------------------------------------------------------------*/
596
read_URL_http(char * url,int msec,char ** data)597 int read_URL_http( char * url , int msec , char ** data )
598 {
599 IOCHAN * ioc ;
600 char * buf=NULL , * cpt , qbuf[QBUF] , qname[256] ;
601 int ii,jj , nall=0 , nuse , nget=0, nmeg=0 ;
602 int cflag , first ;
603 FILE *cfile=NULL ;
604
605 if (use_http_ver == 11) return(read_URL_http11( url , msec , data, NULL ));
606
607 /* sanity check */
608
609 if( url == NULL || data == NULL || msec < 0 ) return( -1 );
610
611 /* open http channel to get url */
612
613 ioc = open_URL_http( url , msec ) ;
614 if( ioc == NULL ){ DMESS("%s","\n"); return( -1 ); }
615
616 /* check if url will be returned gzip-ed */
617
618 ii = strlen(url) ;
619 if( ii > 3 ){
620 cpt = url + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
621 } else {
622 cflag = 0 ;
623 }
624
625 if( cflag ){
626 setup_tmpdir() ;
627 strcpy(qname,tmpdir) ; strcat(qname,"gosiaXXXXXX") ;
628 mkstemp(qname) ;
629 if( qname[0] != '\0' ){
630 strcat(qname,".gz") ; cfile = fopen( qname , "wb" ) ;
631 if( cfile == NULL ) cflag = 0 ;
632 } else {
633 cflag = 0 ;
634 }
635
636 if( cflag == 0 ){
637 DMESS(" **Temp file %s FAILS\n",qname); IOCHAN_CLOSE(ioc); return(-1);
638 }
639 DMESS(" ++Temp file=%s",qname);
640 }
641
642 /* read all of url */
643
644 if( !cflag ){ buf = AFMALL(char, QBUF ) ; nall = QBUF ; }
645 nuse = 0 ; first = 1 ;
646
647 do{
648 if(debug)fprintf(stderr,".");
649 ii = iochan_readcheck( ioc , msec ) ; /* wait for data to be ready */
650 if( ii <= 0 ) break ; /* quit if no data */
651 ii = iochan_recv( ioc , qbuf , QBUF ) ;
652 if( ii <= 0 ) break ; /* quit if no data */
653
654 if( prog ){
655 nget += ii ; jj = nget/(1024*1024) ;
656 if( jj > nmeg ){ nmeg=jj; if(debug)fprintf(stderr,"."); }
657 }
658
659 if( first ){ /* check for "not found" */
660 if( buf == NULL ){ buf = AFMALL(char, ii) ; }
661 memcpy( buf , qbuf , ii ) ;
662 for( jj=0 ; jj < ii ; jj++ ) buf[jj] = toupper(buf[jj]) ;
663 buf[ii-1] = '\0' ;
664 cpt = strstr(buf,"NOT FOUND") ;
665 if( cpt != NULL ){
666 if( cflag ){ fclose(cfile) ; unlink(qname) ; }
667 DMESS("%s"," **NOT FOUND\n");
668 free(buf) ; IOCHAN_CLOSE(ioc) ; return( -1 );
669 }
670 first = 0 ;
671 if( cflag ){ free(buf) ; buf = NULL ; }
672 }
673
674 if( cflag ){ /* write to temp file */
675 nall = fwrite( qbuf , 1 , ii , cfile ) ;
676 if( nall != ii ){ /* write failed? */
677 DMESS("\n** Write to temp file %s FAILED!\n",qname);
678 fclose(cfile) ; unlink(qname) ;
679 IOCHAN_CLOSE(ioc) ; return( -1 );
680 }
681 } else { /* save to buffer */
682 if( nuse+ii > nall ){ /* enlarge buffer? */
683 nall += QBUF ;
684 buf = AFREALL(buf, char, nall) ;
685 }
686 memcpy( buf+nuse , qbuf , ii ) ; /* copy data into buffer */
687 }
688 nuse += ii ; /* how many bytes so far */
689 } while(1) ;
690 IOCHAN_CLOSE(ioc) ;
691
692 if( prog && nmeg > 0 ) fprintf(stderr,"!\n") ;
693
694 /* didn't get anything? */
695
696 if( nuse <= 0 ){
697 if( cflag ){ fclose(cfile) ; unlink(qname) ; }
698 else { free(buf) ; }
699 FAILED; return(-1);
700 }
701 if(debug)fprintf(stderr,"!\n");
702
703 /* uncompression time? */
704
705 if( cflag ){
706 fclose(cfile) ;
707 sprintf( qbuf , "gzip -dq %s" , qname ) ; /* execute gzip */
708 ii = system(qbuf) ;
709 if( ii != 0 ){ DMESS("%s"," **gzip failed!\n");
710 unlink(qname) ; return( -1 ); } /* gzip failed */
711 ii = strlen(qname) ; qname[ii-3] = '\0' ; /* fix filename */
712 nuse = THD_filesize( qname ) ; /* find how big */
713 if( nuse <= 0 ){ DMESS("%s"," **gzip failed!\n");
714 unlink(qname) ; return( -1 ); }
715
716 cfile = fopen( qname , "rb" ) ;
717 if( cfile == NULL ){ DMESS("%s"," **gzip failed!\n");
718 unlink(qname) ; return( -1 ); }
719 buf = AFMALL(char, nuse) ;
720 fread( buf , 1 , nuse , cfile ) ; /* read file in */
721 fclose(cfile) ; unlink(qname) ;
722 }
723
724 /* data is in buf, nuse bytes of it */
725
726 DMESS("%s","\n"); *data = buf ; return( nuse );
727 }
728
729 /*---------------------------------------------------------------------*/
730
read_URL_https(char * url,char ** data)731 int read_URL_https( char *url , char **data ) /* 11 Feb 2016 */
732 {
733 FILE *fp ;
734 char *cmd , *getprog=NULL , *buf=NULL , qbuf[QBUF] ;
735 size_t nbuf=0 , nnn ;
736
737 if( url == NULL || *url == '\0' || data == NULL ) return(-1) ;
738
739 if( getprog == NULL ){
740 getprog = THD_find_executable("wget") ;
741 if( getprog != NULL ){
742 cmd = (char *)malloc(sizeof(char)*(strlen(getprog)+strlen(url)+64)) ;
743 sprintf( cmd , "%s -o /dev/null -O - %s",getprog,url) ;
744 }
745 }
746
747 if( getprog == NULL ){
748 getprog = THD_find_executable("curl") ;
749 if( getprog != NULL ){
750 cmd = (char *)malloc(sizeof(char)*(strlen(getprog)+strlen(url)+64)) ;
751 sprintf( cmd , "%s --silent -f -o - %s",getprog,url) ;
752 }
753 }
754
755 if( getprog == NULL ) return(-1) ;
756
757 fp = popen( cmd , "r" ) ;
758 if( fp == NULL ) return(-1) ;
759
760 iochan_sleep(10) ;
761 while(1){
762 nnn = fread( qbuf , 1,QBUF-1 , fp ) ;
763 if( nnn == 0 ) break ;
764 if( nnn > 10 ){
765 qbuf[nnn] = '\0' ;
766 if( strcasestr(qbuf,"404 Not Found") != NULL ){
767 pclose(fp) ; if( buf != NULL ) free(buf) ;
768 return(-1) ;
769 }
770 }
771 buf = (char *)realloc( buf , nbuf+nnn+1 ) ;
772 memcpy( buf+nbuf , qbuf , nnn ) ;
773 nbuf += nnn ;
774 iochan_sleep(1) ;
775 }
776
777 pclose(fp) ;
778 if( buf == NULL || nbuf == 0 ) return(-1) ;
779
780 *data = buf ; return((int)nbuf) ;
781 }
782
783 /*---------------------------------------------------------------------*/
784
785 static char ftp_name[128] = "anonymous" ;
786 static char ftp_pwd[128] = "AFNI@nowhere.org" ;
787
set_URL_ftp_ident(char * name,char * pwd)788 void set_URL_ftp_ident( char * name , char * pwd ) /* 05 Apr 1999 */
789 {
790 int ll ;
791
792 if( name == NULL || pwd == NULL ) return ;
793
794 ll = strlen(name) ; if( ll < 1 || ll > 127 ) return ;
795 ll = strlen(pwd) ; if( ll < 1 || ll > 127 ) return ;
796
797 strcpy(ftp_name,name) ; strcpy(ftp_pwd,pwd) ; return ;
798 }
799
800 /*---------------------------------------------------------------------
801 Reads an "ftp://" URL, similarly to read_URL_http above;
802 however, staging is always done through a temporary file.
803 -----------------------------------------------------------------------*/
804
read_URL_ftp(char * url,char ** data)805 int read_URL_ftp( char * url , char ** data )
806 {
807 char * s, * h , * file , qname[256] , sname[256] , * cpt , * buf ;
808 char hostname[256] ;
809 int port , ii , cflag , nuse ;
810 FILE * sp ;
811
812 /* sanity check */
813
814 if( url == NULL || data == NULL || strstr(url,FTP) != url ) return( -1 );
815
816 /* parse hostname */
817
818 for( s=url+FTPLEN , h=hostname ;
819 (*s != '\0') && (*s != ':') && (*s != '/') ; s++ , h++ ) *h = *s ;
820
821 *h = '\0' ; if( hostname[0] == '\0' ) return( -1 );
822
823 /* parse port number, if present */
824
825 port = 0 ;
826 if( *s == ':' ){ port = strtol( ++s , &h , 10 ) ; s = h ; }
827
828 /* get the file name (strip off leading "/") */
829
830 if( *s == '/' ){
831 file = s+1 ; if( file[0] == '\0' ) return( -1 );
832 } else {
833 return( -1 );
834 }
835
836 /* check if file will be returned gzip-ed */
837
838 ii = strlen(file) ;
839 if( ii > 3 ){
840 cpt = file + (ii-3) ; cflag = (strcmp(cpt,".gz") == 0) ;
841 } else {
842 cflag = 0 ;
843 }
844
845 /* make name for output file */
846
847 setup_tmpdir() ;
848 strcpy(qname,tmpdir) ; strcat(qname,"elvisXXXXXX") ;
849 mkstemp(qname) ;
850 if( qname[0] == '\0' ) return( -1 );
851 if( cflag ) strcat(qname,".gz") ;
852
853 /* write the script file that will be used to run ftp */
854
855 strcpy(sname,tmpdir) ; strcat(sname,"dahmerXXXXXX") ;
856 mkstemp(sname) ; if( sname[0] == '\0' ) return( -1 );
857 sp = fopen( sname , "w" ) ; if( sp == NULL ) return( -1 );
858
859 fprintf( sp , "#!/bin/sh\n" ) ;
860 fprintf( sp , "ftp -n << EEEEE &> /dev/null\n") ;
861 if( port > 0 )
862 fprintf( sp , "open %s %d\n" , hostname , port ) ;
863 else
864 fprintf( sp , "open %s\n" , hostname ) ;
865 fprintf( sp , "user %s %s\n" , ftp_name, ftp_pwd ) ;
866 fprintf( sp , "binary\n" ) ;
867 fprintf( sp , "passive\n" ) ;
868 fprintf( sp , "get %s %s\n" , file , qname ) ;
869 fprintf( sp , "bye\n" ) ;
870 fprintf( sp , "EEEEE\n" ) ;
871 fprintf( sp , "exit\n" ) ;
872 fclose( sp ) ;
873 chmod( sname , S_IRUSR | S_IWUSR | S_IXUSR ) ;
874
875 /* execute the script, then delete it */
876
877 system( sname ) ; unlink( sname ) ;
878
879 /* check the size of the output file */
880
881 nuse = THD_filesize( qname ) ;
882 if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
883
884 /* uncompress the file, if needed */
885
886 if( cflag ){
887 sprintf( sname , "gzip -dq %s" , qname ) ; /* execute gzip */
888 ii = system(sname) ;
889 if( ii != 0 ){ unlink(qname) ; return( -1 ); } /* gzip failed */
890 ii = strlen(qname) ; qname[ii-3] = '\0' ; /* fix filename */
891 nuse = THD_filesize( qname ) ; /* find how big */
892 if( nuse <= 0 ){ unlink(qname) ; return( -1 ); }
893 }
894
895 /* suck the file into memory */
896
897 sp = fopen( qname , "rb" ) ;
898 if( sp == NULL ){ unlink(qname) ; return( -1 ); }
899 buf = AFMALL(char,nuse) ; if( buf == NULL ){ unlink(qname) ; return( -1 ); }
900
901 fread( buf , 1 , nuse , sp ) ; /* AT LAST! */
902 fclose(sp) ; unlink(qname) ;
903
904 /* data is in buf, nuse bytes of it */
905
906 *data = buf ; return( nuse );
907 }
908
909 /*-------------------------------------------------------------------
910 Read a URL (ftp:// or http://) into memory. The return value
911 is the number of bytes read, and *data points to the data.
912 If the return value is negative, then something bad happened.
913 ---------------------------------------------------------------------*/
914
read_URL(char * url,char ** data)915 int read_URL( char * url , char ** data )
916 {
917 int nn ;
918 if( url == NULL || data == NULL ) return( -1 );
919
920 if( getenv("AFNI_WWW_DEBUG") != NULL ) debug = 1 ;
921
922 if( strstr(url,HTTPS) == url ){ /* 11 Feb 2016 */
923 nn = read_URL_https( url , data ) ; return(nn) ;
924 }
925
926 if( strstr(url,HTTP) == url ){
927 nn = read_URL_http( url , 4000 , data ) ; return(nn) ;
928 }
929
930 else if( strstr(url,FTP) == url ){
931 nn = read_URL_ftp( url , data ) ; return(nn) ;
932 }
933
934 return( -1 );
935 }
936
937 /*------------------------------------------------------------------
938 Read a URL and save it to disk in tmpdir. The filename
939 it is saved in is returned in the malloc-ed space *tname.
940 The byte count is the return value of the function;
941 if <= 0, then an error transpired (and *tname is not set).
942 --------------------------------------------------------------------*/
943
944 extern char * THD_trailname( char * fname , int lev ) ;
945
read_URL_tmpdir(char * url,char ** tname)946 int read_URL_tmpdir( char * url , char ** tname )
947 {
948 int nn , ll ;
949 char * data , * fname , * tt ;
950 FILE * fp ;
951
952 if( url == NULL || tname == NULL ) return( -1 );
953
954 nn = read_URL( url , &data ) ; /* get the data into memory */
955 if( nn <= 0 ) return( -1 ); /* bad */
956
957 /* make the output filename */
958
959 setup_tmpdir() ;
960 fname = AFMALL(char, strlen(url)+strlen(tmpdir)+1) ;
961 tt = THD_trailname(url,0) ;
962 strcpy(fname,tmpdir) ; strcat(fname,tt) ; ll = strlen(fname) ;
963 if( ll > 3 && strcmp(fname+(ll-3),".gz") == 0 ) fname[ll-3] = '\0' ;
964
965 /* open and write output */
966
967 fp = fopen( fname , "wb" ) ;
968 if( fp == NULL ){
969 fprintf(stderr,"** Can't open temporary file %s\n",fname);
970 free(data) ; return( -1 );
971 }
972 ll = fwrite(data,1,nn,fp) ; fclose(fp) ; free(data) ;
973 if( ll != nn ){ unlink(fname); return( -1 ); } /* write failed */
974
975 *tname = fname ; return( nn );
976 }
977