1 /***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
7 *
8 * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
9 *
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.haxx.se/docs/copyright.html.
13 *
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
17 *
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
20 *
21 ***************************************************************************/
22 #include "tool_setup.h"
23
24 #ifdef HAVE_FCNTL_H
25 # include <fcntl.h>
26 #endif
27
28 #ifdef HAVE_LOCALE_H
29 # include <locale.h>
30 #endif
31
32 #ifdef HAVE_SYS_SELECT_H
33 # include <sys/select.h>
34 #endif
35
36 #ifdef __VMS
37 # include <fabdef.h>
38 #endif
39
40 #ifdef __AMIGA__
41 # include <proto/dos.h>
42 #endif
43
44 #include "strcase.h"
45
46 #define ENABLE_CURLX_PRINTF
47 /* use our own printf() functions */
48 #include "curlx.h"
49
50 #include "tool_binmode.h"
51 #include "tool_cfgable.h"
52 #include "tool_cb_dbg.h"
53 #include "tool_cb_hdr.h"
54 #include "tool_cb_prg.h"
55 #include "tool_cb_rea.h"
56 #include "tool_cb_see.h"
57 #include "tool_cb_wrt.h"
58 #include "tool_dirhie.h"
59 #include "tool_doswin.h"
60 #include "tool_easysrc.h"
61 #include "tool_filetime.h"
62 #include "tool_getparam.h"
63 #include "tool_helpers.h"
64 #include "tool_homedir.h"
65 #include "tool_libinfo.h"
66 #include "tool_main.h"
67 #include "tool_metalink.h"
68 #include "tool_msgs.h"
69 #include "tool_operate.h"
70 #include "tool_operhlp.h"
71 #include "tool_paramhlp.h"
72 #include "tool_parsecfg.h"
73 #include "tool_setopt.h"
74 #include "tool_sleep.h"
75 #include "tool_urlglob.h"
76 #include "tool_util.h"
77 #include "tool_writeout.h"
78 #include "tool_xattr.h"
79 #include "tool_vms.h"
80 #include "tool_help.h"
81 #include "tool_hugehelp.h"
82 #include "tool_progress.h"
83
84 #include "memdebug.h" /* keep this as LAST include */
85
86 #ifdef CURLDEBUG
87 /* libcurl's debug builds provide an extra function */
88 CURLcode curl_easy_perform_ev(CURL *easy);
89 #endif
90
91 #define CURLseparator "--_curl_--"
92
93 #ifndef O_BINARY
94 /* since O_BINARY as used in bitmasks, setting it to zero makes it usable in
95 source code but yet it doesn't ruin anything */
96 # define O_BINARY 0
97 #endif
98
99 #define CURL_CA_CERT_ERRORMSG \
100 "More details here: https://curl.haxx.se/docs/sslcerts.html\n\n" \
101 "curl failed to verify the legitimacy of the server and therefore " \
102 "could not\nestablish a secure connection to it. To learn more about " \
103 "this situation and\nhow to fix it, please visit the web page mentioned " \
104 "above.\n"
105
106 static CURLcode create_transfers(struct GlobalConfig *global,
107 struct OperationConfig *config,
108 CURLSH *share,
109 bool capath_from_env);
110
is_fatal_error(CURLcode code)111 static bool is_fatal_error(CURLcode code)
112 {
113 switch(code) {
114 case CURLE_FAILED_INIT:
115 case CURLE_OUT_OF_MEMORY:
116 case CURLE_UNKNOWN_OPTION:
117 case CURLE_FUNCTION_NOT_FOUND:
118 case CURLE_BAD_FUNCTION_ARGUMENT:
119 /* critical error */
120 return TRUE;
121 default:
122 break;
123 }
124
125 /* no error or not critical */
126 return FALSE;
127 }
128
129 /*
130 * Check if a given string is a PKCS#11 URI
131 */
is_pkcs11_uri(const char * string)132 static bool is_pkcs11_uri(const char *string)
133 {
134 if(curl_strnequal(string, "pkcs11:", 7)) {
135 return TRUE;
136 }
137 else {
138 return FALSE;
139 }
140 }
141
142 #ifdef __VMS
143 /*
144 * get_vms_file_size does what it takes to get the real size of the file
145 *
146 * For fixed files, find out the size of the EOF block and adjust.
147 *
148 * For all others, have to read the entire file in, discarding the contents.
149 * Most posted text files will be small, and binary files like zlib archives
150 * and CD/DVD images should be either a STREAM_LF format or a fixed format.
151 *
152 */
vms_realfilesize(const char * name,const struct_stat * stat_buf)153 static curl_off_t vms_realfilesize(const char *name,
154 const struct_stat *stat_buf)
155 {
156 char buffer[8192];
157 curl_off_t count;
158 int ret_stat;
159 FILE * file;
160
161 /* !checksrc! disable FOPENMODE 1 */
162 file = fopen(name, "r"); /* VMS */
163 if(file == NULL) {
164 return 0;
165 }
166 count = 0;
167 ret_stat = 1;
168 while(ret_stat > 0) {
169 ret_stat = fread(buffer, 1, sizeof(buffer), file);
170 if(ret_stat != 0)
171 count += ret_stat;
172 }
173 fclose(file);
174
175 return count;
176 }
177
178 /*
179 *
180 * VmsSpecialSize checks to see if the stat st_size can be trusted and
181 * if not to call a routine to get the correct size.
182 *
183 */
VmsSpecialSize(const char * name,const struct_stat * stat_buf)184 static curl_off_t VmsSpecialSize(const char *name,
185 const struct_stat *stat_buf)
186 {
187 switch(stat_buf->st_fab_rfm) {
188 case FAB$C_VAR:
189 case FAB$C_VFC:
190 return vms_realfilesize(name, stat_buf);
191 break;
192 default:
193 return stat_buf->st_size;
194 }
195 }
196 #endif /* __VMS */
197
198 #define BUFFER_SIZE (100*1024)
199
200 struct per_transfer *transfers; /* first node */
201 static struct per_transfer *transfersl; /* last node */
202
add_transfer(struct per_transfer ** per)203 static CURLcode add_transfer(struct per_transfer **per)
204 {
205 struct per_transfer *p;
206 p = calloc(sizeof(struct per_transfer), 1);
207 if(!p)
208 return CURLE_OUT_OF_MEMORY;
209 if(!transfers)
210 /* first entry */
211 transfersl = transfers = p;
212 else {
213 /* make the last node point to the new node */
214 transfersl->next = p;
215 /* make the new node point back to the formerly last node */
216 p->prev = transfersl;
217 /* move the last node pointer to the new entry */
218 transfersl = p;
219 }
220 *per = p;
221 all_xfers++; /* count total number of transfers added */
222 return CURLE_OK;
223 }
224
225 /* Remove the specified transfer from the list (and free it), return the next
226 in line */
del_transfer(struct per_transfer * per)227 static struct per_transfer *del_transfer(struct per_transfer *per)
228 {
229 struct per_transfer *n;
230 struct per_transfer *p;
231 DEBUGASSERT(transfers);
232 DEBUGASSERT(transfersl);
233 DEBUGASSERT(per);
234
235 n = per->next;
236 p = per->prev;
237
238 if(p)
239 p->next = n;
240 else
241 transfers = n;
242
243 if(n)
244 n->prev = p;
245 else
246 transfersl = p;
247
248 free(per);
249
250 return n;
251 }
252
pre_transfer(struct GlobalConfig * global,struct per_transfer * per)253 static CURLcode pre_transfer(struct GlobalConfig *global,
254 struct per_transfer *per)
255 {
256 curl_off_t uploadfilesize = -1;
257 struct_stat fileinfo;
258 CURLcode result = CURLE_OK;
259
260 if(per->separator_err)
261 fprintf(global->errors, "%s\n", per->separator_err);
262 if(per->separator)
263 printf("%s\n", per->separator);
264
265 if(per->uploadfile && !stdin_upload(per->uploadfile)) {
266 /* VMS Note:
267 *
268 * Reading binary from files can be a problem... Only FIXED, VAR
269 * etc WITHOUT implied CC will work Others need a \n appended to a
270 * line
271 *
272 * - Stat gives a size but this is UNRELIABLE in VMS As a f.e. a
273 * fixed file with implied CC needs to have a byte added for every
274 * record processed, this can by derived from Filesize & recordsize
275 * for VARiable record files the records need to be counted! for
276 * every record add 1 for linefeed and subtract 2 for the record
277 * header for VARIABLE header files only the bare record data needs
278 * to be considered with one appended if implied CC
279 */
280 #ifdef __VMS
281 /* Calculate the real upload size for VMS */
282 per->infd = -1;
283 if(stat(per->uploadfile, &fileinfo) == 0) {
284 fileinfo.st_size = VmsSpecialSize(uploadfile, &fileinfo);
285 switch(fileinfo.st_fab_rfm) {
286 case FAB$C_VAR:
287 case FAB$C_VFC:
288 case FAB$C_STMCR:
289 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
290 break;
291 default:
292 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY,
293 "rfm=stmlf", "ctx=stm");
294 }
295 }
296 if(per->infd == -1)
297 #else
298 per->infd = open(per->uploadfile, O_RDONLY | O_BINARY);
299 if((per->infd == -1) || fstat(per->infd, &fileinfo))
300 #endif
301 {
302 helpf(global->errors, "Can't open '%s'!\n", per->uploadfile);
303 if(per->infd != -1) {
304 close(per->infd);
305 per->infd = STDIN_FILENO;
306 }
307 return CURLE_READ_ERROR;
308 }
309 per->infdopen = TRUE;
310
311 /* we ignore file size for char/block devices, sockets, etc. */
312 if(S_ISREG(fileinfo.st_mode))
313 uploadfilesize = fileinfo.st_size;
314
315 if(uploadfilesize != -1)
316 my_setopt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
317 per->input.fd = per->infd;
318 }
319 show_error:
320 return result;
321 }
322
323 /*
324 * Call this after a transfer has completed.
325 */
post_transfer(struct GlobalConfig * global,CURLSH * share,struct per_transfer * per,CURLcode result,bool * retryp)326 static CURLcode post_transfer(struct GlobalConfig *global,
327 CURLSH *share,
328 struct per_transfer *per,
329 CURLcode result,
330 bool *retryp)
331 {
332 struct OutStruct *outs = &per->outs;
333 CURL *curl = per->curl;
334 struct OperationConfig *config = per->config;
335
336 *retryp = FALSE;
337
338 if(per->infdopen)
339 close(per->infd);
340
341 #ifdef __VMS
342 if(is_vms_shell()) {
343 /* VMS DCL shell behavior */
344 if(!global->showerror)
345 vms_show = VMSSTS_HIDE;
346 }
347 else
348 #endif
349 if(config->synthetic_error) {
350 ;
351 }
352 else if(result && global->showerror) {
353 fprintf(global->errors, "curl: (%d) %s\n", result,
354 (per->errorbuffer[0]) ? per->errorbuffer :
355 curl_easy_strerror(result));
356 if(result == CURLE_PEER_FAILED_VERIFICATION)
357 fputs(CURL_CA_CERT_ERRORMSG, global->errors);
358 }
359
360 /* Set file extended attributes */
361 if(!result && config->xattr && outs->fopened && outs->stream) {
362 int rc = fwrite_xattr(curl, fileno(outs->stream));
363 if(rc)
364 warnf(config->global, "Error setting extended attributes: %s\n",
365 strerror(errno));
366 }
367
368 if(!result && !outs->stream && !outs->bytes) {
369 /* we have received no data despite the transfer was successful
370 ==> force cration of an empty output file (if an output file
371 was specified) */
372 long cond_unmet = 0L;
373 /* do not create (or even overwrite) the file in case we get no
374 data because of unmet condition */
375 curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet);
376 if(!cond_unmet && !tool_create_output_file(outs))
377 result = CURLE_WRITE_ERROR;
378 }
379
380 if(!outs->s_isreg && outs->stream) {
381 /* Dump standard stream buffered data */
382 int rc = fflush(outs->stream);
383 if(!result && rc) {
384 /* something went wrong in the writing process */
385 result = CURLE_WRITE_ERROR;
386 fprintf(global->errors, "(%d) Failed writing body\n", result);
387 }
388 }
389
390 #ifdef USE_METALINK
391 if(per->metalink && !per->metalink_next_res)
392 fprintf(global->errors, "Metalink: fetching (%s) from (%s) OK\n",
393 per->mlfile->filename, per->this_url);
394
395 if(!per->metalink && config->use_metalink && result == CURLE_OK) {
396 int rv = parse_metalink(config, outs, per->this_url);
397 if(!rv) {
398 fprintf(config->global->errors, "Metalink: parsing (%s) OK\n",
399 per->this_url);
400 }
401 else if(rv == -1)
402 fprintf(config->global->errors, "Metalink: parsing (%s) FAILED\n",
403 per->this_url);
404 result = create_transfers(global, config, share, FALSE);
405 }
406 else if(per->metalink && result == CURLE_OK && !per->metalink_next_res) {
407 int rv;
408 (void)fflush(outs->stream);
409 rv = metalink_check_hash(global, per->mlfile, outs->filename);
410 if(!rv)
411 per->metalink_next_res = 1;
412 }
413 #else
414 (void)share;
415 #endif /* USE_METALINK */
416
417 #ifdef USE_METALINK
418 if(outs->metalink_parser)
419 metalink_parser_context_delete(outs->metalink_parser);
420 #endif /* USE_METALINK */
421
422 if(outs->is_cd_filename && outs->stream && !global->mute &&
423 outs->filename)
424 printf("curl: Saved to filename '%s'\n", outs->filename);
425
426 /* if retry-max-time is non-zero, make sure we haven't exceeded the
427 time */
428 if(per->retry_numretries &&
429 (!config->retry_maxtime ||
430 (tvdiff(tvnow(), per->retrystart) <
431 config->retry_maxtime*1000L)) ) {
432 enum {
433 RETRY_NO,
434 RETRY_TIMEOUT,
435 RETRY_CONNREFUSED,
436 RETRY_HTTP,
437 RETRY_FTP,
438 RETRY_LAST /* not used */
439 } retry = RETRY_NO;
440 long response;
441 if((CURLE_OPERATION_TIMEDOUT == result) ||
442 (CURLE_COULDNT_RESOLVE_HOST == result) ||
443 (CURLE_COULDNT_RESOLVE_PROXY == result) ||
444 (CURLE_FTP_ACCEPT_TIMEOUT == result))
445 /* retry timeout always */
446 retry = RETRY_TIMEOUT;
447 else if(config->retry_connrefused &&
448 (CURLE_COULDNT_CONNECT == result)) {
449 long oserrno;
450 curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno);
451 if(ECONNREFUSED == oserrno)
452 retry = RETRY_CONNREFUSED;
453 }
454 else if((CURLE_OK == result) ||
455 (config->failonerror &&
456 (CURLE_HTTP_RETURNED_ERROR == result))) {
457 /* If it returned OK. _or_ failonerror was enabled and it
458 returned due to such an error, check for HTTP transient
459 errors to retry on. */
460 long protocol;
461 curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol);
462 if((protocol == CURLPROTO_HTTP) || (protocol == CURLPROTO_HTTPS)) {
463 /* This was HTTP(S) */
464 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
465
466 switch(response) {
467 case 500: /* Internal Server Error */
468 case 502: /* Bad Gateway */
469 case 503: /* Service Unavailable */
470 case 504: /* Gateway Timeout */
471 retry = RETRY_HTTP;
472 /*
473 * At this point, we have already written data to the output
474 * file (or terminal). If we write to a file, we must rewind
475 * or close/re-open the file so that the next attempt starts
476 * over from the beginning.
477 *
478 * TODO: similar action for the upload case. We might need
479 * to start over reading from a previous point if we have
480 * uploaded something when this was returned.
481 */
482 break;
483 }
484 }
485 } /* if CURLE_OK */
486 else if(result) {
487 long protocol;
488
489 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
490 curl_easy_getinfo(curl, CURLINFO_PROTOCOL, &protocol);
491
492 if((protocol == CURLPROTO_FTP || protocol == CURLPROTO_FTPS) &&
493 response / 100 == 4)
494 /*
495 * This is typically when the FTP server only allows a certain
496 * amount of users and we are not one of them. All 4xx codes
497 * are transient.
498 */
499 retry = RETRY_FTP;
500 }
501
502 if(retry) {
503 long sleeptime = 0;
504 curl_off_t retry_after = 0;
505 static const char * const m[]={
506 NULL,
507 "timeout",
508 "connection refused",
509 "HTTP error",
510 "FTP error"
511 };
512
513 sleeptime = per->retry_sleep;
514 if(RETRY_HTTP == retry) {
515 curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);
516 if(retry_after) {
517 /* store in a 'long', make sure it doesn't overflow */
518 if(retry_after > LONG_MAX/1000)
519 sleeptime = LONG_MAX;
520 else
521 sleeptime = (long)retry_after * 1000; /* milliseconds */
522 }
523 }
524 warnf(config->global, "Transient problem: %s "
525 "Will retry in %ld seconds. "
526 "%ld retries left.\n",
527 m[retry], per->retry_sleep/1000L, per->retry_numretries);
528
529 per->retry_numretries--;
530 tool_go_sleep(sleeptime);
531 if(!config->retry_delay) {
532 per->retry_sleep *= 2;
533 if(per->retry_sleep > RETRY_SLEEP_MAX)
534 per->retry_sleep = RETRY_SLEEP_MAX;
535 }
536 if(outs->bytes && outs->filename && outs->stream) {
537 int rc;
538 /* We have written data to a output file, we truncate file
539 */
540 if(!global->mute)
541 fprintf(global->errors, "Throwing away %"
542 CURL_FORMAT_CURL_OFF_T " bytes\n",
543 outs->bytes);
544 fflush(outs->stream);
545 /* truncate file at the position where we started appending */
546 #ifdef HAVE_FTRUNCATE
547 if(ftruncate(fileno(outs->stream), outs->init)) {
548 /* when truncate fails, we can't just append as then we'll
549 create something strange, bail out */
550 if(!global->mute)
551 fprintf(global->errors,
552 "failed to truncate, exiting\n");
553 return CURLE_WRITE_ERROR;
554 }
555 /* now seek to the end of the file, the position where we
556 just truncated the file in a large file-safe way */
557 rc = fseek(outs->stream, 0, SEEK_END);
558 #else
559 /* ftruncate is not available, so just reposition the file
560 to the location we would have truncated it. This won't
561 work properly with large files on 32-bit systems, but
562 most of those will have ftruncate. */
563 rc = fseek(outs->stream, (long)outs->init, SEEK_SET);
564 #endif
565 if(rc) {
566 if(!global->mute)
567 fprintf(global->errors,
568 "failed seeking to end of file, exiting\n");
569 return CURLE_WRITE_ERROR;
570 }
571 outs->bytes = 0; /* clear for next round */
572 }
573 *retryp = TRUE; /* curl_easy_perform loop */
574 return CURLE_OK;
575 }
576 } /* if retry_numretries */
577 else if(per->metalink) {
578 /* Metalink: Decide to try the next resource or not. Try the next resource
579 if download was not successful. */
580 long response;
581 if(CURLE_OK == result) {
582 /* TODO We want to try next resource when download was
583 not successful. How to know that? */
584 char *effective_url = NULL;
585 curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
586 if(effective_url &&
587 curl_strnequal(effective_url, "http", 4)) {
588 /* This was HTTP(S) */
589 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
590 if(response != 200 && response != 206) {
591 per->metalink_next_res = 1;
592 fprintf(global->errors,
593 "Metalink: fetching (%s) from (%s) FAILED "
594 "(HTTP status code %ld)\n",
595 per->mlfile->filename, per->this_url, response);
596 }
597 }
598 }
599 else {
600 per->metalink_next_res = 1;
601 fprintf(global->errors,
602 "Metalink: fetching (%s) from (%s) FAILED (%s)\n",
603 per->mlfile->filename, per->this_url,
604 curl_easy_strerror(result));
605 }
606 }
607
608 if((global->progressmode == CURL_PROGRESS_BAR) &&
609 per->progressbar.calls)
610 /* if the custom progress bar has been displayed, we output a
611 newline here */
612 fputs("\n", per->progressbar.out);
613
614 if(config->writeout)
615 ourWriteOut(per->curl, &per->outs, config->writeout);
616
617 /* Close the outs file */
618 if(outs->fopened && outs->stream) {
619 int rc = fclose(outs->stream);
620 if(!result && rc) {
621 /* something went wrong in the writing process */
622 result = CURLE_WRITE_ERROR;
623 fprintf(global->errors, "(%d) Failed writing body\n", result);
624 }
625 }
626
627 /* File time can only be set _after_ the file has been closed */
628 if(!result && config->remote_time && outs->s_isreg && outs->filename) {
629 /* Ask libcurl if we got a remote file time */
630 curl_off_t filetime = -1;
631 curl_easy_getinfo(curl, CURLINFO_FILETIME_T, &filetime);
632 setfiletime(filetime, outs->filename, config->global->errors);
633 }
634
635 /* Close function-local opened file descriptors */
636 if(per->heads.fopened && per->heads.stream)
637 fclose(per->heads.stream);
638
639 if(per->heads.alloc_filename)
640 Curl_safefree(per->heads.filename);
641
642 curl_easy_cleanup(per->curl);
643 if(outs->alloc_filename)
644 free(outs->filename);
645 free(per->this_url);
646 free(per->separator_err);
647 free(per->separator);
648 free(per->outfile);
649 free(per->uploadfile);
650
651 return CURLE_OK;
652 }
653
654 /* go through the list of URLs and configs and add transfers */
655
create_transfers(struct GlobalConfig * global,struct OperationConfig * config,CURLSH * share,bool capath_from_env)656 static CURLcode create_transfers(struct GlobalConfig *global,
657 struct OperationConfig *config,
658 CURLSH *share,
659 bool capath_from_env)
660 {
661 CURLcode result = CURLE_OK;
662 struct getout *urlnode;
663 metalinkfile *mlfile_last = NULL;
664 bool orig_noprogress = global->noprogress;
665 bool orig_isatty = global->isatty;
666 char *httpgetfields = NULL;
667
668 if(config->postfields) {
669 if(config->use_httpget) {
670 /* Use the postfields data for a http get */
671 httpgetfields = strdup(config->postfields);
672 Curl_safefree(config->postfields);
673 if(!httpgetfields) {
674 helpf(global->errors, "out of memory\n");
675 result = CURLE_OUT_OF_MEMORY;
676 goto quit_curl;
677 }
678 if(SetHTTPrequest(config,
679 (config->no_body?HTTPREQ_HEAD:HTTPREQ_GET),
680 &config->httpreq)) {
681 result = CURLE_FAILED_INIT;
682 goto quit_curl;
683 }
684 }
685 else {
686 if(SetHTTPrequest(config, HTTPREQ_SIMPLEPOST, &config->httpreq)) {
687 result = CURLE_FAILED_INIT;
688 goto quit_curl;
689 }
690 }
691 }
692
693 for(urlnode = config->url_list; urlnode; urlnode = urlnode->next) {
694 unsigned long li;
695 unsigned long up; /* upload file counter within a single upload glob */
696 char *infiles; /* might be a glob pattern */
697 char *outfiles;
698 unsigned long infilenum;
699 URLGlob *inglob;
700 bool metalink = FALSE; /* metalink download? */
701 metalinkfile *mlfile;
702 metalink_resource *mlres;
703
704 outfiles = NULL;
705 infilenum = 1;
706 inglob = NULL;
707
708 if(urlnode->flags & GETOUT_METALINK) {
709 metalink = 1;
710 if(mlfile_last == NULL) {
711 mlfile_last = config->metalinkfile_list;
712 }
713 mlfile = mlfile_last;
714 mlfile_last = mlfile_last->next;
715 mlres = mlfile->resource;
716 }
717 else {
718 mlfile = NULL;
719 mlres = NULL;
720 }
721
722 /* urlnode->url is the full URL (it might be NULL) */
723
724 if(!urlnode->url) {
725 /* This node has no URL. Free node data without destroying the
726 node itself nor modifying next pointer and continue to next */
727 Curl_safefree(urlnode->outfile);
728 Curl_safefree(urlnode->infile);
729 urlnode->flags = 0;
730 continue; /* next URL please */
731 }
732
733 /* save outfile pattern before expansion */
734 if(urlnode->outfile) {
735 outfiles = strdup(urlnode->outfile);
736 if(!outfiles) {
737 helpf(global->errors, "out of memory\n");
738 result = CURLE_OUT_OF_MEMORY;
739 break;
740 }
741 }
742
743 infiles = urlnode->infile;
744
745 if(!config->globoff && infiles) {
746 /* Unless explicitly shut off */
747 result = glob_url(&inglob, infiles, &infilenum,
748 global->showerror?global->errors:NULL);
749 if(result) {
750 Curl_safefree(outfiles);
751 break;
752 }
753 }
754
755 /* Here's the loop for uploading multiple files within the same
756 single globbed string. If no upload, we enter the loop once anyway. */
757 for(up = 0 ; up < infilenum; up++) {
758
759 char *uploadfile; /* a single file, never a glob */
760 int separator;
761 URLGlob *urls;
762 unsigned long urlnum;
763
764 uploadfile = NULL;
765 urls = NULL;
766 urlnum = 0;
767
768 if(!up && !infiles)
769 Curl_nop_stmt;
770 else {
771 if(inglob) {
772 result = glob_next_url(&uploadfile, inglob);
773 if(result == CURLE_OUT_OF_MEMORY)
774 helpf(global->errors, "out of memory\n");
775 }
776 else if(!up) {
777 uploadfile = strdup(infiles);
778 if(!uploadfile) {
779 helpf(global->errors, "out of memory\n");
780 result = CURLE_OUT_OF_MEMORY;
781 }
782 }
783 if(!uploadfile)
784 break;
785 }
786
787 if(metalink) {
788 /* For Metalink download, we don't use glob. Instead we use
789 the number of resources as urlnum. */
790 urlnum = count_next_metalink_resource(mlfile);
791 }
792 else if(!config->globoff) {
793 /* Unless explicitly shut off, we expand '{...}' and '[...]'
794 expressions and return total number of URLs in pattern set */
795 result = glob_url(&urls, urlnode->url, &urlnum,
796 global->showerror?global->errors:NULL);
797 if(result) {
798 Curl_safefree(uploadfile);
799 break;
800 }
801 }
802 else
803 urlnum = 1; /* without globbing, this is a single URL */
804
805 /* if multiple files extracted to stdout, insert separators! */
806 separator = ((!outfiles || !strcmp(outfiles, "-")) && urlnum > 1);
807
808 /* Here's looping around each globbed URL */
809 for(li = 0 ; li < urlnum; li++) {
810 struct per_transfer *per;
811 struct OutStruct *outs;
812 struct InStruct *input;
813 struct OutStruct *heads;
814 struct HdrCbData *hdrcbdata = NULL;
815 CURL *curl = curl_easy_init();
816
817 result = add_transfer(&per);
818 if(result || !curl) {
819 free(uploadfile);
820 curl_easy_cleanup(curl);
821 result = CURLE_OUT_OF_MEMORY;
822 goto show_error;
823 }
824 per->config = config;
825 per->curl = curl;
826 per->uploadfile = uploadfile;
827
828 /* default headers output stream is stdout */
829 heads = &per->heads;
830 heads->stream = stdout;
831 heads->config = config;
832
833 /* Single header file for all URLs */
834 if(config->headerfile) {
835 /* open file for output: */
836 if(strcmp(config->headerfile, "-")) {
837 FILE *newfile = fopen(config->headerfile, "wb");
838 if(!newfile) {
839 warnf(config->global, "Failed to open %s\n", config->headerfile);
840 result = CURLE_WRITE_ERROR;
841 goto quit_curl;
842 }
843 else {
844 heads->filename = config->headerfile;
845 heads->s_isreg = TRUE;
846 heads->fopened = TRUE;
847 heads->stream = newfile;
848 }
849 }
850 else {
851 /* always use binary mode for protocol header output */
852 set_binmode(heads->stream);
853 }
854 }
855
856
857 hdrcbdata = &per->hdrcbdata;
858
859 outs = &per->outs;
860 input = &per->input;
861
862 per->outfile = NULL;
863 per->infdopen = FALSE;
864 per->infd = STDIN_FILENO;
865
866 /* default output stream is stdout */
867 outs->stream = stdout;
868 outs->config = config;
869
870 if(metalink) {
871 /* For Metalink download, use name in Metalink file as
872 filename. */
873 per->outfile = strdup(mlfile->filename);
874 if(!per->outfile) {
875 result = CURLE_OUT_OF_MEMORY;
876 goto show_error;
877 }
878 per->this_url = strdup(mlres->url);
879 if(!per->this_url) {
880 result = CURLE_OUT_OF_MEMORY;
881 goto show_error;
882 }
883 per->mlfile = mlfile;
884 }
885 else {
886 if(urls) {
887 result = glob_next_url(&per->this_url, urls);
888 if(result)
889 goto show_error;
890 }
891 else if(!li) {
892 per->this_url = strdup(urlnode->url);
893 if(!per->this_url) {
894 result = CURLE_OUT_OF_MEMORY;
895 goto show_error;
896 }
897 }
898 else
899 per->this_url = NULL;
900 if(!per->this_url)
901 break;
902
903 if(outfiles) {
904 per->outfile = strdup(outfiles);
905 if(!per->outfile) {
906 result = CURLE_OUT_OF_MEMORY;
907 goto show_error;
908 }
909 }
910 }
911
912 if(((urlnode->flags&GETOUT_USEREMOTE) ||
913 (per->outfile && strcmp("-", per->outfile))) &&
914 (metalink || !config->use_metalink)) {
915
916 /*
917 * We have specified a file name to store the result in, or we have
918 * decided we want to use the remote file name.
919 */
920
921 if(!per->outfile) {
922 /* extract the file name from the URL */
923 result = get_url_file_name(&per->outfile, per->this_url);
924 if(result)
925 goto show_error;
926 if(!*per->outfile && !config->content_disposition) {
927 helpf(global->errors, "Remote file name has no length!\n");
928 result = CURLE_WRITE_ERROR;
929 goto quit_urls;
930 }
931 }
932 else if(urls) {
933 /* fill '#1' ... '#9' terms from URL pattern */
934 char *storefile = per->outfile;
935 result = glob_match_url(&per->outfile, storefile, urls);
936 Curl_safefree(storefile);
937 if(result) {
938 /* bad globbing */
939 warnf(config->global, "bad output glob!\n");
940 goto quit_urls;
941 }
942 }
943
944 /* Create the directory hierarchy, if not pre-existent to a multiple
945 file output call */
946
947 if(config->create_dirs || metalink) {
948 result = create_dir_hierarchy(per->outfile, global->errors);
949 /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
950 if(result == CURLE_WRITE_ERROR)
951 goto quit_urls;
952 if(result) {
953 goto show_error;
954 }
955 }
956
957 if((urlnode->flags & GETOUT_USEREMOTE)
958 && config->content_disposition) {
959 /* Our header callback MIGHT set the filename */
960 DEBUGASSERT(!outs->filename);
961 }
962
963 if(config->resume_from_current) {
964 /* We're told to continue from where we are now. Get the size
965 of the file as it is now and open it for append instead */
966 struct_stat fileinfo;
967 /* VMS -- Danger, the filesize is only valid for stream files */
968 if(0 == stat(per->outfile, &fileinfo))
969 /* set offset to current file size: */
970 config->resume_from = fileinfo.st_size;
971 else
972 /* let offset be 0 */
973 config->resume_from = 0;
974 }
975
976 if(config->resume_from) {
977 #ifdef __VMS
978 /* open file for output, forcing VMS output format into stream
979 mode which is needed for stat() call above to always work. */
980 FILE *file = fopen(outfile, config->resume_from?"ab":"wb",
981 "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0");
982 #else
983 /* open file for output: */
984 FILE *file = fopen(per->outfile, config->resume_from?"ab":"wb");
985 #endif
986 if(!file) {
987 helpf(global->errors, "Can't open '%s'!\n", per->outfile);
988 result = CURLE_WRITE_ERROR;
989 goto quit_urls;
990 }
991 outs->fopened = TRUE;
992 outs->stream = file;
993 outs->init = config->resume_from;
994 }
995 else {
996 outs->stream = NULL; /* open when needed */
997 }
998 outs->filename = per->outfile;
999 outs->s_isreg = TRUE;
1000 }
1001
1002 if(per->uploadfile && !stdin_upload(per->uploadfile)) {
1003 /*
1004 * We have specified a file to upload and it isn't "-".
1005 */
1006 char *nurl = add_file_name_to_url(per->this_url, per->uploadfile);
1007 if(!nurl) {
1008 result = CURLE_OUT_OF_MEMORY;
1009 goto show_error;
1010 }
1011 per->this_url = nurl;
1012 }
1013 else if(per->uploadfile && stdin_upload(per->uploadfile)) {
1014 /* count to see if there are more than one auth bit set
1015 in the authtype field */
1016 int authbits = 0;
1017 int bitcheck = 0;
1018 while(bitcheck < 32) {
1019 if(config->authtype & (1UL << bitcheck++)) {
1020 authbits++;
1021 if(authbits > 1) {
1022 /* more than one, we're done! */
1023 break;
1024 }
1025 }
1026 }
1027
1028 /*
1029 * If the user has also selected --anyauth or --proxy-anyauth
1030 * we should warn him/her.
1031 */
1032 if(config->proxyanyauth || (authbits>1)) {
1033 warnf(config->global,
1034 "Using --anyauth or --proxy-anyauth with upload from stdin"
1035 " involves a big risk of it not working. Use a temporary"
1036 " file or a fixed auth type instead!\n");
1037 }
1038
1039 DEBUGASSERT(per->infdopen == FALSE);
1040 DEBUGASSERT(per->infd == STDIN_FILENO);
1041
1042 set_binmode(stdin);
1043 if(!strcmp(per->uploadfile, ".")) {
1044 if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0)
1045 warnf(config->global,
1046 "fcntl failed on fd=%d: %s\n", per->infd, strerror(errno));
1047 }
1048 }
1049
1050 if(per->uploadfile && config->resume_from_current)
1051 config->resume_from = -1; /* -1 will then force get-it-yourself */
1052
1053 if(output_expected(per->this_url, per->uploadfile) && outs->stream &&
1054 isatty(fileno(outs->stream)))
1055 /* we send the output to a tty, therefore we switch off the progress
1056 meter */
1057 global->noprogress = global->isatty = TRUE;
1058 else {
1059 /* progress meter is per download, so restore config
1060 values */
1061 global->noprogress = orig_noprogress;
1062 global->isatty = orig_isatty;
1063 }
1064
1065 if(urlnum > 1 && !global->mute) {
1066 per->separator_err =
1067 aprintf("\n[%lu/%lu]: %s --> %s",
1068 li + 1, urlnum, per->this_url,
1069 per->outfile ? per->outfile : "<stdout>");
1070 if(separator)
1071 per->separator = aprintf("%s%s", CURLseparator, per->this_url);
1072 }
1073 if(httpgetfields) {
1074 char *urlbuffer;
1075 /* Find out whether the url contains a file name */
1076 const char *pc = strstr(per->this_url, "://");
1077 char sep = '?';
1078 if(pc)
1079 pc += 3;
1080 else
1081 pc = per->this_url;
1082
1083 pc = strrchr(pc, '/'); /* check for a slash */
1084
1085 if(pc) {
1086 /* there is a slash present in the URL */
1087
1088 if(strchr(pc, '?'))
1089 /* Ouch, there's already a question mark in the URL string, we
1090 then append the data with an ampersand separator instead! */
1091 sep = '&';
1092 }
1093 /*
1094 * Then append ? followed by the get fields to the url.
1095 */
1096 if(pc)
1097 urlbuffer = aprintf("%s%c%s", per->this_url, sep, httpgetfields);
1098 else
1099 /* Append / before the ? to create a well-formed url
1100 if the url contains a hostname only
1101 */
1102 urlbuffer = aprintf("%s/?%s", per->this_url, httpgetfields);
1103
1104 if(!urlbuffer) {
1105 result = CURLE_OUT_OF_MEMORY;
1106 goto show_error;
1107 }
1108
1109 Curl_safefree(per->this_url); /* free previous URL */
1110 per->this_url = urlbuffer; /* use our new URL instead! */
1111 }
1112
1113 if(!global->errors)
1114 global->errors = stderr;
1115
1116 if((!per->outfile || !strcmp(per->outfile, "-")) &&
1117 !config->use_ascii) {
1118 /* We get the output to stdout and we have not got the ASCII/text
1119 flag, then set stdout to be binary */
1120 set_binmode(stdout);
1121 }
1122
1123 /* explicitly passed to stdout means okaying binary gunk */
1124 config->terminal_binary_ok =
1125 (per->outfile && !strcmp(per->outfile, "-"));
1126
1127 /* avoid having this setopt added to the --libcurl source
1128 output */
1129 result = curl_easy_setopt(curl, CURLOPT_SHARE, share);
1130 if(result)
1131 goto show_error;
1132
1133 if(!config->tcp_nodelay)
1134 my_setopt(curl, CURLOPT_TCP_NODELAY, 0L);
1135
1136 if(config->tcp_fastopen)
1137 my_setopt(curl, CURLOPT_TCP_FASTOPEN, 1L);
1138
1139 /* where to store */
1140 my_setopt(curl, CURLOPT_WRITEDATA, per);
1141 my_setopt(curl, CURLOPT_INTERLEAVEDATA, per);
1142
1143 if(metalink || !config->use_metalink)
1144 /* what call to write */
1145 my_setopt(curl, CURLOPT_WRITEFUNCTION, tool_write_cb);
1146 #ifdef USE_METALINK
1147 else
1148 /* Set Metalink specific write callback function to parse
1149 XML data progressively. */
1150 my_setopt(curl, CURLOPT_WRITEFUNCTION, metalink_write_cb);
1151 #endif /* USE_METALINK */
1152
1153 /* for uploads */
1154 input->config = config;
1155 /* Note that if CURLOPT_READFUNCTION is fread (the default), then
1156 * lib/telnet.c will Curl_poll() on the input file descriptor
1157 * rather then calling the READFUNCTION at regular intervals.
1158 * The circumstances in which it is preferable to enable this
1159 * behaviour, by omitting to set the READFUNCTION & READDATA options,
1160 * have not been determined.
1161 */
1162 my_setopt(curl, CURLOPT_READDATA, input);
1163 /* what call to read */
1164 my_setopt(curl, CURLOPT_READFUNCTION, tool_read_cb);
1165
1166 /* in 7.18.0, the CURLOPT_SEEKFUNCTION/DATA pair is taking over what
1167 CURLOPT_IOCTLFUNCTION/DATA pair previously provided for seeking */
1168 my_setopt(curl, CURLOPT_SEEKDATA, input);
1169 my_setopt(curl, CURLOPT_SEEKFUNCTION, tool_seek_cb);
1170
1171 if(config->recvpersecond &&
1172 (config->recvpersecond < BUFFER_SIZE))
1173 /* use a smaller sized buffer for better sleeps */
1174 my_setopt(curl, CURLOPT_BUFFERSIZE, (long)config->recvpersecond);
1175 else
1176 my_setopt(curl, CURLOPT_BUFFERSIZE, (long)BUFFER_SIZE);
1177
1178 my_setopt_str(curl, CURLOPT_URL, per->this_url);
1179 my_setopt(curl, CURLOPT_NOPROGRESS, global->noprogress?1L:0L);
1180 if(config->no_body)
1181 my_setopt(curl, CURLOPT_NOBODY, 1L);
1182
1183 if(config->oauth_bearer)
1184 my_setopt_str(curl, CURLOPT_XOAUTH2_BEARER, config->oauth_bearer);
1185
1186 {
1187 my_setopt_str(curl, CURLOPT_PROXY, config->proxy);
1188 /* new in libcurl 7.5 */
1189 if(config->proxy)
1190 my_setopt_enum(curl, CURLOPT_PROXYTYPE, config->proxyver);
1191
1192 my_setopt_str(curl, CURLOPT_PROXYUSERPWD, config->proxyuserpwd);
1193
1194 /* new in libcurl 7.3 */
1195 my_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, config->proxytunnel?1L:0L);
1196
1197 /* new in libcurl 7.52.0 */
1198 if(config->preproxy)
1199 my_setopt_str(curl, CURLOPT_PRE_PROXY, config->preproxy);
1200
1201 /* new in libcurl 7.10.6 */
1202 if(config->proxyanyauth)
1203 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1204 (long)CURLAUTH_ANY);
1205 else if(config->proxynegotiate)
1206 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1207 (long)CURLAUTH_GSSNEGOTIATE);
1208 else if(config->proxyntlm)
1209 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1210 (long)CURLAUTH_NTLM);
1211 else if(config->proxydigest)
1212 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1213 (long)CURLAUTH_DIGEST);
1214 else if(config->proxybasic)
1215 my_setopt_bitmask(curl, CURLOPT_PROXYAUTH,
1216 (long)CURLAUTH_BASIC);
1217
1218 /* new in libcurl 7.19.4 */
1219 my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy);
1220
1221 my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS,
1222 config->suppress_connect_headers?1L:0L);
1223 }
1224
1225 my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L);
1226 my_setopt(curl, CURLOPT_REQUEST_TARGET, config->request_target);
1227 my_setopt(curl, CURLOPT_UPLOAD, per->uploadfile?1L:0L);
1228 my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly?1L:0L);
1229 my_setopt(curl, CURLOPT_APPEND, config->ftp_append?1L:0L);
1230
1231 if(config->netrc_opt)
1232 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_OPTIONAL);
1233 else if(config->netrc || config->netrc_file)
1234 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_REQUIRED);
1235 else
1236 my_setopt_enum(curl, CURLOPT_NETRC, (long)CURL_NETRC_IGNORED);
1237
1238 if(config->netrc_file)
1239 my_setopt_str(curl, CURLOPT_NETRC_FILE, config->netrc_file);
1240
1241 my_setopt(curl, CURLOPT_TRANSFERTEXT, config->use_ascii?1L:0L);
1242 if(config->login_options)
1243 my_setopt_str(curl, CURLOPT_LOGIN_OPTIONS, config->login_options);
1244 my_setopt_str(curl, CURLOPT_USERPWD, config->userpwd);
1245 my_setopt_str(curl, CURLOPT_RANGE, config->range);
1246 my_setopt(curl, CURLOPT_ERRORBUFFER, per->errorbuffer);
1247 my_setopt(curl, CURLOPT_TIMEOUT_MS, (long)(config->timeout * 1000));
1248
1249 switch(config->httpreq) {
1250 case HTTPREQ_SIMPLEPOST:
1251 my_setopt_str(curl, CURLOPT_POSTFIELDS,
1252 config->postfields);
1253 my_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE,
1254 config->postfieldsize);
1255 break;
1256 case HTTPREQ_MIMEPOST:
1257 result = tool2curlmime(curl, config->mimeroot, &config->mimepost);
1258 if(result)
1259 goto show_error;
1260 my_setopt_mimepost(curl, CURLOPT_MIMEPOST, config->mimepost);
1261 break;
1262 default:
1263 break;
1264 }
1265
1266 /* new in libcurl 7.10.6 (default is Basic) */
1267 if(config->authtype)
1268 my_setopt_bitmask(curl, CURLOPT_HTTPAUTH, (long)config->authtype);
1269
1270 my_setopt_slist(curl, CURLOPT_HTTPHEADER, config->headers);
1271
1272 if(built_in_protos & (CURLPROTO_HTTP | CURLPROTO_RTSP)) {
1273 my_setopt_str(curl, CURLOPT_REFERER, config->referer);
1274 my_setopt_str(curl, CURLOPT_USERAGENT, config->useragent);
1275 }
1276
1277 if(built_in_protos & CURLPROTO_HTTP) {
1278
1279 long postRedir = 0;
1280
1281 my_setopt(curl, CURLOPT_FOLLOWLOCATION,
1282 config->followlocation?1L:0L);
1283 my_setopt(curl, CURLOPT_UNRESTRICTED_AUTH,
1284 config->unrestricted_auth?1L:0L);
1285
1286 my_setopt(curl, CURLOPT_AUTOREFERER, config->autoreferer?1L:0L);
1287
1288 /* new in libcurl 7.36.0 */
1289 if(config->proxyheaders) {
1290 my_setopt_slist(curl, CURLOPT_PROXYHEADER, config->proxyheaders);
1291 my_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE);
1292 }
1293
1294 /* new in libcurl 7.5 */
1295 my_setopt(curl, CURLOPT_MAXREDIRS, config->maxredirs);
1296
1297 if(config->httpversion)
1298 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, config->httpversion);
1299 else if(curlinfo->features & CURL_VERSION_HTTP2) {
1300 my_setopt_enum(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
1301 }
1302
1303 /* curl 7.19.1 (the 301 version existed in 7.18.2),
1304 303 was added in 7.26.0 */
1305 if(config->post301)
1306 postRedir |= CURL_REDIR_POST_301;
1307 if(config->post302)
1308 postRedir |= CURL_REDIR_POST_302;
1309 if(config->post303)
1310 postRedir |= CURL_REDIR_POST_303;
1311 my_setopt(curl, CURLOPT_POSTREDIR, postRedir);
1312
1313 /* new in libcurl 7.21.6 */
1314 if(config->encoding)
1315 my_setopt_str(curl, CURLOPT_ACCEPT_ENCODING, "");
1316
1317 /* new in libcurl 7.21.6 */
1318 if(config->tr_encoding)
1319 my_setopt(curl, CURLOPT_TRANSFER_ENCODING, 1L);
1320 /* new in libcurl 7.64.0 */
1321 my_setopt(curl, CURLOPT_HTTP09_ALLOWED,
1322 config->http09_allowed ? 1L : 0L);
1323
1324 } /* (built_in_protos & CURLPROTO_HTTP) */
1325
1326 my_setopt_str(curl, CURLOPT_FTPPORT, config->ftpport);
1327 my_setopt(curl, CURLOPT_LOW_SPEED_LIMIT,
1328 config->low_speed_limit);
1329 my_setopt(curl, CURLOPT_LOW_SPEED_TIME, config->low_speed_time);
1330 my_setopt(curl, CURLOPT_MAX_SEND_SPEED_LARGE,
1331 config->sendpersecond);
1332 my_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE,
1333 config->recvpersecond);
1334
1335 if(config->use_resume)
1336 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from);
1337 else
1338 my_setopt(curl, CURLOPT_RESUME_FROM_LARGE, CURL_OFF_T_C(0));
1339
1340 my_setopt_str(curl, CURLOPT_KEYPASSWD, config->key_passwd);
1341 my_setopt_str(curl, CURLOPT_PROXY_KEYPASSWD, config->proxy_key_passwd);
1342
1343 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
1344
1345 /* SSH and SSL private key uses same command-line option */
1346 /* new in libcurl 7.16.1 */
1347 my_setopt_str(curl, CURLOPT_SSH_PRIVATE_KEYFILE, config->key);
1348 /* new in libcurl 7.16.1 */
1349 my_setopt_str(curl, CURLOPT_SSH_PUBLIC_KEYFILE, config->pubkey);
1350
1351 /* new in libcurl 7.17.1: SSH host key md5 checking allows us
1352 to fail if we are not talking to who we think we should */
1353 my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5,
1354 config->hostpubmd5);
1355
1356 /* new in libcurl 7.56.0 */
1357 if(config->ssh_compression)
1358 my_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L);
1359 }
1360
1361 if(config->cacert)
1362 my_setopt_str(curl, CURLOPT_CAINFO, config->cacert);
1363 if(config->proxy_cacert)
1364 my_setopt_str(curl, CURLOPT_PROXY_CAINFO, config->proxy_cacert);
1365
1366 if(config->capath) {
1367 result = res_setopt_str(curl, CURLOPT_CAPATH, config->capath);
1368 if(result == CURLE_NOT_BUILT_IN) {
1369 warnf(config->global, "ignoring %s, not supported by libcurl\n",
1370 capath_from_env?
1371 "SSL_CERT_DIR environment variable":"--capath");
1372 }
1373 else if(result)
1374 goto show_error;
1375 }
1376 /* For the time being if --proxy-capath is not set then we use the
1377 --capath value for it, if any. See #1257 */
1378 if((config->proxy_capath || config->capath) &&
1379 !tool_setopt_skip(CURLOPT_PROXY_CAPATH)) {
1380 result = res_setopt_str(curl, CURLOPT_PROXY_CAPATH,
1381 (config->proxy_capath ?
1382 config->proxy_capath :
1383 config->capath));
1384 if(result == CURLE_NOT_BUILT_IN) {
1385 if(config->proxy_capath) {
1386 warnf(config->global,
1387 "ignoring --proxy-capath, not supported by libcurl\n");
1388 }
1389 }
1390 else if(result)
1391 goto show_error;
1392 }
1393
1394 if(config->crlfile)
1395 my_setopt_str(curl, CURLOPT_CRLFILE, config->crlfile);
1396 if(config->proxy_crlfile)
1397 my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->proxy_crlfile);
1398 else if(config->crlfile) /* CURLOPT_PROXY_CRLFILE default is crlfile */
1399 my_setopt_str(curl, CURLOPT_PROXY_CRLFILE, config->crlfile);
1400
1401 if(config->pinnedpubkey)
1402 my_setopt_str(curl, CURLOPT_PINNEDPUBLICKEY, config->pinnedpubkey);
1403
1404 if(curlinfo->features & CURL_VERSION_SSL) {
1405 /* Check if config->cert is a PKCS#11 URI and set the
1406 * config->cert_type if necessary */
1407 if(config->cert) {
1408 if(!config->cert_type) {
1409 if(is_pkcs11_uri(config->cert)) {
1410 config->cert_type = strdup("ENG");
1411 }
1412 }
1413 }
1414
1415 /* Check if config->key is a PKCS#11 URI and set the
1416 * config->key_type if necessary */
1417 if(config->key) {
1418 if(!config->key_type) {
1419 if(is_pkcs11_uri(config->key)) {
1420 config->key_type = strdup("ENG");
1421 }
1422 }
1423 }
1424
1425 /* Check if config->proxy_cert is a PKCS#11 URI and set the
1426 * config->proxy_type if necessary */
1427 if(config->proxy_cert) {
1428 if(!config->proxy_cert_type) {
1429 if(is_pkcs11_uri(config->proxy_cert)) {
1430 config->proxy_cert_type = strdup("ENG");
1431 }
1432 }
1433 }
1434
1435 /* Check if config->proxy_key is a PKCS#11 URI and set the
1436 * config->proxy_key_type if necessary */
1437 if(config->proxy_key) {
1438 if(!config->proxy_key_type) {
1439 if(is_pkcs11_uri(config->proxy_key)) {
1440 config->proxy_key_type = strdup("ENG");
1441 }
1442 }
1443 }
1444
1445 my_setopt_str(curl, CURLOPT_SSLCERT, config->cert);
1446 my_setopt_str(curl, CURLOPT_PROXY_SSLCERT, config->proxy_cert);
1447 my_setopt_str(curl, CURLOPT_SSLCERTTYPE, config->cert_type);
1448 my_setopt_str(curl, CURLOPT_PROXY_SSLCERTTYPE,
1449 config->proxy_cert_type);
1450 my_setopt_str(curl, CURLOPT_SSLKEY, config->key);
1451 my_setopt_str(curl, CURLOPT_PROXY_SSLKEY, config->proxy_key);
1452 my_setopt_str(curl, CURLOPT_SSLKEYTYPE, config->key_type);
1453 my_setopt_str(curl, CURLOPT_PROXY_SSLKEYTYPE,
1454 config->proxy_key_type);
1455
1456 if(config->insecure_ok) {
1457 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
1458 my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
1459 }
1460 else {
1461 my_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
1462 /* libcurl default is strict verifyhost -> 2L */
1463 /* my_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); */
1464 }
1465 if(config->proxy_insecure_ok) {
1466 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYPEER, 0L);
1467 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYHOST, 0L);
1468 }
1469 else {
1470 my_setopt(curl, CURLOPT_PROXY_SSL_VERIFYPEER, 1L);
1471 }
1472
1473 if(config->verifystatus)
1474 my_setopt(curl, CURLOPT_SSL_VERIFYSTATUS, 1L);
1475
1476 if(config->falsestart)
1477 my_setopt(curl, CURLOPT_SSL_FALSESTART, 1L);
1478
1479 my_setopt_enum(curl, CURLOPT_SSLVERSION,
1480 config->ssl_version | config->ssl_version_max);
1481 my_setopt_enum(curl, CURLOPT_PROXY_SSLVERSION,
1482 config->proxy_ssl_version);
1483 }
1484 if(config->path_as_is)
1485 my_setopt(curl, CURLOPT_PATH_AS_IS, 1L);
1486
1487 if(built_in_protos & (CURLPROTO_SCP|CURLPROTO_SFTP)) {
1488 if(!config->insecure_ok) {
1489 char *home;
1490 char *file;
1491 result = CURLE_OUT_OF_MEMORY;
1492 home = homedir();
1493 if(home) {
1494 file = aprintf("%s/.ssh/known_hosts", home);
1495 if(file) {
1496 /* new in curl 7.19.6 */
1497 result = res_setopt_str(curl, CURLOPT_SSH_KNOWNHOSTS, file);
1498 curl_free(file);
1499 if(result == CURLE_UNKNOWN_OPTION)
1500 /* libssh2 version older than 1.1.1 */
1501 result = CURLE_OK;
1502 }
1503 Curl_safefree(home);
1504 }
1505 if(result)
1506 goto show_error;
1507 }
1508 }
1509
1510 if(config->no_body || config->remote_time) {
1511 /* no body or use remote time */
1512 my_setopt(curl, CURLOPT_FILETIME, 1L);
1513 }
1514
1515 my_setopt(curl, CURLOPT_CRLF, config->crlf?1L:0L);
1516 my_setopt_slist(curl, CURLOPT_QUOTE, config->quote);
1517 my_setopt_slist(curl, CURLOPT_POSTQUOTE, config->postquote);
1518 my_setopt_slist(curl, CURLOPT_PREQUOTE, config->prequote);
1519
1520 if(config->cookie)
1521 my_setopt_str(curl, CURLOPT_COOKIE, config->cookie);
1522
1523 if(config->cookiefile)
1524 my_setopt_str(curl, CURLOPT_COOKIEFILE, config->cookiefile);
1525
1526 /* new in libcurl 7.9 */
1527 if(config->cookiejar)
1528 my_setopt_str(curl, CURLOPT_COOKIEJAR, config->cookiejar);
1529
1530 /* new in libcurl 7.9.7 */
1531 my_setopt(curl, CURLOPT_COOKIESESSION, config->cookiesession?1L:0L);
1532
1533 my_setopt_enum(curl, CURLOPT_TIMECONDITION, (long)config->timecond);
1534 my_setopt(curl, CURLOPT_TIMEVALUE_LARGE, config->condtime);
1535 my_setopt_str(curl, CURLOPT_CUSTOMREQUEST, config->customrequest);
1536 customrequest_helper(config, config->httpreq, config->customrequest);
1537 my_setopt(curl, CURLOPT_STDERR, global->errors);
1538
1539 /* three new ones in libcurl 7.3: */
1540 my_setopt_str(curl, CURLOPT_INTERFACE, config->iface);
1541 my_setopt_str(curl, CURLOPT_KRBLEVEL, config->krblevel);
1542 progressbarinit(&per->progressbar, config);
1543
1544 if((global->progressmode == CURL_PROGRESS_BAR) &&
1545 !global->noprogress && !global->mute) {
1546 /* we want the alternative style, then we have to implement it
1547 ourselves! */
1548 my_setopt(curl, CURLOPT_XFERINFOFUNCTION, tool_progress_cb);
1549 my_setopt(curl, CURLOPT_XFERINFODATA, &per->progressbar);
1550 }
1551
1552 /* new in libcurl 7.24.0: */
1553 if(config->dns_servers)
1554 my_setopt_str(curl, CURLOPT_DNS_SERVERS, config->dns_servers);
1555
1556 /* new in libcurl 7.33.0: */
1557 if(config->dns_interface)
1558 my_setopt_str(curl, CURLOPT_DNS_INTERFACE, config->dns_interface);
1559 if(config->dns_ipv4_addr)
1560 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP4, config->dns_ipv4_addr);
1561 if(config->dns_ipv6_addr)
1562 my_setopt_str(curl, CURLOPT_DNS_LOCAL_IP6, config->dns_ipv6_addr);
1563
1564 /* new in libcurl 7.6.2: */
1565 my_setopt_slist(curl, CURLOPT_TELNETOPTIONS, config->telnet_options);
1566
1567 /* new in libcurl 7.7: */
1568 my_setopt_str(curl, CURLOPT_RANDOM_FILE, config->random_file);
1569 my_setopt_str(curl, CURLOPT_EGDSOCKET, config->egd_file);
1570 my_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
1571 (long)(config->connecttimeout * 1000));
1572
1573 if(config->doh_url)
1574 my_setopt_str(curl, CURLOPT_DOH_URL, config->doh_url);
1575
1576 if(config->cipher_list)
1577 my_setopt_str(curl, CURLOPT_SSL_CIPHER_LIST, config->cipher_list);
1578
1579 if(config->proxy_cipher_list)
1580 my_setopt_str(curl, CURLOPT_PROXY_SSL_CIPHER_LIST,
1581 config->proxy_cipher_list);
1582
1583 if(config->cipher13_list)
1584 my_setopt_str(curl, CURLOPT_TLS13_CIPHERS, config->cipher13_list);
1585
1586 if(config->proxy_cipher13_list)
1587 my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS,
1588 config->proxy_cipher13_list);
1589
1590 /* new in libcurl 7.9.2: */
1591 if(config->disable_epsv)
1592 /* disable it */
1593 my_setopt(curl, CURLOPT_FTP_USE_EPSV, 0L);
1594
1595 /* new in libcurl 7.10.5 */
1596 if(config->disable_eprt)
1597 /* disable it */
1598 my_setopt(curl, CURLOPT_FTP_USE_EPRT, 0L);
1599
1600 if(global->tracetype != TRACE_NONE) {
1601 my_setopt(curl, CURLOPT_DEBUGFUNCTION, tool_debug_cb);
1602 my_setopt(curl, CURLOPT_DEBUGDATA, config);
1603 my_setopt(curl, CURLOPT_VERBOSE, 1L);
1604 }
1605
1606 /* new in curl 7.9.3 */
1607 if(config->engine) {
1608 result = res_setopt_str(curl, CURLOPT_SSLENGINE, config->engine);
1609 if(result)
1610 goto show_error;
1611 }
1612
1613 /* new in curl 7.10.7, extended in 7.19.4. Modified to use
1614 CREATE_DIR_RETRY in 7.49.0 */
1615 my_setopt(curl, CURLOPT_FTP_CREATE_MISSING_DIRS,
1616 (long)(config->ftp_create_dirs?
1617 CURLFTP_CREATE_DIR_RETRY:
1618 CURLFTP_CREATE_DIR_NONE));
1619
1620 /* new in curl 7.10.8 */
1621 if(config->max_filesize)
1622 my_setopt(curl, CURLOPT_MAXFILESIZE_LARGE,
1623 config->max_filesize);
1624
1625 if(4 == config->ip_version)
1626 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
1627 else if(6 == config->ip_version)
1628 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
1629 else
1630 my_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER);
1631
1632 /* new in curl 7.15.5 */
1633 if(config->ftp_ssl_reqd)
1634 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_ALL);
1635
1636 /* new in curl 7.11.0 */
1637 else if(config->ftp_ssl)
1638 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_TRY);
1639
1640 /* new in curl 7.16.0 */
1641 else if(config->ftp_ssl_control)
1642 my_setopt_enum(curl, CURLOPT_USE_SSL, (long)CURLUSESSL_CONTROL);
1643
1644 /* new in curl 7.16.1 */
1645 if(config->ftp_ssl_ccc)
1646 my_setopt_enum(curl, CURLOPT_FTP_SSL_CCC,
1647 (long)config->ftp_ssl_ccc_mode);
1648
1649 /* new in curl 7.19.4 */
1650 if(config->socks5_gssapi_nec)
1651 my_setopt_str(curl, CURLOPT_SOCKS5_GSSAPI_NEC,
1652 config->socks5_gssapi_nec);
1653
1654 /* new in curl 7.55.0 */
1655 if(config->socks5_auth)
1656 my_setopt_bitmask(curl, CURLOPT_SOCKS5_AUTH,
1657 (long)config->socks5_auth);
1658
1659 /* new in curl 7.43.0 */
1660 if(config->proxy_service_name)
1661 my_setopt_str(curl, CURLOPT_PROXY_SERVICE_NAME,
1662 config->proxy_service_name);
1663
1664 /* new in curl 7.43.0 */
1665 if(config->service_name)
1666 my_setopt_str(curl, CURLOPT_SERVICE_NAME,
1667 config->service_name);
1668
1669 /* curl 7.13.0 */
1670 my_setopt_str(curl, CURLOPT_FTP_ACCOUNT, config->ftp_account);
1671 my_setopt(curl, CURLOPT_IGNORE_CONTENT_LENGTH, config->ignorecl?1L:0L);
1672
1673 /* curl 7.14.2 */
1674 my_setopt(curl, CURLOPT_FTP_SKIP_PASV_IP, config->ftp_skip_ip?1L:0L);
1675
1676 /* curl 7.15.1 */
1677 my_setopt(curl, CURLOPT_FTP_FILEMETHOD, (long)config->ftp_filemethod);
1678
1679 /* curl 7.15.2 */
1680 if(config->localport) {
1681 my_setopt(curl, CURLOPT_LOCALPORT, config->localport);
1682 my_setopt_str(curl, CURLOPT_LOCALPORTRANGE, config->localportrange);
1683 }
1684
1685 /* curl 7.15.5 */
1686 my_setopt_str(curl, CURLOPT_FTP_ALTERNATIVE_TO_USER,
1687 config->ftp_alternative_to_user);
1688
1689 /* curl 7.16.0 */
1690 if(config->disable_sessionid)
1691 /* disable it */
1692 my_setopt(curl, CURLOPT_SSL_SESSIONID_CACHE, 0L);
1693
1694 /* curl 7.16.2 */
1695 if(config->raw) {
1696 my_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 0L);
1697 my_setopt(curl, CURLOPT_HTTP_TRANSFER_DECODING, 0L);
1698 }
1699
1700 /* curl 7.17.1 */
1701 if(!config->nokeepalive) {
1702 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
1703 if(config->alivetime != 0) {
1704 my_setopt(curl, CURLOPT_TCP_KEEPIDLE, config->alivetime);
1705 my_setopt(curl, CURLOPT_TCP_KEEPINTVL, config->alivetime);
1706 }
1707 }
1708 else
1709 my_setopt(curl, CURLOPT_TCP_KEEPALIVE, 0L);
1710
1711 /* curl 7.20.0 */
1712 if(config->tftp_blksize)
1713 my_setopt(curl, CURLOPT_TFTP_BLKSIZE, config->tftp_blksize);
1714
1715 if(config->mail_from)
1716 my_setopt_str(curl, CURLOPT_MAIL_FROM, config->mail_from);
1717
1718 if(config->mail_rcpt)
1719 my_setopt_slist(curl, CURLOPT_MAIL_RCPT, config->mail_rcpt);
1720
1721 /* curl 7.20.x */
1722 if(config->ftp_pret)
1723 my_setopt(curl, CURLOPT_FTP_USE_PRET, 1L);
1724
1725 if(config->proto_present)
1726 my_setopt_flags(curl, CURLOPT_PROTOCOLS, config->proto);
1727 if(config->proto_redir_present)
1728 my_setopt_flags(curl, CURLOPT_REDIR_PROTOCOLS, config->proto_redir);
1729
1730 if(config->content_disposition
1731 && (urlnode->flags & GETOUT_USEREMOTE))
1732 hdrcbdata->honor_cd_filename = TRUE;
1733 else
1734 hdrcbdata->honor_cd_filename = FALSE;
1735
1736 hdrcbdata->outs = outs;
1737 hdrcbdata->heads = heads;
1738 hdrcbdata->global = global;
1739 hdrcbdata->config = config;
1740
1741 my_setopt(curl, CURLOPT_HEADERFUNCTION, tool_header_cb);
1742 my_setopt(curl, CURLOPT_HEADERDATA, per);
1743
1744 if(config->resolve)
1745 /* new in 7.21.3 */
1746 my_setopt_slist(curl, CURLOPT_RESOLVE, config->resolve);
1747
1748 if(config->connect_to)
1749 /* new in 7.49.0 */
1750 my_setopt_slist(curl, CURLOPT_CONNECT_TO, config->connect_to);
1751
1752 /* new in 7.21.4 */
1753 if(curlinfo->features & CURL_VERSION_TLSAUTH_SRP) {
1754 if(config->tls_username)
1755 my_setopt_str(curl, CURLOPT_TLSAUTH_USERNAME,
1756 config->tls_username);
1757 if(config->tls_password)
1758 my_setopt_str(curl, CURLOPT_TLSAUTH_PASSWORD,
1759 config->tls_password);
1760 if(config->tls_authtype)
1761 my_setopt_str(curl, CURLOPT_TLSAUTH_TYPE,
1762 config->tls_authtype);
1763 if(config->proxy_tls_username)
1764 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_USERNAME,
1765 config->proxy_tls_username);
1766 if(config->proxy_tls_password)
1767 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_PASSWORD,
1768 config->proxy_tls_password);
1769 if(config->proxy_tls_authtype)
1770 my_setopt_str(curl, CURLOPT_PROXY_TLSAUTH_TYPE,
1771 config->proxy_tls_authtype);
1772 }
1773
1774 /* new in 7.22.0 */
1775 if(config->gssapi_delegation)
1776 my_setopt_str(curl, CURLOPT_GSSAPI_DELEGATION,
1777 config->gssapi_delegation);
1778
1779 /* new in 7.25.0 and 7.44.0 */
1780 {
1781 long mask = (config->ssl_allow_beast ? CURLSSLOPT_ALLOW_BEAST : 0) |
1782 (config->ssl_no_revoke ? CURLSSLOPT_NO_REVOKE : 0);
1783 if(mask)
1784 my_setopt_bitmask(curl, CURLOPT_SSL_OPTIONS, mask);
1785 }
1786
1787 if(config->proxy_ssl_allow_beast)
1788 my_setopt(curl, CURLOPT_PROXY_SSL_OPTIONS,
1789 (long)CURLSSLOPT_ALLOW_BEAST);
1790
1791 if(config->mail_auth)
1792 my_setopt_str(curl, CURLOPT_MAIL_AUTH, config->mail_auth);
1793
1794 /* new in 7.66.0 */
1795 if(config->sasl_authzid)
1796 my_setopt_str(curl, CURLOPT_SASL_AUTHZID, config->sasl_authzid);
1797
1798 /* new in 7.31.0 */
1799 if(config->sasl_ir)
1800 my_setopt(curl, CURLOPT_SASL_IR, 1L);
1801
1802 if(config->nonpn) {
1803 my_setopt(curl, CURLOPT_SSL_ENABLE_NPN, 0L);
1804 }
1805
1806 if(config->noalpn) {
1807 my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L);
1808 }
1809
1810 /* new in 7.40.0, abstract support added in 7.53.0 */
1811 if(config->unix_socket_path) {
1812 if(config->abstract_unix_socket) {
1813 my_setopt_str(curl, CURLOPT_ABSTRACT_UNIX_SOCKET,
1814 config->unix_socket_path);
1815 }
1816 else {
1817 my_setopt_str(curl, CURLOPT_UNIX_SOCKET_PATH,
1818 config->unix_socket_path);
1819 }
1820 }
1821
1822 /* new in 7.45.0 */
1823 if(config->proto_default)
1824 my_setopt_str(curl, CURLOPT_DEFAULT_PROTOCOL, config->proto_default);
1825
1826 /* new in 7.47.0 */
1827 if(config->expect100timeout > 0)
1828 my_setopt_str(curl, CURLOPT_EXPECT_100_TIMEOUT_MS,
1829 (long)(config->expect100timeout*1000));
1830
1831 /* new in 7.48.0 */
1832 if(config->tftp_no_options)
1833 my_setopt(curl, CURLOPT_TFTP_NO_OPTIONS, 1L);
1834
1835 /* new in 7.59.0 */
1836 if(config->happy_eyeballs_timeout_ms != CURL_HET_DEFAULT)
1837 my_setopt(curl, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS,
1838 config->happy_eyeballs_timeout_ms);
1839
1840 /* new in 7.60.0 */
1841 if(config->haproxy_protocol)
1842 my_setopt(curl, CURLOPT_HAPROXYPROTOCOL, 1L);
1843
1844 if(config->disallow_username_in_url)
1845 my_setopt(curl, CURLOPT_DISALLOW_USERNAME_IN_URL, 1L);
1846
1847 #ifdef USE_ALTSVC
1848 /* only if explicitly enabled in configure */
1849 if(config->altsvc)
1850 my_setopt_str(curl, CURLOPT_ALTSVC, config->altsvc);
1851 #endif
1852
1853 #ifdef USE_METALINK
1854 if(!metalink && config->use_metalink) {
1855 outs->metalink_parser = metalink_parser_context_new();
1856 if(outs->metalink_parser == NULL) {
1857 result = CURLE_OUT_OF_MEMORY;
1858 goto show_error;
1859 }
1860 fprintf(config->global->errors,
1861 "Metalink: parsing (%s) metalink/XML...\n", per->this_url);
1862 }
1863 else if(metalink)
1864 fprintf(config->global->errors,
1865 "Metalink: fetching (%s) from (%s)...\n",
1866 mlfile->filename, per->this_url);
1867 #endif /* USE_METALINK */
1868
1869 per->metalink = metalink;
1870 /* initialize retry vars for loop below */
1871 per->retry_sleep_default = (config->retry_delay) ?
1872 config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */
1873 per->retry_numretries = config->req_retry;
1874 per->retry_sleep = per->retry_sleep_default; /* ms */
1875 per->retrystart = tvnow();
1876
1877 } /* loop to the next URL */
1878
1879 show_error:
1880 quit_urls:
1881
1882 if(urls) {
1883 /* Free list of remaining URLs */
1884 glob_cleanup(urls);
1885 urls = NULL;
1886 }
1887
1888 if(infilenum > 1) {
1889 /* when file globbing, exit loop upon critical error */
1890 if(is_fatal_error(result))
1891 break;
1892 }
1893 else if(result)
1894 /* when not file globbing, exit loop upon any error */
1895 break;
1896
1897 } /* loop to the next globbed upload file */
1898
1899 /* Free loop-local allocated memory */
1900
1901 Curl_safefree(outfiles);
1902
1903 if(inglob) {
1904 /* Free list of globbed upload files */
1905 glob_cleanup(inglob);
1906 inglob = NULL;
1907 }
1908
1909 /* Free this URL node data without destroying the
1910 the node itself nor modifying next pointer. */
1911 Curl_safefree(urlnode->url);
1912 Curl_safefree(urlnode->outfile);
1913 Curl_safefree(urlnode->infile);
1914 urlnode->flags = 0;
1915
1916 } /* for-loop through all URLs */
1917 quit_curl:
1918
1919 /* Free function-local referenced allocated memory */
1920 Curl_safefree(httpgetfields);
1921
1922 return result;
1923 }
1924
1925 static long all_added; /* number of easy handles currently added */
1926
1927 /*
1928 * add_parallel_transfers() sets 'morep' to TRUE if there are more transfers
1929 * to add even after this call returns. sets 'addedp' to TRUE if one or more
1930 * transfers were added.
1931 */
add_parallel_transfers(struct GlobalConfig * global,CURLM * multi,bool * morep,bool * addedp)1932 static int add_parallel_transfers(struct GlobalConfig *global,
1933 CURLM *multi,
1934 bool *morep,
1935 bool *addedp)
1936 {
1937 struct per_transfer *per;
1938 CURLcode result;
1939 CURLMcode mcode;
1940 *addedp = FALSE;
1941 *morep = FALSE;
1942 for(per = transfers; per && (all_added < global->parallel_max);
1943 per = per->next) {
1944 if(per->added)
1945 /* already added */
1946 continue;
1947
1948 result = pre_transfer(global, per);
1949 if(result)
1950 break;
1951
1952 (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per);
1953 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
1954 (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per);
1955
1956 mcode = curl_multi_add_handle(multi, per->curl);
1957 if(mcode)
1958 return CURLE_OUT_OF_MEMORY;
1959 per->added = TRUE;
1960 all_added++;
1961 *addedp = TRUE;
1962 }
1963 *morep = per ? TRUE : FALSE;
1964 return CURLE_OK;
1965 }
1966
parallel_transfers(struct GlobalConfig * global,CURLSH * share)1967 static CURLcode parallel_transfers(struct GlobalConfig *global,
1968 CURLSH *share)
1969 {
1970 CURLM *multi;
1971 bool done = FALSE;
1972 CURLMcode mcode = CURLM_OK;
1973 CURLcode result = CURLE_OK;
1974 int still_running = 1;
1975 struct timeval start = tvnow();
1976 bool more_transfers;
1977 bool added_transfers;
1978
1979 multi = curl_multi_init();
1980 if(!multi)
1981 return CURLE_OUT_OF_MEMORY;
1982
1983 result = add_parallel_transfers(global, multi,
1984 &more_transfers, &added_transfers);
1985 if(result)
1986 return result;
1987
1988 while(!done && !mcode && (still_running || more_transfers)) {
1989 mcode = curl_multi_poll(multi, NULL, 0, 1000, NULL);
1990 if(!mcode)
1991 mcode = curl_multi_perform(multi, &still_running);
1992
1993 progress_meter(global, &start, FALSE);
1994
1995 if(!mcode) {
1996 int rc;
1997 CURLMsg *msg;
1998 bool removed = FALSE;
1999 do {
2000 msg = curl_multi_info_read(multi, &rc);
2001 if(msg) {
2002 bool retry;
2003 struct per_transfer *ended;
2004 CURL *easy = msg->easy_handle;
2005 result = msg->data.result;
2006 curl_easy_getinfo(easy, CURLINFO_PRIVATE, (void *)&ended);
2007 curl_multi_remove_handle(multi, easy);
2008
2009 result = post_transfer(global, share, ended, result, &retry);
2010 if(retry)
2011 continue;
2012 progress_finalize(ended); /* before it goes away */
2013 all_added--; /* one fewer added */
2014 removed = TRUE;
2015 (void)del_transfer(ended);
2016 }
2017 } while(msg);
2018 if(removed) {
2019 /* one or more transfers completed, add more! */
2020 (void)add_parallel_transfers(global, multi, &more_transfers,
2021 &added_transfers);
2022 if(added_transfers)
2023 /* we added new ones, make sure the loop doesn't exit yet */
2024 still_running = 1;
2025 }
2026 }
2027 }
2028
2029 (void)progress_meter(global, &start, TRUE);
2030
2031 /* Make sure to return some kind of error if there was a multi problem */
2032 if(mcode) {
2033 result = (mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
2034 /* The other multi errors should never happen, so return
2035 something suitably generic */
2036 CURLE_BAD_FUNCTION_ARGUMENT;
2037 }
2038
2039 curl_multi_cleanup(multi);
2040
2041 return result;
2042 }
2043
serial_transfers(struct GlobalConfig * global,CURLSH * share)2044 static CURLcode serial_transfers(struct GlobalConfig *global,
2045 CURLSH *share)
2046 {
2047 CURLcode returncode = CURLE_OK;
2048 CURLcode result = CURLE_OK;
2049 struct per_transfer *per;
2050 for(per = transfers; per;) {
2051 bool retry;
2052 result = pre_transfer(global, per);
2053 if(result)
2054 break;
2055
2056 #ifndef CURL_DISABLE_LIBCURL_OPTION
2057 if(global->libcurl) {
2058 result = easysrc_perform();
2059 if(result)
2060 break;
2061 }
2062 #endif
2063 #ifdef CURLDEBUG
2064 if(global->test_event_based)
2065 result = curl_easy_perform_ev(per->curl);
2066 else
2067 #endif
2068 result = curl_easy_perform(per->curl);
2069
2070 /* store the result of the actual transfer */
2071 returncode = result;
2072
2073 result = post_transfer(global, share, per, result, &retry);
2074 if(retry)
2075 continue;
2076 per = del_transfer(per);
2077
2078 /* Bail out upon critical errors or --fail-early */
2079 if(result || is_fatal_error(returncode) ||
2080 (returncode && global->fail_early))
2081 break;
2082 }
2083 if(returncode)
2084 /* returncode errors have priority */
2085 result = returncode;
2086 return result;
2087 }
2088
operate_do(struct GlobalConfig * global,struct OperationConfig * config,CURLSH * share)2089 static CURLcode operate_do(struct GlobalConfig *global,
2090 struct OperationConfig *config,
2091 CURLSH *share)
2092 {
2093 CURLcode result = CURLE_OK;
2094 bool capath_from_env;
2095
2096 /* Check we have a url */
2097 if(!config->url_list || !config->url_list->url) {
2098 helpf(global->errors, "no URL specified!\n");
2099 return CURLE_FAILED_INIT;
2100 }
2101
2102 /* On WIN32 we can't set the path to curl-ca-bundle.crt
2103 * at compile time. So we look here for the file in two ways:
2104 * 1: look at the environment variable CURL_CA_BUNDLE for a path
2105 * 2: if #1 isn't found, use the windows API function SearchPath()
2106 * to find it along the app's path (includes app's dir and CWD)
2107 *
2108 * We support the environment variable thing for non-Windows platforms
2109 * too. Just for the sake of it.
2110 */
2111 capath_from_env = false;
2112 if(!config->cacert &&
2113 !config->capath &&
2114 !config->insecure_ok) {
2115 CURL *curltls = curl_easy_init();
2116 struct curl_tlssessioninfo *tls_backend_info = NULL;
2117
2118 /* With the addition of CAINFO support for Schannel, this search could find
2119 * a certificate bundle that was previously ignored. To maintain backward
2120 * compatibility, only perform this search if not using Schannel.
2121 */
2122 result = curl_easy_getinfo(curltls, CURLINFO_TLS_SSL_PTR,
2123 &tls_backend_info);
2124 if(result)
2125 return result;
2126
2127 /* Set the CA cert locations specified in the environment. For Windows if
2128 * no environment-specified filename is found then check for CA bundle
2129 * default filename curl-ca-bundle.crt in the user's PATH.
2130 *
2131 * If Schannel is the selected SSL backend then these locations are
2132 * ignored. We allow setting CA location for schannel only when explicitly
2133 * specified by the user via CURLOPT_CAINFO / --cacert.
2134 */
2135 if(tls_backend_info->backend != CURLSSLBACKEND_SCHANNEL) {
2136 char *env;
2137 env = curlx_getenv("CURL_CA_BUNDLE");
2138 if(env) {
2139 config->cacert = strdup(env);
2140 if(!config->cacert) {
2141 curl_free(env);
2142 helpf(global->errors, "out of memory\n");
2143 return CURLE_OUT_OF_MEMORY;
2144 }
2145 }
2146 else {
2147 env = curlx_getenv("SSL_CERT_DIR");
2148 if(env) {
2149 config->capath = strdup(env);
2150 if(!config->capath) {
2151 curl_free(env);
2152 helpf(global->errors, "out of memory\n");
2153 return CURLE_OUT_OF_MEMORY;
2154 }
2155 capath_from_env = true;
2156 }
2157 else {
2158 env = curlx_getenv("SSL_CERT_FILE");
2159 if(env) {
2160 config->cacert = strdup(env);
2161 if(!config->cacert) {
2162 curl_free(env);
2163 helpf(global->errors, "out of memory\n");
2164 return CURLE_OUT_OF_MEMORY;
2165 }
2166 }
2167 }
2168 }
2169
2170 if(env)
2171 curl_free(env);
2172 #ifdef WIN32
2173 else {
2174 result = FindWin32CACert(config, tls_backend_info->backend,
2175 "curl-ca-bundle.crt");
2176 }
2177 #endif
2178 }
2179 curl_easy_cleanup(curltls);
2180 }
2181
2182 if(!result)
2183 /* loop through the list of given URLs */
2184 result = create_transfers(global, config, share, capath_from_env);
2185
2186 return result;
2187 }
2188
operate_transfers(struct GlobalConfig * global,CURLSH * share,CURLcode result)2189 static CURLcode operate_transfers(struct GlobalConfig *global,
2190 CURLSH *share,
2191 CURLcode result)
2192 {
2193 /* Save the values of noprogress and isatty to restore them later on */
2194 bool orig_noprogress = global->noprogress;
2195 bool orig_isatty = global->isatty;
2196 struct per_transfer *per;
2197
2198 /* Time to actually do the transfers */
2199 if(!result) {
2200 if(global->parallel)
2201 result = parallel_transfers(global, share);
2202 else
2203 result = serial_transfers(global, share);
2204 }
2205
2206 /* cleanup if there are any left */
2207 for(per = transfers; per;) {
2208 bool retry;
2209 (void)post_transfer(global, share, per, result, &retry);
2210 /* Free list of given URLs */
2211 clean_getout(per->config);
2212
2213 /* Release metalink related resources here */
2214 clean_metalink(per->config);
2215 per = del_transfer(per);
2216 }
2217
2218 /* Reset the global config variables */
2219 global->noprogress = orig_noprogress;
2220 global->isatty = orig_isatty;
2221
2222
2223 return result;
2224 }
2225
operate(struct GlobalConfig * config,int argc,argv_item_t argv[])2226 CURLcode operate(struct GlobalConfig *config, int argc, argv_item_t argv[])
2227 {
2228 CURLcode result = CURLE_OK;
2229
2230 /* Setup proper locale from environment */
2231 #ifdef HAVE_SETLOCALE
2232 setlocale(LC_ALL, "");
2233 #endif
2234
2235 /* Parse .curlrc if necessary */
2236 if((argc == 1) ||
2237 (!curl_strequal(argv[1], "-q") &&
2238 !curl_strequal(argv[1], "--disable"))) {
2239 parseconfig(NULL, config); /* ignore possible failure */
2240
2241 /* If we had no arguments then make sure a url was specified in .curlrc */
2242 if((argc < 2) && (!config->first->url_list)) {
2243 helpf(config->errors, NULL);
2244 result = CURLE_FAILED_INIT;
2245 }
2246 }
2247
2248 if(!result) {
2249 /* Parse the command line arguments */
2250 ParameterError res = parse_args(config, argc, argv);
2251 if(res) {
2252 result = CURLE_OK;
2253
2254 /* Check if we were asked for the help */
2255 if(res == PARAM_HELP_REQUESTED)
2256 tool_help();
2257 /* Check if we were asked for the manual */
2258 else if(res == PARAM_MANUAL_REQUESTED)
2259 hugehelp();
2260 /* Check if we were asked for the version information */
2261 else if(res == PARAM_VERSION_INFO_REQUESTED)
2262 tool_version_info();
2263 /* Check if we were asked to list the SSL engines */
2264 else if(res == PARAM_ENGINES_REQUESTED)
2265 tool_list_engines();
2266 else if(res == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL)
2267 result = CURLE_UNSUPPORTED_PROTOCOL;
2268 else
2269 result = CURLE_FAILED_INIT;
2270 }
2271 else {
2272 #ifndef CURL_DISABLE_LIBCURL_OPTION
2273 if(config->libcurl) {
2274 /* Initialise the libcurl source output */
2275 result = easysrc_init();
2276 }
2277 #endif
2278
2279 /* Perform the main operations */
2280 if(!result) {
2281 size_t count = 0;
2282 struct OperationConfig *operation = config->first;
2283 CURLSH *share = curl_share_init();
2284 if(!share) {
2285 #ifndef CURL_DISABLE_LIBCURL_OPTION
2286 if(config->libcurl) {
2287 /* Cleanup the libcurl source output */
2288 easysrc_cleanup();
2289 }
2290 #endif
2291 return CURLE_OUT_OF_MEMORY;
2292 }
2293
2294 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
2295 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
2296 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
2297 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT);
2298 curl_share_setopt(share, CURLSHOPT_SHARE, CURL_LOCK_DATA_PSL);
2299
2300 /* Get the required arguments for each operation */
2301 do {
2302 result = get_args(operation, count++);
2303
2304 operation = operation->next;
2305 } while(!result && operation);
2306
2307 /* Set the current operation pointer */
2308 config->current = config->first;
2309
2310 /* Setup all transfers */
2311 while(!result && config->current) {
2312 result = operate_do(config, config->current, share);
2313 config->current = config->current->next;
2314 }
2315
2316 /* now run! */
2317 result = operate_transfers(config, share, result);
2318
2319 curl_share_cleanup(share);
2320 #ifndef CURL_DISABLE_LIBCURL_OPTION
2321 if(config->libcurl) {
2322 /* Cleanup the libcurl source output */
2323 easysrc_cleanup();
2324
2325 /* Dump the libcurl code if previously enabled */
2326 dumpeasysrc(config);
2327 }
2328 #endif
2329 }
2330 else
2331 helpf(config->errors, "out of memory\n");
2332 }
2333 }
2334
2335 return result;
2336 }
2337