1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2000-2014 Dag-Erling Smørgrav
5 * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer
13 * in this position and unchanged.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. The name of the author may not be used to endorse or promote products
18 * derived from this software without specific prior written permission
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * $FreeBSD: head/usr.bin/fetch/fetch.c 338572 2018-09-10 19:39:20Z des $
32 */
33
34 #include <sys/param.h>
35 #include <sys/socket.h>
36 #include <sys/stat.h>
37 #include <sys/time.h>
38
39 #include <ctype.h>
40 #include <err.h>
41 #include <errno.h>
42 #include <getopt.h>
43 #include <signal.h>
44 #include <stdint.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48 #include <termios.h>
49 #include <unistd.h>
50
51 #include <fetch.h>
52
53 #define MINBUFSIZE 16384
54 #define TIMEOUT 120
55
56 /* Option flags */
57 static int A_flag; /* -A: do not follow 302 redirects */
58 static int a_flag; /* -a: auto retry */
59 static off_t B_size; /* -B: buffer size */
60 static int b_flag; /*! -b: workaround TCP bug */
61 static char *c_dirname; /* -c: remote directory */
62 static int d_flag; /* -d: direct connection */
63 static int F_flag; /* -F: restart without checking mtime */
64 static char *f_filename; /* -f: file to fetch */
65 static char *h_hostname; /* -h: host to fetch from */
66 static int i_flag; /* -i: specify file for mtime comparison */
67 static char *i_filename; /* name of input file */
68 static int l_flag; /* -l: link rather than copy file: URLs */
69 static int m_flag; /* -[Mm]: mirror mode */
70 static char *N_filename; /* -N: netrc file name */
71 static int n_flag; /* -n: do not preserve modification time */
72 static int o_flag; /* -o: specify output file */
73 static int o_directory; /* output file is a directory */
74 static char *o_filename; /* name of output file */
75 static int o_stdout; /* output file is stdout */
76 static int once_flag; /* -1: stop at first successful file */
77 static int p_flag; /* -[Pp]: use passive FTP */
78 static int R_flag; /* -R: don't delete partial files */
79 static int r_flag; /* -r: restart previous transfer */
80 static off_t S_size; /* -S: require size to match */
81 static int s_flag; /* -s: show size, don't fetch */
82 static long T_secs; /* -T: transfer timeout in seconds */
83 static int t_flag; /*! -t: workaround TCP bug */
84 static int U_flag; /* -U: do not use high ports */
85 static int v_level = 1; /* -v: verbosity level */
86 static int v_tty; /* stdout is a tty */
87 static int v_progress; /* whether to display progress */
88 static pid_t pgrp; /* our process group */
89 static long w_secs; /* -w: retry delay */
90 static int family = PF_UNSPEC; /* -[46]: address family to use */
91
92 static volatile int sigalrm_enable; /* interlock */
93 static volatile int sigalrm; /* SIGALRM received */
94 static volatile int siginfo; /* SIGINFO received */
95 static volatile int sigint; /* SIGINT received */
96
97 static long ftp_timeout = TIMEOUT; /* default timeout for FTP transfers */
98 static long http_timeout = TIMEOUT;/* default timeout for HTTP transfers */
99 static char *buf; /* transfer buffer */
100
101 enum options
102 {
103 OPTION_BIND_ADDRESS,
104 OPTION_NO_FTP_PASSIVE_MODE,
105 OPTION_HTTP_REFERER,
106 OPTION_HTTP_USER_AGENT,
107 OPTION_NO_PROXY,
108 OPTION_SSL_CA_CERT_FILE,
109 OPTION_SSL_CA_CERT_PATH,
110 OPTION_SSL_CLIENT_CERT_FILE,
111 OPTION_SSL_CLIENT_KEY_FILE,
112 OPTION_SSL_CRL_FILE,
113 OPTION_SSL_NO_SSL3,
114 OPTION_SSL_NO_TLS1,
115 OPTION_SSL_NO_VERIFY_HOSTNAME,
116 OPTION_SSL_NO_VERIFY_PEER
117 };
118
119
120 static struct option longopts[] =
121 {
122 /* mapping to single character argument */
123 { "one-file", no_argument, NULL, '1' },
124 { "ipv4-only", no_argument, NULL, '4' },
125 { "ipv6-only", no_argument, NULL, '6' },
126 { "no-redirect", no_argument, NULL, 'A' },
127 { "retry", no_argument, NULL, 'a' },
128 { "buffer-size", required_argument, NULL, 'B' },
129 /* -c not mapped, since it's deprecated */
130 { "direct", no_argument, NULL, 'd' },
131 { "force-restart", no_argument, NULL, 'F' },
132 /* -f not mapped, since it's deprecated */
133 /* -h not mapped, since it's deprecated */
134 { "if-modified-since", required_argument, NULL, 'i' },
135 { "symlink", no_argument, NULL, 'l' },
136 /* -M not mapped since it's the same as -m */
137 { "mirror", no_argument, NULL, 'm' },
138 { "netrc", required_argument, NULL, 'N' },
139 { "no-mtime", no_argument, NULL, 'n' },
140 { "output", required_argument, NULL, 'o' },
141 /* -P not mapped since it's the same as -p */
142 { "passive", no_argument, NULL, 'p' },
143 { "quiet", no_argument, NULL, 'q' },
144 { "keep-output", no_argument, NULL, 'R' },
145 { "restart", no_argument, NULL, 'r' },
146 { "require-size", required_argument, NULL, 'S' },
147 { "print-size", no_argument, NULL, 's' },
148 { "timeout", required_argument, NULL, 'T' },
149 { "passive-portrange-default", no_argument, NULL, 'T' },
150 { "verbose", no_argument, NULL, 'v' },
151 { "retry-delay", required_argument, NULL, 'w' },
152
153 /* options without a single character equivalent */
154 { "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS },
155 { "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE },
156 { "referer", required_argument, NULL, OPTION_HTTP_REFERER },
157 { "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
158 { "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
159 { "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
160 { "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
161 { "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
162 { "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE },
163 { "crl", required_argument, NULL, OPTION_SSL_CRL_FILE },
164 { "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 },
165 { "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 },
166 { "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME },
167 { "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER },
168
169 { NULL, 0, NULL, 0 }
170 };
171
172 /*
173 * Signal handler
174 */
175 static void
sig_handler(int sig)176 sig_handler(int sig)
177 {
178 switch (sig) {
179 case SIGALRM:
180 /*
181 * Realarm every second after the first alarm just in case
182 * we do not catch the code in a blocking condition.
183 */
184 if (sigalrm_enable) {
185 sigalrm = 1;
186 alarm(1);
187 }
188 break;
189 case SIGINFO:
190 siginfo = 1;
191 break;
192 case SIGINT:
193 sigint = 1;
194 break;
195 }
196 }
197
198 struct xferstat {
199 char name[64];
200 struct timeval start; /* start of transfer */
201 struct timeval last; /* time of last update */
202 struct timeval last2; /* time of previous last update */
203 off_t size; /* size of file per HTTP hdr */
204 off_t offset; /* starting offset in file */
205 off_t rcvd; /* bytes already received */
206 off_t lastrcvd; /* bytes received since last update */
207 };
208
209 /*
210 * Format a number of seconds as either XXdYYh, XXhYYm, XXmYYs, or XXs
211 * depending on its magnitude
212 */
213 static void
stat_seconds(char * str,size_t strsz,long seconds)214 stat_seconds(char *str, size_t strsz, long seconds)
215 {
216
217 if (seconds > 86400)
218 snprintf(str, strsz, "%02ldd%02ldh",
219 seconds / 86400, (seconds % 86400) / 3600);
220 else if (seconds > 3600)
221 snprintf(str, strsz, "%02ldh%02ldm",
222 seconds / 3600, (seconds % 3600) / 60);
223 else if (seconds > 60)
224 snprintf(str, strsz, "%02ldm%02lds",
225 seconds / 60, seconds % 60);
226 else
227 snprintf(str, strsz, " %02lds",
228 seconds);
229 }
230
231 /*
232 * Compute and display ETA
233 */
234 static void
stat_eta(char * str,size_t strsz,const struct xferstat * xs)235 stat_eta(char *str, size_t strsz, const struct xferstat *xs)
236 {
237 long elapsed, eta;
238 off_t received, expected;
239
240 elapsed = xs->last.tv_sec - xs->start.tv_sec;
241 received = xs->rcvd - xs->offset;
242 expected = xs->size - xs->rcvd;
243 eta = (long)((double)elapsed * expected / received);
244 if (eta > 0)
245 stat_seconds(str, strsz, eta);
246 else
247 stat_seconds(str, strsz, elapsed);
248 }
249
250 /*
251 * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
252 */
253 static const char *prefixes = " kMGTP";
254 static void
stat_bytes(char * str,size_t strsz,off_t bytes)255 stat_bytes(char *str, size_t strsz, off_t bytes)
256 {
257 const char *prefix = prefixes;
258
259 while (bytes > 9999 && prefix[1] != '\0') {
260 bytes /= 1024;
261 prefix++;
262 }
263 snprintf(str, strsz, "%4ju %cB", (uintmax_t)bytes, *prefix);
264 }
265
266 /*
267 * Compute and display transfer rate
268 */
269 static void
stat_bps(char * str,size_t strsz,struct xferstat * xs)270 stat_bps(char *str, size_t strsz, struct xferstat *xs)
271 {
272 char bytes[16];
273 double delta, bps;
274
275 delta = ((double)xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
276 - ((double)xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6));
277
278 if (delta == 0.0) {
279 snprintf(str, strsz, "?? Bps");
280 } else {
281 bps = (xs->rcvd - xs->lastrcvd) / delta;
282 stat_bytes(bytes, sizeof bytes, (off_t)bps);
283 snprintf(str, strsz, "%sps", bytes);
284 }
285 }
286
287 /*
288 * Update the stats display
289 */
290 static void
stat_display(struct xferstat * xs,int force)291 stat_display(struct xferstat *xs, int force)
292 {
293 char bytes[16], bps[16], eta[16];
294 struct timeval now;
295 int ctty_pgrp;
296
297 /* check if we're the foreground process */
298 if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) != 0 ||
299 (pid_t)ctty_pgrp != pgrp)
300 return;
301
302 gettimeofday(&now, NULL);
303 if (!force && now.tv_sec <= xs->last.tv_sec)
304 return;
305 xs->last2 = xs->last;
306 xs->last = now;
307
308 fprintf(stderr, "\r%-46.46s", xs->name);
309 if (xs->rcvd >= xs->size) {
310 stat_bytes(bytes, sizeof bytes, xs->rcvd);
311 setproctitle("%s [%s]", xs->name, bytes);
312 fprintf(stderr, " %s", bytes);
313 } else {
314 stat_bytes(bytes, sizeof bytes, xs->size);
315 setproctitle("%s [%d%% of %s]", xs->name,
316 (int)((100.0 * xs->rcvd) / xs->size),
317 bytes);
318 fprintf(stderr, "%3d%% of %s",
319 (int)((100.0 * xs->rcvd) / xs->size),
320 bytes);
321 }
322 if (force == 2) {
323 xs->lastrcvd = xs->offset;
324 xs->last2 = xs->start;
325 }
326 stat_bps(bps, sizeof bps, xs);
327 fprintf(stderr, " %s", bps);
328 if ((xs->size > 0 && xs->rcvd > 0 &&
329 xs->last.tv_sec >= xs->start.tv_sec + 3) ||
330 force == 2) {
331 stat_eta(eta, sizeof eta, xs);
332 fprintf(stderr, " %s", eta);
333 }
334 xs->lastrcvd = xs->rcvd;
335 }
336
337 /*
338 * Initialize the transfer statistics
339 */
340 static void
stat_start(struct xferstat * xs,const char * name,off_t size,off_t offset)341 stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
342 {
343
344 memset(xs, 0, sizeof *xs);
345 snprintf(xs->name, sizeof xs->name, "%s", name);
346 gettimeofday(&xs->start, NULL);
347 xs->last2 = xs->last = xs->start;
348 xs->size = size;
349 xs->offset = offset;
350 xs->rcvd = offset;
351 xs->lastrcvd = offset;
352 if (v_progress)
353 stat_display(xs, 1);
354 else if (v_level > 0)
355 fprintf(stderr, "%-46s", xs->name);
356 }
357
358 /*
359 * Update the transfer statistics
360 */
361 static void
stat_update(struct xferstat * xs,off_t rcvd)362 stat_update(struct xferstat *xs, off_t rcvd)
363 {
364
365 xs->rcvd = rcvd;
366 if (v_progress)
367 stat_display(xs, 0);
368 }
369
370 /*
371 * Finalize the transfer statistics
372 */
373 static void
stat_end(struct xferstat * xs)374 stat_end(struct xferstat *xs)
375 {
376 char bytes[16], bps[16], eta[16];
377
378 gettimeofday(&xs->last, NULL);
379 if (v_progress) {
380 stat_display(xs, 2);
381 putc('\n', stderr);
382 } else if (v_level > 0) {
383 stat_bytes(bytes, sizeof bytes, xs->rcvd);
384 stat_bps(bps, sizeof bps, xs);
385 stat_eta(eta, sizeof eta, xs);
386 fprintf(stderr, " %s %s %s\n", bytes, bps, eta);
387 }
388 }
389
390 /*
391 * Ask the user for authentication details
392 */
393 static int
query_auth(struct url * URL)394 query_auth(struct url *URL)
395 {
396 struct termios tios;
397 tcflag_t saved_flags;
398 int i, nopwd;
399
400 fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n",
401 URL->scheme, URL->host, URL->port);
402
403 fprintf(stderr, "Login: ");
404 if (fgets(URL->user, sizeof URL->user, stdin) == NULL)
405 return (-1);
406 for (i = strlen(URL->user); i >= 0; --i)
407 if (URL->user[i] == '\r' || URL->user[i] == '\n')
408 URL->user[i] = '\0';
409
410 fprintf(stderr, "Password: ");
411 if (tcgetattr(STDIN_FILENO, &tios) == 0) {
412 saved_flags = tios.c_lflag;
413 tios.c_lflag &= ~ECHO;
414 tios.c_lflag |= ECHONL|ICANON;
415 tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios);
416 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
417 tios.c_lflag = saved_flags;
418 tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios);
419 } else {
420 nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
421 }
422 if (nopwd)
423 return (-1);
424 for (i = strlen(URL->pwd); i >= 0; --i)
425 if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n')
426 URL->pwd[i] = '\0';
427
428 return (0);
429 }
430
431 /*
432 * Fetch a file
433 */
434 static int
fetch(char * URL,const char * path)435 fetch(char *URL, const char *path)
436 {
437 struct url *url;
438 struct url_stat us;
439 struct stat sb, nsb;
440 struct xferstat xs;
441 FILE *f, *of;
442 size_t size, readcnt, wr;
443 off_t count;
444 char flags[8];
445 const char *slash;
446 char *tmppath;
447 int r;
448 unsigned timeout;
449 char *ptr;
450
451 f = of = NULL;
452 tmppath = NULL;
453
454 timeout = 0;
455 *flags = 0;
456 count = 0;
457
458 /* set verbosity level */
459 if (v_level > 1)
460 strcat(flags, "v");
461 if (v_level > 2)
462 fetchDebug = 1;
463
464 /* parse URL */
465 url = NULL;
466 if (*URL == '\0') {
467 warnx("empty URL");
468 goto failure;
469 }
470 if ((url = fetchParseURL(URL)) == NULL) {
471 warnx("%s: parse error", URL);
472 goto failure;
473 }
474
475 /* if no scheme was specified, take a guess */
476 if (!*url->scheme) {
477 if (!*url->host)
478 strcpy(url->scheme, SCHEME_FILE);
479 else if (strncasecmp(url->host, "ftp.", 4) == 0)
480 strcpy(url->scheme, SCHEME_FTP);
481 else if (strncasecmp(url->host, "www.", 4) == 0)
482 strcpy(url->scheme, SCHEME_HTTP);
483 }
484
485 /* common flags */
486 switch (family) {
487 case PF_INET:
488 strcat(flags, "4");
489 break;
490 case PF_INET6:
491 strcat(flags, "6");
492 break;
493 }
494
495 /* FTP specific flags */
496 if (strcmp(url->scheme, SCHEME_FTP) == 0) {
497 if (p_flag)
498 strcat(flags, "p");
499 if (d_flag)
500 strcat(flags, "d");
501 if (U_flag)
502 strcat(flags, "l");
503 timeout = T_secs ? T_secs : ftp_timeout;
504 }
505
506 /* HTTP specific flags */
507 if (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
508 strcmp(url->scheme, SCHEME_HTTPS) == 0) {
509 if (d_flag)
510 strcat(flags, "d");
511 if (A_flag)
512 strcat(flags, "A");
513 timeout = T_secs ? T_secs : http_timeout;
514 if (i_flag) {
515 if (stat(i_filename, &sb)) {
516 warn("%s: stat()", i_filename);
517 goto failure;
518 }
519 url->ims_time = sb.st_mtime;
520 strcat(flags, "i");
521 }
522 }
523
524 /* set the protocol timeout. */
525 fetchTimeout = timeout;
526
527 /* just print size */
528 if (s_flag) {
529 if (timeout) {
530 sigalrm_enable = 1;
531 alarm(timeout);
532 }
533 r = fetchStat(url, &us, flags);
534 if (timeout) {
535 sigalrm_enable = 0;
536 alarm(0);
537 }
538 if (sigalrm || sigint)
539 goto signal;
540 if (r == -1) {
541 warnx("%s", fetchLastErrString);
542 goto failure;
543 }
544 if (us.size == -1)
545 printf("Unknown\n");
546 else
547 printf("%jd\n", (intmax_t)us.size);
548 goto success;
549 }
550
551 /*
552 * If the -r flag was specified, we have to compare the local
553 * and remote files, so we should really do a fetchStat()
554 * first, but I know of at least one HTTP server that only
555 * sends the content size in response to GET requests, and
556 * leaves it out of replies to HEAD requests. Also, in the
557 * (frequent) case that the local and remote files match but
558 * the local file is truncated, we have sufficient information
559 * before the compare to issue a correct request. Therefore,
560 * we always issue a GET request as if we were sure the local
561 * file was a truncated copy of the remote file; we can drop
562 * the connection later if we change our minds.
563 */
564 sb.st_size = -1;
565 if (!o_stdout) {
566 r = stat(path, &sb);
567 if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
568 url->offset = sb.st_size;
569 } else if (r == -1 || !S_ISREG(sb.st_mode)) {
570 /*
571 * Whatever value sb.st_size has now is either
572 * wrong (if stat(2) failed) or irrelevant (if the
573 * path does not refer to a regular file)
574 */
575 sb.st_size = -1;
576 }
577 if (r == -1 && errno != ENOENT) {
578 warnx("%s: stat()", path);
579 goto failure;
580 }
581 }
582
583 /* start the transfer */
584 if (timeout) {
585 sigalrm_enable = 1;
586 alarm(timeout);
587 }
588 f = fetchXGet(url, &us, flags);
589 if (timeout) {
590 sigalrm_enable = 0;
591 alarm(0);
592 }
593 if (sigalrm || sigint)
594 goto signal;
595 if (f == NULL) {
596 warnx("%s: %s", URL, fetchLastErrString);
597 if (i_flag && (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
598 strcmp(url->scheme, SCHEME_HTTPS) == 0) &&
599 fetchLastErrCode == FETCH_OK &&
600 strcmp(fetchLastErrString, "Not Modified") == 0) {
601 /* HTTP Not Modified Response, return OK. */
602 r = 0;
603 goto done;
604 } else
605 goto failure;
606 }
607 if (sigint)
608 goto signal;
609
610 /* check that size is as expected */
611 if (S_size) {
612 if (us.size == -1) {
613 warnx("%s: size unknown", URL);
614 } else if (us.size != S_size) {
615 warnx("%s: size mismatch: expected %jd, actual %jd",
616 URL, (intmax_t)S_size, (intmax_t)us.size);
617 goto failure;
618 }
619 }
620
621 /* symlink instead of copy */
622 if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) {
623 if (symlink(url->doc, path) == -1) {
624 warn("%s: symlink()", path);
625 goto failure;
626 }
627 goto success;
628 }
629
630 if (us.size == -1 && !o_stdout && v_level > 0)
631 warnx("%s: size of remote file is not known", URL);
632 if (v_level > 1) {
633 if (sb.st_size != -1)
634 fprintf(stderr, "local size / mtime: %jd / %ld\n",
635 (intmax_t)sb.st_size, (long)sb.st_mtime);
636 if (us.size != -1)
637 fprintf(stderr, "remote size / mtime: %jd / %ld\n",
638 (intmax_t)us.size, (long)us.mtime);
639 }
640
641 /* open output file */
642 if (o_stdout) {
643 /* output to stdout */
644 of = stdout;
645 } else if (r_flag && sb.st_size != -1) {
646 /* resume mode, local file exists */
647 if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
648 /* no match! have to refetch */
649 fclose(f);
650 /* if precious, warn the user and give up */
651 if (R_flag) {
652 warnx("%s: local modification time "
653 "does not match remote", path);
654 goto failure_keep;
655 }
656 } else if (url->offset > sb.st_size) {
657 /* gap between what we asked for and what we got */
658 warnx("%s: gap in resume mode", URL);
659 fclose(of);
660 of = NULL;
661 /* picked up again later */
662 } else if (us.size != -1) {
663 if (us.size == sb.st_size)
664 /* nothing to do */
665 goto success;
666 if (sb.st_size > us.size) {
667 /* local file too long! */
668 warnx("%s: local file (%jd bytes) is longer "
669 "than remote file (%jd bytes)", path,
670 (intmax_t)sb.st_size, (intmax_t)us.size);
671 goto failure;
672 }
673 /* we got it, open local file */
674 if ((of = fopen(path, "r+")) == NULL) {
675 warn("%s: fopen()", path);
676 goto failure;
677 }
678 /* check that it didn't move under our feet */
679 if (fstat(fileno(of), &nsb) == -1) {
680 /* can't happen! */
681 warn("%s: fstat()", path);
682 goto failure;
683 }
684 if (nsb.st_dev != sb.st_dev ||
685 nsb.st_ino != sb.st_ino ||
686 nsb.st_size != sb.st_size) {
687 warnx("%s: file has changed", URL);
688 fclose(of);
689 of = NULL;
690 sb = nsb;
691 /* picked up again later */
692 }
693 }
694 /* seek to where we left off */
695 if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) {
696 warn("%s: fseeko()", path);
697 fclose(of);
698 of = NULL;
699 /* picked up again later */
700 }
701 } else if (m_flag && sb.st_size != -1) {
702 /* mirror mode, local file exists */
703 if (sb.st_size == us.size && sb.st_mtime == us.mtime)
704 goto success;
705 }
706
707 if (of == NULL) {
708 /*
709 * We don't yet have an output file; either this is a
710 * vanilla run with no special flags, or the local and
711 * remote files didn't match.
712 */
713
714 if (url->offset > 0) {
715 /*
716 * We tried to restart a transfer, but for
717 * some reason gave up - so we have to restart
718 * from scratch if we want the whole file
719 */
720 url->offset = 0;
721 if ((f = fetchXGet(url, &us, flags)) == NULL) {
722 warnx("%s: %s", URL, fetchLastErrString);
723 goto failure;
724 }
725 if (sigint)
726 goto signal;
727 }
728
729 /* construct a temp file name */
730 if (sb.st_size != -1 && S_ISREG(sb.st_mode)) {
731 if ((slash = strrchr(path, '/')) == NULL)
732 slash = path;
733 else
734 ++slash;
735 asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
736 (int)(slash - path), path, slash);
737 if (tmppath != NULL) {
738 if (mkstemps(tmppath, strlen(slash) + 1) == -1) {
739 warn("%s: mkstemps()", path);
740 goto failure;
741 }
742 of = fopen(tmppath, "w");
743 chown(tmppath, sb.st_uid, sb.st_gid);
744 chmod(tmppath, sb.st_mode & ALLPERMS);
745 }
746 }
747 if (of == NULL)
748 of = fopen(path, "w");
749 if (of == NULL) {
750 warn("%s: open()", path);
751 goto failure;
752 }
753 }
754 count = url->offset;
755
756 /* start the counter */
757 stat_start(&xs, path, us.size, count);
758
759 sigalrm = siginfo = sigint = 0;
760
761 /* suck in the data */
762 setvbuf(f, NULL, _IOFBF, B_size);
763 signal(SIGINFO, sig_handler);
764 while (!sigint) {
765 if (us.size != -1 && us.size - count < B_size &&
766 us.size - count >= 0)
767 size = us.size - count;
768 else
769 size = B_size;
770 if (siginfo) {
771 stat_end(&xs);
772 siginfo = 0;
773 }
774
775 if (size == 0)
776 break;
777
778 if (timeout) {
779 sigalrm_enable = 1;
780 alarm(timeout);
781 }
782 readcnt = fread(buf, 1, size, f);
783 if (timeout) {
784 sigalrm_enable = 0;
785 alarm(0);
786 }
787
788 if (readcnt < size) {
789 if (ferror(f) && errno == EINTR && !sigint)
790 clearerr(f);
791 else if (readcnt == 0)
792 break;
793 }
794
795 stat_update(&xs, count += readcnt);
796 for (ptr = buf; readcnt > 0; ptr += wr, readcnt -= wr) {
797 if ((wr = fwrite(ptr, 1, readcnt, of)) < readcnt) {
798 if (ferror(of) && errno == EINTR && !sigint)
799 clearerr(of);
800 else
801 break;
802 }
803 }
804 if (readcnt != 0)
805 break;
806 }
807 if (!sigalrm)
808 sigalrm = ferror(f) && errno == ETIMEDOUT;
809 signal(SIGINFO, SIG_DFL);
810
811 stat_end(&xs);
812
813 /*
814 * If the transfer timed out or was interrupted, we still want to
815 * set the mtime in case the file is not removed (-r or -R) and
816 * the user later restarts the transfer.
817 */
818 signal:
819 /* set mtime of local file */
820 if (!n_flag && us.mtime && !o_stdout && of != NULL &&
821 (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) {
822 struct timeval tv[2];
823
824 fflush(of);
825 tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime);
826 tv[1].tv_sec = (long)us.mtime;
827 tv[0].tv_usec = tv[1].tv_usec = 0;
828 if (utimes(tmppath ? tmppath : path, tv))
829 warn("%s: utimes()", tmppath ? tmppath : path);
830 }
831
832 /* timed out or interrupted? */
833 if (sigalrm)
834 warnx("transfer timed out");
835 if (sigint) {
836 warnx("transfer interrupted");
837 goto failure;
838 }
839
840 /* timeout / interrupt before connection completley established? */
841 if (f == NULL)
842 goto failure;
843
844 if (!sigalrm) {
845 /* check the status of our files */
846 if (ferror(f))
847 warn("%s", URL);
848 if (ferror(of))
849 warn("%s", path);
850 if (ferror(f) || ferror(of))
851 goto failure;
852 }
853
854 /* did the transfer complete normally? */
855 if (us.size != -1 && count < us.size) {
856 warnx("%s appears to be truncated: %jd/%jd bytes",
857 path, (intmax_t)count, (intmax_t)us.size);
858 goto failure_keep;
859 }
860
861 /*
862 * If the transfer timed out and we didn't know how much to
863 * expect, assume the worst (i.e. we didn't get all of it)
864 */
865 if (sigalrm && us.size == -1) {
866 warnx("%s may be truncated", path);
867 goto failure_keep;
868 }
869
870 success:
871 r = 0;
872 if (tmppath != NULL && rename(tmppath, path) == -1) {
873 warn("%s: rename()", path);
874 goto failure_keep;
875 }
876 goto done;
877 failure:
878 if (of && of != stdout && !R_flag && !r_flag)
879 if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG))
880 unlink(tmppath ? tmppath : path);
881 if (R_flag && tmppath != NULL && sb.st_size == -1)
882 rename(tmppath, path); /* ignore errors here */
883 failure_keep:
884 r = -1;
885 goto done;
886 done:
887 if (f)
888 fclose(f);
889 if (of && of != stdout)
890 fclose(of);
891 if (url)
892 fetchFreeURL(url);
893 if (tmppath != NULL)
894 free(tmppath);
895 return (r);
896 }
897
898 static void
usage(void)899 usage(void)
900 {
901 fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
902 "usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
903 " [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
904 " [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
905 " [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
906 " [-o file] [--referer=URL] [-S bytes] [-T seconds]",
907 " [--user-agent=agent-string] [-w seconds] URL ...",
908 " fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
909 " [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
910 " [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
911 " [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
912 " [-o file] [--referer=URL] [-S bytes] [-T seconds]",
913 " [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
914 }
915
916
917 /*
918 * Entry point
919 */
920 int
main(int argc,char * argv[])921 main(int argc, char *argv[])
922 {
923 struct stat sb;
924 struct sigaction sa;
925 const char *p, *s;
926 char *end, *q;
927 int c, e, r;
928
929
930 while ((c = getopt_long(argc, argv,
931 "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:",
932 longopts, NULL)) != -1)
933 switch (c) {
934 case '1':
935 once_flag = 1;
936 break;
937 case '4':
938 family = PF_INET;
939 break;
940 case '6':
941 family = PF_INET6;
942 break;
943 case 'A':
944 A_flag = 1;
945 break;
946 case 'a':
947 a_flag = 1;
948 break;
949 case 'B':
950 B_size = (off_t)strtol(optarg, &end, 10);
951 if (*optarg == '\0' || *end != '\0')
952 errx(1, "invalid buffer size (%s)", optarg);
953 break;
954 case 'b':
955 warnx("warning: the -b option is deprecated");
956 b_flag = 1;
957 break;
958 case 'c':
959 c_dirname = optarg;
960 break;
961 case 'd':
962 d_flag = 1;
963 break;
964 case 'F':
965 F_flag = 1;
966 break;
967 case 'f':
968 f_filename = optarg;
969 break;
970 case 'H':
971 warnx("the -H option is now implicit, "
972 "use -U to disable");
973 break;
974 case 'h':
975 h_hostname = optarg;
976 break;
977 case 'i':
978 i_flag = 1;
979 i_filename = optarg;
980 break;
981 case 'l':
982 l_flag = 1;
983 break;
984 case 'o':
985 o_flag = 1;
986 o_filename = optarg;
987 break;
988 case 'M':
989 case 'm':
990 if (r_flag)
991 errx(1, "the -m and -r flags "
992 "are mutually exclusive");
993 m_flag = 1;
994 break;
995 case 'N':
996 N_filename = optarg;
997 break;
998 case 'n':
999 n_flag = 1;
1000 break;
1001 case 'P':
1002 case 'p':
1003 p_flag = 1;
1004 break;
1005 case 'q':
1006 v_level = 0;
1007 break;
1008 case 'R':
1009 R_flag = 1;
1010 break;
1011 case 'r':
1012 if (m_flag)
1013 errx(1, "the -m and -r flags "
1014 "are mutually exclusive");
1015 r_flag = 1;
1016 break;
1017 case 'S':
1018 S_size = (off_t)strtol(optarg, &end, 10);
1019 if (*optarg == '\0' || *end != '\0')
1020 errx(1, "invalid size (%s)", optarg);
1021 break;
1022 case 's':
1023 s_flag = 1;
1024 break;
1025 case 'T':
1026 T_secs = strtol(optarg, &end, 10);
1027 if (*optarg == '\0' || *end != '\0')
1028 errx(1, "invalid timeout (%s)", optarg);
1029 break;
1030 case 't':
1031 t_flag = 1;
1032 warnx("warning: the -t option is deprecated");
1033 break;
1034 case 'U':
1035 U_flag = 1;
1036 break;
1037 case 'v':
1038 v_level++;
1039 break;
1040 case 'w':
1041 a_flag = 1;
1042 w_secs = strtol(optarg, &end, 10);
1043 if (*optarg == '\0' || *end != '\0')
1044 errx(1, "invalid delay (%s)", optarg);
1045 break;
1046 case OPTION_BIND_ADDRESS:
1047 setenv("FETCH_BIND_ADDRESS", optarg, 1);
1048 break;
1049 case OPTION_NO_FTP_PASSIVE_MODE:
1050 setenv("FTP_PASSIVE_MODE", "no", 1);
1051 break;
1052 case OPTION_HTTP_REFERER:
1053 setenv("HTTP_REFERER", optarg, 1);
1054 break;
1055 case OPTION_HTTP_USER_AGENT:
1056 setenv("HTTP_USER_AGENT", optarg, 1);
1057 break;
1058 case OPTION_NO_PROXY:
1059 setenv("NO_PROXY", optarg, 1);
1060 break;
1061 case OPTION_SSL_CA_CERT_FILE:
1062 setenv("SSL_CA_CERT_FILE", optarg, 1);
1063 break;
1064 case OPTION_SSL_CA_CERT_PATH:
1065 setenv("SSL_CA_CERT_PATH", optarg, 1);
1066 break;
1067 case OPTION_SSL_CLIENT_CERT_FILE:
1068 setenv("SSL_CLIENT_CERT_FILE", optarg, 1);
1069 break;
1070 case OPTION_SSL_CLIENT_KEY_FILE:
1071 setenv("SSL_CLIENT_KEY_FILE", optarg, 1);
1072 break;
1073 case OPTION_SSL_CRL_FILE:
1074 setenv("SSL_CLIENT_CRL_FILE", optarg, 1);
1075 break;
1076 case OPTION_SSL_NO_SSL3:
1077 setenv("SSL_NO_SSL3", "", 1);
1078 break;
1079 case OPTION_SSL_NO_TLS1:
1080 setenv("SSL_NO_TLS1", "", 1);
1081 break;
1082 case OPTION_SSL_NO_VERIFY_HOSTNAME:
1083 setenv("SSL_NO_VERIFY_HOSTNAME", "", 1);
1084 break;
1085 case OPTION_SSL_NO_VERIFY_PEER:
1086 setenv("SSL_NO_VERIFY_PEER", "", 1);
1087 break;
1088 default:
1089 usage();
1090 exit(1);
1091 }
1092
1093 argc -= optind;
1094 argv += optind;
1095
1096 if (h_hostname || f_filename || c_dirname) {
1097 if (!h_hostname || !f_filename || argc) {
1098 usage();
1099 exit(1);
1100 }
1101 /* XXX this is a hack. */
1102 if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
1103 errx(1, "invalid hostname");
1104 if (asprintf(argv, "ftp://%s/%s/%s", h_hostname,
1105 c_dirname ? c_dirname : "", f_filename) == -1)
1106 errx(1, "%s", strerror(ENOMEM));
1107 argc++;
1108 }
1109
1110 if (!argc) {
1111 usage();
1112 exit(1);
1113 }
1114
1115 /* allocate buffer */
1116 if (B_size < MINBUFSIZE)
1117 B_size = MINBUFSIZE;
1118 if ((buf = malloc(B_size)) == NULL)
1119 errx(1, "%s", strerror(ENOMEM));
1120
1121 /* timeouts */
1122 if ((s = getenv("FTP_TIMEOUT")) != NULL) {
1123 ftp_timeout = strtol(s, &end, 10);
1124 if (*s == '\0' || *end != '\0' || ftp_timeout < 0) {
1125 warnx("FTP_TIMEOUT (%s) is not a positive integer", s);
1126 ftp_timeout = 0;
1127 }
1128 }
1129 if ((s = getenv("HTTP_TIMEOUT")) != NULL) {
1130 http_timeout = strtol(s, &end, 10);
1131 if (*s == '\0' || *end != '\0' || http_timeout < 0) {
1132 warnx("HTTP_TIMEOUT (%s) is not a positive integer", s);
1133 http_timeout = 0;
1134 }
1135 }
1136
1137 /* signal handling */
1138 sa.sa_flags = 0;
1139 sa.sa_handler = sig_handler;
1140 sigemptyset(&sa.sa_mask);
1141 sigaction(SIGALRM, &sa, NULL);
1142 sa.sa_flags = SA_RESETHAND;
1143 sigaction(SIGINT, &sa, NULL);
1144 fetchRestartCalls = 0;
1145
1146 /* output file */
1147 if (o_flag) {
1148 if (strcmp(o_filename, "-") == 0) {
1149 o_stdout = 1;
1150 } else if (stat(o_filename, &sb) == -1) {
1151 if (errno == ENOENT) {
1152 if (argc > 1)
1153 errx(1, "%s is not a directory",
1154 o_filename);
1155 } else {
1156 err(1, "%s", o_filename);
1157 }
1158 } else {
1159 if (sb.st_mode & S_IFDIR)
1160 o_directory = 1;
1161 }
1162 }
1163
1164 /* check if output is to a tty (for progress report) */
1165 v_tty = isatty(STDERR_FILENO);
1166 v_progress = v_tty && v_level > 0;
1167 if (v_progress)
1168 pgrp = getpgrp();
1169
1170 r = 0;
1171
1172 /* authentication */
1173 if (v_tty)
1174 fetchAuthMethod = query_auth;
1175 if (N_filename != NULL) {
1176 if (setenv("NETRC", N_filename, 1) == -1)
1177 err(1, "setenv: cannot set NETRC=%s", N_filename);
1178 }
1179
1180 while (argc) {
1181 if ((p = strrchr(*argv, '/')) == NULL)
1182 p = *argv;
1183 else
1184 p++;
1185
1186 if (!*p)
1187 p = "fetch.out";
1188
1189 fetchLastErrCode = 0;
1190
1191 if (o_flag) {
1192 if (o_stdout) {
1193 e = fetch(*argv, "-");
1194 } else if (o_directory) {
1195 asprintf(&q, "%s/%s", o_filename, p);
1196 e = fetch(*argv, q);
1197 free(q);
1198 } else {
1199 e = fetch(*argv, o_filename);
1200 }
1201 } else {
1202 e = fetch(*argv, p);
1203 }
1204
1205 if (sigint)
1206 kill(getpid(), SIGINT);
1207
1208 if (e == 0 && once_flag)
1209 exit(0);
1210
1211 if (e) {
1212 r = 1;
1213 if ((fetchLastErrCode
1214 && fetchLastErrCode != FETCH_UNAVAIL
1215 && fetchLastErrCode != FETCH_MOVED
1216 && fetchLastErrCode != FETCH_URL
1217 && fetchLastErrCode != FETCH_RESOLV
1218 && fetchLastErrCode != FETCH_UNKNOWN)) {
1219 if (w_secs && v_level)
1220 fprintf(stderr, "Waiting %ld seconds "
1221 "before retrying\n", w_secs);
1222 if (w_secs)
1223 sleep(w_secs);
1224 if (a_flag)
1225 continue;
1226 }
1227 }
1228
1229 argc--, argv++;
1230 }
1231
1232 exit(r);
1233 }
1234