xref: /dragonfly/usr.bin/fetch/fetch.c (revision f50a4783)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 2000-2014 Dag-Erling Smørgrav
5  * Copyright (c) 2013 Michael Gmelin <freebsd@grem.de>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer
13  *    in this position and unchanged.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. The name of the author may not be used to endorse or promote products
18  *    derived from this software without specific prior written permission
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  *
31  * $FreeBSD: head/usr.bin/fetch/fetch.c 338572 2018-09-10 19:39:20Z des $
32  */
33 
34 #include <sys/param.h>
35 #include <sys/socket.h>
36 #include <sys/stat.h>
37 #include <sys/time.h>
38 
39 #include <ctype.h>
40 #include <err.h>
41 #include <errno.h>
42 #include <getopt.h>
43 #include <signal.h>
44 #include <stdint.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48 #include <termios.h>
49 #include <unistd.h>
50 
51 #include <fetch.h>
52 
53 #define MINBUFSIZE	16384
54 #define TIMEOUT		120
55 
56 /* Option flags */
57 static int	 A_flag;	/*    -A: do not follow 302 redirects */
58 static int	 a_flag;	/*    -a: auto retry */
59 static off_t	 B_size;	/*    -B: buffer size */
60 static int	 b_flag;	/*!   -b: workaround TCP bug */
61 static char    *c_dirname;	/*    -c: remote directory */
62 static int	 d_flag;	/*    -d: direct connection */
63 static int	 F_flag;	/*    -F: restart without checking mtime  */
64 static char	*f_filename;	/*    -f: file to fetch */
65 static char	*h_hostname;	/*    -h: host to fetch from */
66 static int	 i_flag;	/*    -i: specify file for mtime comparison */
67 static char	*i_filename;	/*        name of input file */
68 static int	 l_flag;	/*    -l: link rather than copy file: URLs */
69 static int	 m_flag;	/* -[Mm]: mirror mode */
70 static char	*N_filename;	/*    -N: netrc file name */
71 static int	 n_flag;	/*    -n: do not preserve modification time */
72 static int	 o_flag;	/*    -o: specify output file */
73 static int	 o_directory;	/*        output file is a directory */
74 static char	*o_filename;	/*        name of output file */
75 static int	 o_stdout;	/*        output file is stdout */
76 static int	 once_flag;	/*    -1: stop at first successful file */
77 static int	 p_flag;	/* -[Pp]: use passive FTP */
78 static int	 R_flag;	/*    -R: don't delete partial files */
79 static int	 r_flag;	/*    -r: restart previous transfer */
80 static off_t	 S_size;        /*    -S: require size to match */
81 static int	 s_flag;        /*    -s: show size, don't fetch */
82 static long	 T_secs;	/*    -T: transfer timeout in seconds */
83 static int	 t_flag;	/*!   -t: workaround TCP bug */
84 static int	 U_flag;	/*    -U: do not use high ports */
85 static int	 v_level = 1;	/*    -v: verbosity level */
86 static int	 v_tty;		/*        stdout is a tty */
87 static int	 v_progress;	/*        whether to display progress */
88 static pid_t	 pgrp;		/*        our process group */
89 static long	 w_secs;	/*    -w: retry delay */
90 static int	 family = PF_UNSPEC;	/* -[46]: address family to use */
91 
92 static volatile int sigalrm_enable; /* interlock */
93 static volatile int sigalrm;	/* SIGALRM received */
94 static volatile int siginfo;	/* SIGINFO received */
95 static volatile int sigint;	/* SIGINT received */
96 
97 static long	 ftp_timeout = TIMEOUT;	/* default timeout for FTP transfers */
98 static long	 http_timeout = TIMEOUT;/* default timeout for HTTP transfers */
99 static char	*buf;		/* transfer buffer */
100 
101 enum options
102 {
103 	OPTION_BIND_ADDRESS,
104 	OPTION_NO_FTP_PASSIVE_MODE,
105 	OPTION_HTTP_REFERER,
106 	OPTION_HTTP_USER_AGENT,
107 	OPTION_NO_PROXY,
108 	OPTION_SSL_CA_CERT_FILE,
109 	OPTION_SSL_CA_CERT_PATH,
110 	OPTION_SSL_CLIENT_CERT_FILE,
111 	OPTION_SSL_CLIENT_KEY_FILE,
112 	OPTION_SSL_CRL_FILE,
113 	OPTION_SSL_NO_SSL3,
114 	OPTION_SSL_NO_TLS1,
115 	OPTION_SSL_NO_VERIFY_HOSTNAME,
116 	OPTION_SSL_NO_VERIFY_PEER
117 };
118 
119 
120 static struct option longopts[] =
121 {
122 	/* mapping to single character argument */
123 	{ "one-file", no_argument, NULL, '1' },
124 	{ "ipv4-only", no_argument, NULL, '4' },
125 	{ "ipv6-only", no_argument, NULL, '6' },
126 	{ "no-redirect", no_argument, NULL, 'A' },
127 	{ "retry", no_argument, NULL, 'a' },
128 	{ "buffer-size", required_argument, NULL, 'B' },
129 	/* -c not mapped, since it's deprecated */
130 	{ "direct", no_argument, NULL, 'd' },
131 	{ "force-restart", no_argument, NULL, 'F' },
132 	/* -f not mapped, since it's deprecated */
133 	/* -h not mapped, since it's deprecated */
134 	{ "if-modified-since", required_argument, NULL, 'i' },
135 	{ "symlink", no_argument, NULL, 'l' },
136 	/* -M not mapped since it's the same as -m */
137 	{ "mirror", no_argument, NULL, 'm' },
138 	{ "netrc", required_argument, NULL, 'N' },
139 	{ "no-mtime", no_argument, NULL, 'n' },
140 	{ "output", required_argument, NULL, 'o' },
141 	/* -P not mapped since it's the same as -p */
142 	{ "passive", no_argument, NULL, 'p' },
143 	{ "quiet", no_argument, NULL, 'q' },
144 	{ "keep-output", no_argument, NULL, 'R' },
145 	{ "restart", no_argument, NULL, 'r' },
146 	{ "require-size", required_argument, NULL, 'S' },
147 	{ "print-size", no_argument, NULL, 's' },
148 	{ "timeout", required_argument, NULL, 'T' },
149 	{ "passive-portrange-default", no_argument, NULL, 'T' },
150 	{ "verbose", no_argument, NULL, 'v' },
151 	{ "retry-delay", required_argument, NULL, 'w' },
152 
153 	/* options without a single character equivalent */
154 	{ "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS },
155 	{ "no-passive", no_argument, NULL, OPTION_NO_FTP_PASSIVE_MODE },
156 	{ "referer", required_argument, NULL, OPTION_HTTP_REFERER },
157 	{ "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
158 	{ "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
159 	{ "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
160 	{ "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
161 	{ "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
162 	{ "key", required_argument, NULL, OPTION_SSL_CLIENT_KEY_FILE },
163 	{ "crl", required_argument, NULL, OPTION_SSL_CRL_FILE },
164 	{ "no-sslv3", no_argument, NULL, OPTION_SSL_NO_SSL3 },
165 	{ "no-tlsv1", no_argument, NULL, OPTION_SSL_NO_TLS1 },
166 	{ "no-verify-hostname", no_argument, NULL, OPTION_SSL_NO_VERIFY_HOSTNAME },
167 	{ "no-verify-peer", no_argument, NULL, OPTION_SSL_NO_VERIFY_PEER },
168 
169 	{ NULL, 0, NULL, 0 }
170 };
171 
172 /*
173  * Signal handler
174  */
175 static void
sig_handler(int sig)176 sig_handler(int sig)
177 {
178 	switch (sig) {
179 	case SIGALRM:
180 		/*
181 		 * Realarm every second after the first alarm just in case
182 		 * we do not catch the code in a blocking condition.
183 		 */
184 		if (sigalrm_enable) {
185 			sigalrm = 1;
186 			alarm(1);
187 		}
188 		break;
189 	case SIGINFO:
190 		siginfo = 1;
191 		break;
192 	case SIGINT:
193 		sigint = 1;
194 		break;
195 	}
196 }
197 
198 struct xferstat {
199 	char		 name[64];
200 	struct timeval	 start;		/* start of transfer */
201 	struct timeval	 last;		/* time of last update */
202 	struct timeval	 last2;		/* time of previous last update */
203 	off_t		 size;		/* size of file per HTTP hdr */
204 	off_t		 offset;	/* starting offset in file */
205 	off_t		 rcvd;		/* bytes already received */
206 	off_t		 lastrcvd;	/* bytes received since last update */
207 };
208 
209 /*
210  * Format a number of seconds as either XXdYYh, XXhYYm, XXmYYs, or XXs
211  * depending on its magnitude
212  */
213 static void
stat_seconds(char * str,size_t strsz,long seconds)214 stat_seconds(char *str, size_t strsz, long seconds)
215 {
216 
217 	if (seconds > 86400)
218 		snprintf(str, strsz, "%02ldd%02ldh",
219 		    seconds / 86400, (seconds % 86400) / 3600);
220 	else if (seconds > 3600)
221 		snprintf(str, strsz, "%02ldh%02ldm",
222 		    seconds / 3600, (seconds % 3600) / 60);
223 	else if (seconds > 60)
224 		snprintf(str, strsz, "%02ldm%02lds",
225 		    seconds / 60, seconds % 60);
226 	else
227 		snprintf(str, strsz, "   %02lds",
228 		    seconds);
229 }
230 
231 /*
232  * Compute and display ETA
233  */
234 static void
stat_eta(char * str,size_t strsz,const struct xferstat * xs)235 stat_eta(char *str, size_t strsz, const struct xferstat *xs)
236 {
237 	long elapsed, eta;
238 	off_t received, expected;
239 
240 	elapsed = xs->last.tv_sec - xs->start.tv_sec;
241 	received = xs->rcvd - xs->offset;
242 	expected = xs->size - xs->rcvd;
243 	eta = (long)((double)elapsed * expected / received);
244 	if (eta > 0)
245 		stat_seconds(str, strsz, eta);
246 	else
247 		stat_seconds(str, strsz, elapsed);
248 }
249 
250 /*
251  * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'...
252  */
253 static const char *prefixes = " kMGTP";
254 static void
stat_bytes(char * str,size_t strsz,off_t bytes)255 stat_bytes(char *str, size_t strsz, off_t bytes)
256 {
257 	const char *prefix = prefixes;
258 
259 	while (bytes > 9999 && prefix[1] != '\0') {
260 		bytes /= 1024;
261 		prefix++;
262 	}
263 	snprintf(str, strsz, "%4ju %cB", (uintmax_t)bytes, *prefix);
264 }
265 
266 /*
267  * Compute and display transfer rate
268  */
269 static void
stat_bps(char * str,size_t strsz,struct xferstat * xs)270 stat_bps(char *str, size_t strsz, struct xferstat *xs)
271 {
272 	char bytes[16];
273 	double delta, bps;
274 
275 	delta = ((double)xs->last.tv_sec + (xs->last.tv_usec / 1.e6))
276 	    - ((double)xs->last2.tv_sec + (xs->last2.tv_usec / 1.e6));
277 
278 	if (delta == 0.0) {
279 		snprintf(str, strsz, "?? Bps");
280 	} else {
281 		bps = (xs->rcvd - xs->lastrcvd) / delta;
282 		stat_bytes(bytes, sizeof bytes, (off_t)bps);
283 		snprintf(str, strsz, "%sps", bytes);
284 	}
285 }
286 
287 /*
288  * Update the stats display
289  */
290 static void
stat_display(struct xferstat * xs,int force)291 stat_display(struct xferstat *xs, int force)
292 {
293 	char bytes[16], bps[16], eta[16];
294 	struct timeval now;
295 	int ctty_pgrp;
296 
297 	/* check if we're the foreground process */
298 	if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) != 0 ||
299 	    (pid_t)ctty_pgrp != pgrp)
300 		return;
301 
302 	gettimeofday(&now, NULL);
303 	if (!force && now.tv_sec <= xs->last.tv_sec)
304 		return;
305 	xs->last2 = xs->last;
306 	xs->last = now;
307 
308 	fprintf(stderr, "\r%-46.46s", xs->name);
309 	if (xs->rcvd >= xs->size) {
310 		stat_bytes(bytes, sizeof bytes, xs->rcvd);
311 		setproctitle("%s [%s]", xs->name, bytes);
312 		fprintf(stderr, "        %s", bytes);
313 	} else {
314 		stat_bytes(bytes, sizeof bytes, xs->size);
315 		setproctitle("%s [%d%% of %s]", xs->name,
316 		    (int)((100.0 * xs->rcvd) / xs->size),
317 		    bytes);
318 		fprintf(stderr, "%3d%% of %s",
319 		    (int)((100.0 * xs->rcvd) / xs->size),
320 		    bytes);
321 	}
322 	if (force == 2) {
323 		xs->lastrcvd = xs->offset;
324 		xs->last2 = xs->start;
325 	}
326 	stat_bps(bps, sizeof bps, xs);
327 	fprintf(stderr, " %s", bps);
328 	if ((xs->size > 0 && xs->rcvd > 0 &&
329 	     xs->last.tv_sec >= xs->start.tv_sec + 3) ||
330 	    force == 2) {
331 		stat_eta(eta, sizeof eta, xs);
332 		fprintf(stderr, " %s", eta);
333 	}
334 	xs->lastrcvd = xs->rcvd;
335 }
336 
337 /*
338  * Initialize the transfer statistics
339  */
340 static void
stat_start(struct xferstat * xs,const char * name,off_t size,off_t offset)341 stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset)
342 {
343 
344 	memset(xs, 0, sizeof *xs);
345 	snprintf(xs->name, sizeof xs->name, "%s", name);
346 	gettimeofday(&xs->start, NULL);
347 	xs->last2 = xs->last = xs->start;
348 	xs->size = size;
349 	xs->offset = offset;
350 	xs->rcvd = offset;
351 	xs->lastrcvd = offset;
352 	if (v_progress)
353 		stat_display(xs, 1);
354 	else if (v_level > 0)
355 		fprintf(stderr, "%-46s", xs->name);
356 }
357 
358 /*
359  * Update the transfer statistics
360  */
361 static void
stat_update(struct xferstat * xs,off_t rcvd)362 stat_update(struct xferstat *xs, off_t rcvd)
363 {
364 
365 	xs->rcvd = rcvd;
366 	if (v_progress)
367 		stat_display(xs, 0);
368 }
369 
370 /*
371  * Finalize the transfer statistics
372  */
373 static void
stat_end(struct xferstat * xs)374 stat_end(struct xferstat *xs)
375 {
376 	char bytes[16], bps[16], eta[16];
377 
378 	gettimeofday(&xs->last, NULL);
379 	if (v_progress) {
380 		stat_display(xs, 2);
381 		putc('\n', stderr);
382 	} else if (v_level > 0) {
383 		stat_bytes(bytes, sizeof bytes, xs->rcvd);
384 		stat_bps(bps, sizeof bps, xs);
385 		stat_eta(eta, sizeof eta, xs);
386 		fprintf(stderr, "        %s %s %s\n", bytes, bps, eta);
387 	}
388 }
389 
390 /*
391  * Ask the user for authentication details
392  */
393 static int
query_auth(struct url * URL)394 query_auth(struct url *URL)
395 {
396 	struct termios tios;
397 	tcflag_t saved_flags;
398 	int i, nopwd;
399 
400 	fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n",
401 	    URL->scheme, URL->host, URL->port);
402 
403 	fprintf(stderr, "Login: ");
404 	if (fgets(URL->user, sizeof URL->user, stdin) == NULL)
405 		return (-1);
406 	for (i = strlen(URL->user); i >= 0; --i)
407 		if (URL->user[i] == '\r' || URL->user[i] == '\n')
408 			URL->user[i] = '\0';
409 
410 	fprintf(stderr, "Password: ");
411 	if (tcgetattr(STDIN_FILENO, &tios) == 0) {
412 		saved_flags = tios.c_lflag;
413 		tios.c_lflag &= ~ECHO;
414 		tios.c_lflag |= ECHONL|ICANON;
415 		tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios);
416 		nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
417 		tios.c_lflag = saved_flags;
418 		tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios);
419 	} else {
420 		nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL);
421 	}
422 	if (nopwd)
423 		return (-1);
424 	for (i = strlen(URL->pwd); i >= 0; --i)
425 		if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n')
426 			URL->pwd[i] = '\0';
427 
428 	return (0);
429 }
430 
431 /*
432  * Fetch a file
433  */
434 static int
fetch(char * URL,const char * path)435 fetch(char *URL, const char *path)
436 {
437 	struct url *url;
438 	struct url_stat us;
439 	struct stat sb, nsb;
440 	struct xferstat xs;
441 	FILE *f, *of;
442 	size_t size, readcnt, wr;
443 	off_t count;
444 	char flags[8];
445 	const char *slash;
446 	char *tmppath;
447 	int r;
448 	unsigned timeout;
449 	char *ptr;
450 
451 	f = of = NULL;
452 	tmppath = NULL;
453 
454 	timeout = 0;
455 	*flags = 0;
456 	count = 0;
457 
458 	/* set verbosity level */
459 	if (v_level > 1)
460 		strcat(flags, "v");
461 	if (v_level > 2)
462 		fetchDebug = 1;
463 
464 	/* parse URL */
465 	url = NULL;
466 	if (*URL == '\0') {
467 		warnx("empty URL");
468 		goto failure;
469 	}
470 	if ((url = fetchParseURL(URL)) == NULL) {
471 		warnx("%s: parse error", URL);
472 		goto failure;
473 	}
474 
475 	/* if no scheme was specified, take a guess */
476 	if (!*url->scheme) {
477 		if (!*url->host)
478 			strcpy(url->scheme, SCHEME_FILE);
479 		else if (strncasecmp(url->host, "ftp.", 4) == 0)
480 			strcpy(url->scheme, SCHEME_FTP);
481 		else if (strncasecmp(url->host, "www.", 4) == 0)
482 			strcpy(url->scheme, SCHEME_HTTP);
483 	}
484 
485 	/* common flags */
486 	switch (family) {
487 	case PF_INET:
488 		strcat(flags, "4");
489 		break;
490 	case PF_INET6:
491 		strcat(flags, "6");
492 		break;
493 	}
494 
495 	/* FTP specific flags */
496 	if (strcmp(url->scheme, SCHEME_FTP) == 0) {
497 		if (p_flag)
498 			strcat(flags, "p");
499 		if (d_flag)
500 			strcat(flags, "d");
501 		if (U_flag)
502 			strcat(flags, "l");
503 		timeout = T_secs ? T_secs : ftp_timeout;
504 	}
505 
506 	/* HTTP specific flags */
507 	if (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
508 	    strcmp(url->scheme, SCHEME_HTTPS) == 0) {
509 		if (d_flag)
510 			strcat(flags, "d");
511 		if (A_flag)
512 			strcat(flags, "A");
513 		timeout = T_secs ? T_secs : http_timeout;
514 		if (i_flag) {
515 			if (stat(i_filename, &sb)) {
516 				warn("%s: stat()", i_filename);
517 				goto failure;
518 			}
519 			url->ims_time = sb.st_mtime;
520 			strcat(flags, "i");
521 		}
522 	}
523 
524 	/* set the protocol timeout. */
525 	fetchTimeout = timeout;
526 
527 	/* just print size */
528 	if (s_flag) {
529 		if (timeout) {
530 			sigalrm_enable = 1;
531 			alarm(timeout);
532 		}
533 		r = fetchStat(url, &us, flags);
534 		if (timeout) {
535 			sigalrm_enable = 0;
536 			alarm(0);
537 		}
538 		if (sigalrm || sigint)
539 			goto signal;
540 		if (r == -1) {
541 			warnx("%s", fetchLastErrString);
542 			goto failure;
543 		}
544 		if (us.size == -1)
545 			printf("Unknown\n");
546 		else
547 			printf("%jd\n", (intmax_t)us.size);
548 		goto success;
549 	}
550 
551 	/*
552 	 * If the -r flag was specified, we have to compare the local
553 	 * and remote files, so we should really do a fetchStat()
554 	 * first, but I know of at least one HTTP server that only
555 	 * sends the content size in response to GET requests, and
556 	 * leaves it out of replies to HEAD requests.  Also, in the
557 	 * (frequent) case that the local and remote files match but
558 	 * the local file is truncated, we have sufficient information
559 	 * before the compare to issue a correct request.  Therefore,
560 	 * we always issue a GET request as if we were sure the local
561 	 * file was a truncated copy of the remote file; we can drop
562 	 * the connection later if we change our minds.
563 	 */
564 	sb.st_size = -1;
565 	if (!o_stdout) {
566 		r = stat(path, &sb);
567 		if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
568 			url->offset = sb.st_size;
569 		} else if (r == -1 || !S_ISREG(sb.st_mode)) {
570 			/*
571 			 * Whatever value sb.st_size has now is either
572 			 * wrong (if stat(2) failed) or irrelevant (if the
573 			 * path does not refer to a regular file)
574 			 */
575 			sb.st_size = -1;
576 		}
577 		if (r == -1 && errno != ENOENT) {
578 			warnx("%s: stat()", path);
579 			goto failure;
580 		}
581 	}
582 
583 	/* start the transfer */
584 	if (timeout) {
585 		sigalrm_enable = 1;
586 		alarm(timeout);
587 	}
588 	f = fetchXGet(url, &us, flags);
589 	if (timeout) {
590 		sigalrm_enable = 0;
591 		alarm(0);
592 	}
593 	if (sigalrm || sigint)
594 		goto signal;
595 	if (f == NULL) {
596 		warnx("%s: %s", URL, fetchLastErrString);
597 		if (i_flag && (strcmp(url->scheme, SCHEME_HTTP) == 0 ||
598 		    strcmp(url->scheme, SCHEME_HTTPS) == 0) &&
599 		    fetchLastErrCode == FETCH_OK &&
600 		    strcmp(fetchLastErrString, "Not Modified") == 0) {
601 			/* HTTP Not Modified Response, return OK. */
602 			r = 0;
603 			goto done;
604 		} else
605 			goto failure;
606 	}
607 	if (sigint)
608 		goto signal;
609 
610 	/* check that size is as expected */
611 	if (S_size) {
612 		if (us.size == -1) {
613 			warnx("%s: size unknown", URL);
614 		} else if (us.size != S_size) {
615 			warnx("%s: size mismatch: expected %jd, actual %jd",
616 			    URL, (intmax_t)S_size, (intmax_t)us.size);
617 			goto failure;
618 		}
619 	}
620 
621 	/* symlink instead of copy */
622 	if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) {
623 		if (symlink(url->doc, path) == -1) {
624 			warn("%s: symlink()", path);
625 			goto failure;
626 		}
627 		goto success;
628 	}
629 
630 	if (us.size == -1 && !o_stdout && v_level > 0)
631 		warnx("%s: size of remote file is not known", URL);
632 	if (v_level > 1) {
633 		if (sb.st_size != -1)
634 			fprintf(stderr, "local size / mtime: %jd / %ld\n",
635 			    (intmax_t)sb.st_size, (long)sb.st_mtime);
636 		if (us.size != -1)
637 			fprintf(stderr, "remote size / mtime: %jd / %ld\n",
638 			    (intmax_t)us.size, (long)us.mtime);
639 	}
640 
641 	/* open output file */
642 	if (o_stdout) {
643 		/* output to stdout */
644 		of = stdout;
645 	} else if (r_flag && sb.st_size != -1) {
646 		/* resume mode, local file exists */
647 		if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
648 			/* no match! have to refetch */
649 			fclose(f);
650 			/* if precious, warn the user and give up */
651 			if (R_flag) {
652 				warnx("%s: local modification time "
653 				    "does not match remote", path);
654 				goto failure_keep;
655 			}
656 		} else if (url->offset > sb.st_size) {
657 			/* gap between what we asked for and what we got */
658 			warnx("%s: gap in resume mode", URL);
659 			fclose(of);
660 			of = NULL;
661 			/* picked up again later */
662 		} else if (us.size != -1) {
663 			if (us.size == sb.st_size)
664 				/* nothing to do */
665 				goto success;
666 			if (sb.st_size > us.size) {
667 				/* local file too long! */
668 				warnx("%s: local file (%jd bytes) is longer "
669 				    "than remote file (%jd bytes)", path,
670 				    (intmax_t)sb.st_size, (intmax_t)us.size);
671 				goto failure;
672 			}
673 			/* we got it, open local file */
674 			if ((of = fopen(path, "r+")) == NULL) {
675 				warn("%s: fopen()", path);
676 				goto failure;
677 			}
678 			/* check that it didn't move under our feet */
679 			if (fstat(fileno(of), &nsb) == -1) {
680 				/* can't happen! */
681 				warn("%s: fstat()", path);
682 				goto failure;
683 			}
684 			if (nsb.st_dev != sb.st_dev ||
685 			    nsb.st_ino != sb.st_ino ||
686 			    nsb.st_size != sb.st_size) {
687 				warnx("%s: file has changed", URL);
688 				fclose(of);
689 				of = NULL;
690 				sb = nsb;
691 				/* picked up again later */
692 			}
693 		}
694 		/* seek to where we left off */
695 		if (of != NULL && fseeko(of, url->offset, SEEK_SET) != 0) {
696 			warn("%s: fseeko()", path);
697 			fclose(of);
698 			of = NULL;
699 			/* picked up again later */
700 		}
701 	} else if (m_flag && sb.st_size != -1) {
702 		/* mirror mode, local file exists */
703 		if (sb.st_size == us.size && sb.st_mtime == us.mtime)
704 			goto success;
705 	}
706 
707 	if (of == NULL) {
708 		/*
709 		 * We don't yet have an output file; either this is a
710 		 * vanilla run with no special flags, or the local and
711 		 * remote files didn't match.
712 		 */
713 
714 		if (url->offset > 0) {
715 			/*
716 			 * We tried to restart a transfer, but for
717 			 * some reason gave up - so we have to restart
718 			 * from scratch if we want the whole file
719 			 */
720 			url->offset = 0;
721 			if ((f = fetchXGet(url, &us, flags)) == NULL) {
722 				warnx("%s: %s", URL, fetchLastErrString);
723 				goto failure;
724 			}
725 			if (sigint)
726 				goto signal;
727 		}
728 
729 		/* construct a temp file name */
730 		if (sb.st_size != -1 && S_ISREG(sb.st_mode)) {
731 			if ((slash = strrchr(path, '/')) == NULL)
732 				slash = path;
733 			else
734 				++slash;
735 			asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
736 			    (int)(slash - path), path, slash);
737 			if (tmppath != NULL) {
738 				if (mkstemps(tmppath, strlen(slash) + 1) == -1) {
739 					warn("%s: mkstemps()", path);
740 					goto failure;
741 				}
742 				of = fopen(tmppath, "w");
743 				chown(tmppath, sb.st_uid, sb.st_gid);
744 				chmod(tmppath, sb.st_mode & ALLPERMS);
745 			}
746 		}
747 		if (of == NULL)
748 			of = fopen(path, "w");
749 		if (of == NULL) {
750 			warn("%s: open()", path);
751 			goto failure;
752 		}
753 	}
754 	count = url->offset;
755 
756 	/* start the counter */
757 	stat_start(&xs, path, us.size, count);
758 
759 	sigalrm = siginfo = sigint = 0;
760 
761 	/* suck in the data */
762 	setvbuf(f, NULL, _IOFBF, B_size);
763 	signal(SIGINFO, sig_handler);
764 	while (!sigint) {
765 		if (us.size != -1 && us.size - count < B_size &&
766 		    us.size - count >= 0)
767 			size = us.size - count;
768 		else
769 			size = B_size;
770 		if (siginfo) {
771 			stat_end(&xs);
772 			siginfo = 0;
773 		}
774 
775 		if (size == 0)
776 			break;
777 
778 		if (timeout) {
779 			sigalrm_enable = 1;
780 			alarm(timeout);
781 		}
782 		readcnt = fread(buf, 1, size, f);
783 		if (timeout) {
784 			sigalrm_enable = 0;
785 			alarm(0);
786 		}
787 
788 		if (readcnt < size) {
789 			if (ferror(f) && errno == EINTR && !sigint)
790 				clearerr(f);
791 			else if (readcnt == 0)
792 				break;
793 		}
794 
795 		stat_update(&xs, count += readcnt);
796 		for (ptr = buf; readcnt > 0; ptr += wr, readcnt -= wr) {
797 			if ((wr = fwrite(ptr, 1, readcnt, of)) < readcnt) {
798 				if (ferror(of) && errno == EINTR && !sigint)
799 					clearerr(of);
800 				else
801 					break;
802 			}
803 		}
804 		if (readcnt != 0)
805 			break;
806 	}
807 	if (!sigalrm)
808 		sigalrm = ferror(f) && errno == ETIMEDOUT;
809 	signal(SIGINFO, SIG_DFL);
810 
811 	stat_end(&xs);
812 
813 	/*
814 	 * If the transfer timed out or was interrupted, we still want to
815 	 * set the mtime in case the file is not removed (-r or -R) and
816 	 * the user later restarts the transfer.
817 	 */
818  signal:
819 	/* set mtime of local file */
820 	if (!n_flag && us.mtime && !o_stdout && of != NULL &&
821 	    (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) {
822 		struct timeval tv[2];
823 
824 		fflush(of);
825 		tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime);
826 		tv[1].tv_sec = (long)us.mtime;
827 		tv[0].tv_usec = tv[1].tv_usec = 0;
828 		if (utimes(tmppath ? tmppath : path, tv))
829 			warn("%s: utimes()", tmppath ? tmppath : path);
830 	}
831 
832 	/* timed out or interrupted? */
833 	if (sigalrm)
834 		warnx("transfer timed out");
835 	if (sigint) {
836 		warnx("transfer interrupted");
837 		goto failure;
838 	}
839 
840 	/* timeout / interrupt before connection completley established? */
841 	if (f == NULL)
842 		goto failure;
843 
844 	if (!sigalrm) {
845 		/* check the status of our files */
846 		if (ferror(f))
847 			warn("%s", URL);
848 		if (ferror(of))
849 			warn("%s", path);
850 		if (ferror(f) || ferror(of))
851 			goto failure;
852 	}
853 
854 	/* did the transfer complete normally? */
855 	if (us.size != -1 && count < us.size) {
856 		warnx("%s appears to be truncated: %jd/%jd bytes",
857 		    path, (intmax_t)count, (intmax_t)us.size);
858 		goto failure_keep;
859 	}
860 
861 	/*
862 	 * If the transfer timed out and we didn't know how much to
863 	 * expect, assume the worst (i.e. we didn't get all of it)
864 	 */
865 	if (sigalrm && us.size == -1) {
866 		warnx("%s may be truncated", path);
867 		goto failure_keep;
868 	}
869 
870  success:
871 	r = 0;
872 	if (tmppath != NULL && rename(tmppath, path) == -1) {
873 		warn("%s: rename()", path);
874 		goto failure_keep;
875 	}
876 	goto done;
877  failure:
878 	if (of && of != stdout && !R_flag && !r_flag)
879 		if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG))
880 			unlink(tmppath ? tmppath : path);
881 	if (R_flag && tmppath != NULL && sb.st_size == -1)
882 		rename(tmppath, path); /* ignore errors here */
883  failure_keep:
884 	r = -1;
885 	goto done;
886  done:
887 	if (f)
888 		fclose(f);
889 	if (of && of != stdout)
890 		fclose(of);
891 	if (url)
892 		fetchFreeURL(url);
893 	if (tmppath != NULL)
894 		free(tmppath);
895 	return (r);
896 }
897 
898 static void
usage(void)899 usage(void)
900 {
901 	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
902 "usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
903 "       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
904 "       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
905 "       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
906 "       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
907 "       [--user-agent=agent-string] [-w seconds] URL ...",
908 "       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
909 "       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
910 "       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
911 "       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
912 "       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
913 "       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
914 }
915 
916 
917 /*
918  * Entry point
919  */
920 int
main(int argc,char * argv[])921 main(int argc, char *argv[])
922 {
923 	struct stat sb;
924 	struct sigaction sa;
925 	const char *p, *s;
926 	char *end, *q;
927 	int c, e, r;
928 
929 
930 	while ((c = getopt_long(argc, argv,
931 	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:",
932 	    longopts, NULL)) != -1)
933 		switch (c) {
934 		case '1':
935 			once_flag = 1;
936 			break;
937 		case '4':
938 			family = PF_INET;
939 			break;
940 		case '6':
941 			family = PF_INET6;
942 			break;
943 		case 'A':
944 			A_flag = 1;
945 			break;
946 		case 'a':
947 			a_flag = 1;
948 			break;
949 		case 'B':
950 			B_size = (off_t)strtol(optarg, &end, 10);
951 			if (*optarg == '\0' || *end != '\0')
952 				errx(1, "invalid buffer size (%s)", optarg);
953 			break;
954 		case 'b':
955 			warnx("warning: the -b option is deprecated");
956 			b_flag = 1;
957 			break;
958 		case 'c':
959 			c_dirname = optarg;
960 			break;
961 		case 'd':
962 			d_flag = 1;
963 			break;
964 		case 'F':
965 			F_flag = 1;
966 			break;
967 		case 'f':
968 			f_filename = optarg;
969 			break;
970 		case 'H':
971 			warnx("the -H option is now implicit, "
972 			    "use -U to disable");
973 			break;
974 		case 'h':
975 			h_hostname = optarg;
976 			break;
977 		case 'i':
978 			i_flag = 1;
979 			i_filename = optarg;
980 			break;
981 		case 'l':
982 			l_flag = 1;
983 			break;
984 		case 'o':
985 			o_flag = 1;
986 			o_filename = optarg;
987 			break;
988 		case 'M':
989 		case 'm':
990 			if (r_flag)
991 				errx(1, "the -m and -r flags "
992 				    "are mutually exclusive");
993 			m_flag = 1;
994 			break;
995 		case 'N':
996 			N_filename = optarg;
997 			break;
998 		case 'n':
999 			n_flag = 1;
1000 			break;
1001 		case 'P':
1002 		case 'p':
1003 			p_flag = 1;
1004 			break;
1005 		case 'q':
1006 			v_level = 0;
1007 			break;
1008 		case 'R':
1009 			R_flag = 1;
1010 			break;
1011 		case 'r':
1012 			if (m_flag)
1013 				errx(1, "the -m and -r flags "
1014 				    "are mutually exclusive");
1015 			r_flag = 1;
1016 			break;
1017 		case 'S':
1018 			S_size = (off_t)strtol(optarg, &end, 10);
1019 			if (*optarg == '\0' || *end != '\0')
1020 				errx(1, "invalid size (%s)", optarg);
1021 			break;
1022 		case 's':
1023 			s_flag = 1;
1024 			break;
1025 		case 'T':
1026 			T_secs = strtol(optarg, &end, 10);
1027 			if (*optarg == '\0' || *end != '\0')
1028 				errx(1, "invalid timeout (%s)", optarg);
1029 			break;
1030 		case 't':
1031 			t_flag = 1;
1032 			warnx("warning: the -t option is deprecated");
1033 			break;
1034 		case 'U':
1035 			U_flag = 1;
1036 			break;
1037 		case 'v':
1038 			v_level++;
1039 			break;
1040 		case 'w':
1041 			a_flag = 1;
1042 			w_secs = strtol(optarg, &end, 10);
1043 			if (*optarg == '\0' || *end != '\0')
1044 				errx(1, "invalid delay (%s)", optarg);
1045 			break;
1046 		case OPTION_BIND_ADDRESS:
1047 			setenv("FETCH_BIND_ADDRESS", optarg, 1);
1048 			break;
1049 		case OPTION_NO_FTP_PASSIVE_MODE:
1050 			setenv("FTP_PASSIVE_MODE", "no", 1);
1051 			break;
1052 		case OPTION_HTTP_REFERER:
1053 			setenv("HTTP_REFERER", optarg, 1);
1054 			break;
1055 		case OPTION_HTTP_USER_AGENT:
1056 			setenv("HTTP_USER_AGENT", optarg, 1);
1057 			break;
1058 		case OPTION_NO_PROXY:
1059 			setenv("NO_PROXY", optarg, 1);
1060 			break;
1061 		case OPTION_SSL_CA_CERT_FILE:
1062 			setenv("SSL_CA_CERT_FILE", optarg, 1);
1063 			break;
1064 		case OPTION_SSL_CA_CERT_PATH:
1065 			setenv("SSL_CA_CERT_PATH", optarg, 1);
1066 			break;
1067 		case OPTION_SSL_CLIENT_CERT_FILE:
1068 			setenv("SSL_CLIENT_CERT_FILE", optarg, 1);
1069 			break;
1070 		case OPTION_SSL_CLIENT_KEY_FILE:
1071 			setenv("SSL_CLIENT_KEY_FILE", optarg, 1);
1072 			break;
1073 		case OPTION_SSL_CRL_FILE:
1074 			setenv("SSL_CLIENT_CRL_FILE", optarg, 1);
1075 			break;
1076 		case OPTION_SSL_NO_SSL3:
1077 			setenv("SSL_NO_SSL3", "", 1);
1078 			break;
1079 		case OPTION_SSL_NO_TLS1:
1080 			setenv("SSL_NO_TLS1", "", 1);
1081 			break;
1082 		case OPTION_SSL_NO_VERIFY_HOSTNAME:
1083 			setenv("SSL_NO_VERIFY_HOSTNAME", "", 1);
1084 			break;
1085 		case OPTION_SSL_NO_VERIFY_PEER:
1086 			setenv("SSL_NO_VERIFY_PEER", "", 1);
1087 			break;
1088 		default:
1089 			usage();
1090 			exit(1);
1091 		}
1092 
1093 	argc -= optind;
1094 	argv += optind;
1095 
1096 	if (h_hostname || f_filename || c_dirname) {
1097 		if (!h_hostname || !f_filename || argc) {
1098 			usage();
1099 			exit(1);
1100 		}
1101 		/* XXX this is a hack. */
1102 		if (strcspn(h_hostname, "@:/") != strlen(h_hostname))
1103 			errx(1, "invalid hostname");
1104 		if (asprintf(argv, "ftp://%s/%s/%s", h_hostname,
1105 		    c_dirname ? c_dirname : "", f_filename) == -1)
1106 			errx(1, "%s", strerror(ENOMEM));
1107 		argc++;
1108 	}
1109 
1110 	if (!argc) {
1111 		usage();
1112 		exit(1);
1113 	}
1114 
1115 	/* allocate buffer */
1116 	if (B_size < MINBUFSIZE)
1117 		B_size = MINBUFSIZE;
1118 	if ((buf = malloc(B_size)) == NULL)
1119 		errx(1, "%s", strerror(ENOMEM));
1120 
1121 	/* timeouts */
1122 	if ((s = getenv("FTP_TIMEOUT")) != NULL) {
1123 		ftp_timeout = strtol(s, &end, 10);
1124 		if (*s == '\0' || *end != '\0' || ftp_timeout < 0) {
1125 			warnx("FTP_TIMEOUT (%s) is not a positive integer", s);
1126 			ftp_timeout = 0;
1127 		}
1128 	}
1129 	if ((s = getenv("HTTP_TIMEOUT")) != NULL) {
1130 		http_timeout = strtol(s, &end, 10);
1131 		if (*s == '\0' || *end != '\0' || http_timeout < 0) {
1132 			warnx("HTTP_TIMEOUT (%s) is not a positive integer", s);
1133 			http_timeout = 0;
1134 		}
1135 	}
1136 
1137 	/* signal handling */
1138 	sa.sa_flags = 0;
1139 	sa.sa_handler = sig_handler;
1140 	sigemptyset(&sa.sa_mask);
1141 	sigaction(SIGALRM, &sa, NULL);
1142 	sa.sa_flags = SA_RESETHAND;
1143 	sigaction(SIGINT, &sa, NULL);
1144 	fetchRestartCalls = 0;
1145 
1146 	/* output file */
1147 	if (o_flag) {
1148 		if (strcmp(o_filename, "-") == 0) {
1149 			o_stdout = 1;
1150 		} else if (stat(o_filename, &sb) == -1) {
1151 			if (errno == ENOENT) {
1152 				if (argc > 1)
1153 					errx(1, "%s is not a directory",
1154 					    o_filename);
1155 			} else {
1156 				err(1, "%s", o_filename);
1157 			}
1158 		} else {
1159 			if (sb.st_mode & S_IFDIR)
1160 				o_directory = 1;
1161 		}
1162 	}
1163 
1164 	/* check if output is to a tty (for progress report) */
1165 	v_tty = isatty(STDERR_FILENO);
1166 	v_progress = v_tty && v_level > 0;
1167 	if (v_progress)
1168 		pgrp = getpgrp();
1169 
1170 	r = 0;
1171 
1172 	/* authentication */
1173 	if (v_tty)
1174 		fetchAuthMethod = query_auth;
1175 	if (N_filename != NULL) {
1176 		if (setenv("NETRC", N_filename, 1) == -1)
1177 			err(1, "setenv: cannot set NETRC=%s", N_filename);
1178 	}
1179 
1180 	while (argc) {
1181 		if ((p = strrchr(*argv, '/')) == NULL)
1182 			p = *argv;
1183 		else
1184 			p++;
1185 
1186 		if (!*p)
1187 			p = "fetch.out";
1188 
1189 		fetchLastErrCode = 0;
1190 
1191 		if (o_flag) {
1192 			if (o_stdout) {
1193 				e = fetch(*argv, "-");
1194 			} else if (o_directory) {
1195 				asprintf(&q, "%s/%s", o_filename, p);
1196 				e = fetch(*argv, q);
1197 				free(q);
1198 			} else {
1199 				e = fetch(*argv, o_filename);
1200 			}
1201 		} else {
1202 			e = fetch(*argv, p);
1203 		}
1204 
1205 		if (sigint)
1206 			kill(getpid(), SIGINT);
1207 
1208 		if (e == 0 && once_flag)
1209 			exit(0);
1210 
1211 		if (e) {
1212 			r = 1;
1213 			if ((fetchLastErrCode
1214 			    && fetchLastErrCode != FETCH_UNAVAIL
1215 			    && fetchLastErrCode != FETCH_MOVED
1216 			    && fetchLastErrCode != FETCH_URL
1217 			    && fetchLastErrCode != FETCH_RESOLV
1218 			    && fetchLastErrCode != FETCH_UNKNOWN)) {
1219 				if (w_secs && v_level)
1220 					fprintf(stderr, "Waiting %ld seconds "
1221 					    "before retrying\n", w_secs);
1222 				if (w_secs)
1223 					sleep(w_secs);
1224 				if (a_flag)
1225 					continue;
1226 			}
1227 		}
1228 
1229 		argc--, argv++;
1230 	}
1231 
1232 	exit(r);
1233 }
1234