xref: /dragonfly/lib/libfetch/fetch.c (revision a9783bc6) 
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1998-2004 Dag-Erling Smørgrav
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer
12  *    in this position and unchanged.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. The name of the author may not be used to endorse or promote products
17  *    derived from this software without specific prior written permission
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  *
30  * $FreeBSD: head/lib/libfetch/fetch.c 341013 2018-11-27 10:45:14Z des $
31  */
32 
33 #include <sys/param.h>
34 
35 #include <netinet/in.h>
36 
37 #include <errno.h>
38 #include <ctype.h>
39 #include <stdio.h>
40 #include <stdlib.h>
41 #include <string.h>
42 
43 #include "fetch.h"
44 #include "common.h"
45 
46 auth_t	 fetchAuthMethod;
47 int	 fetchLastErrCode;
48 char	 fetchLastErrString[MAXERRSTRING];
49 int	 fetchTimeout;
50 int	 fetchRestartCalls = 1;
51 int	 fetchDebug;
52 
53 
54 /*** Local data **************************************************************/
55 
56 /*
57  * Error messages for parser errors
58  */
59 #define URL_MALFORMED		1
60 #define URL_BAD_SCHEME		2
61 #define URL_BAD_PORT		3
62 static struct fetcherr url_errlist[] = {
63 	{ URL_MALFORMED,	FETCH_URL,	"Malformed URL" },
64 	{ URL_BAD_SCHEME,	FETCH_URL,	"Invalid URL scheme" },
65 	{ URL_BAD_PORT,		FETCH_URL,	"Invalid server port" },
66 	{ -1,			FETCH_UNKNOWN,	"Unknown parser error" }
67 };
68 
69 
70 /*** Public API **************************************************************/
71 
72 /*
73  * Select the appropriate protocol for the URL scheme, and return a
74  * read-only stream connected to the document referenced by the URL.
75  * Also fill out the struct url_stat.
76  */
77 FILE *
78 fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
79 {
80 
81 	if (us != NULL) {
82 		us->size = -1;
83 		us->atime = us->mtime = 0;
84 	}
85 	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
86 		return (fetchXGetFile(URL, us, flags));
87 	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
88 		return (fetchXGetFTP(URL, us, flags));
89 	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
90 		return (fetchXGetHTTP(URL, us, flags));
91 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
92 		return (fetchXGetHTTP(URL, us, flags));
93 	url_seterr(URL_BAD_SCHEME);
94 	return (NULL);
95 }
96 
97 /*
98  * Select the appropriate protocol for the URL scheme, and return a
99  * read-only stream connected to the document referenced by the URL.
100  */
101 FILE *
102 fetchGet(struct url *URL, const char *flags)
103 {
104 	return (fetchXGet(URL, NULL, flags));
105 }
106 
107 /*
108  * Select the appropriate protocol for the URL scheme, and return a
109  * write-only stream connected to the document referenced by the URL.
110  */
111 FILE *
112 fetchPut(struct url *URL, const char *flags)
113 {
114 
115 	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
116 		return (fetchPutFile(URL, flags));
117 	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
118 		return (fetchPutFTP(URL, flags));
119 	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
120 		return (fetchPutHTTP(URL, flags));
121 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
122 		return (fetchPutHTTP(URL, flags));
123 	url_seterr(URL_BAD_SCHEME);
124 	return (NULL);
125 }
126 
127 /*
128  * Select the appropriate protocol for the URL scheme, and return the
129  * size of the document referenced by the URL if it exists.
130  */
131 int
132 fetchStat(struct url *URL, struct url_stat *us, const char *flags)
133 {
134 
135 	if (us != NULL) {
136 		us->size = -1;
137 		us->atime = us->mtime = 0;
138 	}
139 	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
140 		return (fetchStatFile(URL, us, flags));
141 	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
142 		return (fetchStatFTP(URL, us, flags));
143 	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
144 		return (fetchStatHTTP(URL, us, flags));
145 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
146 		return (fetchStatHTTP(URL, us, flags));
147 	url_seterr(URL_BAD_SCHEME);
148 	return (-1);
149 }
150 
151 /*
152  * Select the appropriate protocol for the URL scheme, and return a
153  * list of files in the directory pointed to by the URL.
154  */
155 struct url_ent *
156 fetchList(struct url *URL, const char *flags)
157 {
158 
159 	if (strcmp(URL->scheme, SCHEME_FILE) == 0)
160 		return (fetchListFile(URL, flags));
161 	else if (strcmp(URL->scheme, SCHEME_FTP) == 0)
162 		return (fetchListFTP(URL, flags));
163 	else if (strcmp(URL->scheme, SCHEME_HTTP) == 0)
164 		return (fetchListHTTP(URL, flags));
165 	else if (strcmp(URL->scheme, SCHEME_HTTPS) == 0)
166 		return (fetchListHTTP(URL, flags));
167 	url_seterr(URL_BAD_SCHEME);
168 	return (NULL);
169 }
170 
171 /*
172  * Attempt to parse the given URL; if successful, call fetchXGet().
173  */
174 FILE *
175 fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
176 {
177 	struct url *u;
178 	FILE *f;
179 
180 	if ((u = fetchParseURL(URL)) == NULL)
181 		return (NULL);
182 
183 	f = fetchXGet(u, us, flags);
184 
185 	fetchFreeURL(u);
186 	return (f);
187 }
188 
189 /*
190  * Attempt to parse the given URL; if successful, call fetchGet().
191  */
192 FILE *
193 fetchGetURL(const char *URL, const char *flags)
194 {
195 	return (fetchXGetURL(URL, NULL, flags));
196 }
197 
198 /*
199  * Attempt to parse the given URL; if successful, call fetchPut().
200  */
201 FILE *
202 fetchPutURL(const char *URL, const char *flags)
203 {
204 	struct url *u;
205 	FILE *f;
206 
207 	if ((u = fetchParseURL(URL)) == NULL)
208 		return (NULL);
209 
210 	f = fetchPut(u, flags);
211 
212 	fetchFreeURL(u);
213 	return (f);
214 }
215 
216 /*
217  * Attempt to parse the given URL; if successful, call fetchStat().
218  */
219 int
220 fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
221 {
222 	struct url *u;
223 	int s;
224 
225 	if ((u = fetchParseURL(URL)) == NULL)
226 		return (-1);
227 
228 	s = fetchStat(u, us, flags);
229 
230 	fetchFreeURL(u);
231 	return (s);
232 }
233 
234 /*
235  * Attempt to parse the given URL; if successful, call fetchList().
236  */
237 struct url_ent *
238 fetchListURL(const char *URL, const char *flags)
239 {
240 	struct url *u;
241 	struct url_ent *ue;
242 
243 	if ((u = fetchParseURL(URL)) == NULL)
244 		return (NULL);
245 
246 	ue = fetchList(u, flags);
247 
248 	fetchFreeURL(u);
249 	return (ue);
250 }
251 
252 /*
253  * Make a URL
254  */
255 struct url *
256 fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
257     const char *user, const char *pwd)
258 {
259 	struct url *u;
260 
261 	if (!scheme || (!host && !doc)) {
262 		url_seterr(URL_MALFORMED);
263 		return (NULL);
264 	}
265 
266 	if (port < 0 || port > 65535) {
267 		url_seterr(URL_BAD_PORT);
268 		return (NULL);
269 	}
270 
271 	/* allocate struct url */
272 	if ((u = calloc(1, sizeof(*u))) == NULL) {
273 		fetch_syserr();
274 		return (NULL);
275 	}
276 	u->netrcfd = -1;
277 
278 	if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
279 		fetch_syserr();
280 		free(u);
281 		return (NULL);
282 	}
283 
284 #define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
285 	seturl(scheme);
286 	seturl(host);
287 	seturl(user);
288 	seturl(pwd);
289 #undef seturl
290 	u->port = port;
291 
292 	return (u);
293 }
294 
295 /*
296  * Return value of the given hex digit.
297  */
298 static int
299 fetch_hexval(char ch)
300 {
301 
302 	if (ch >= '0' && ch <= '9')
303 		return (ch - '0');
304 	else if (ch >= 'a' && ch <= 'f')
305 		return (ch - 'a' + 10);
306 	else if (ch >= 'A' && ch <= 'F')
307 		return (ch - 'A' + 10);
308 	return (-1);
309 }
310 
311 /*
312  * Decode percent-encoded URL component from src into dst, stopping at end
313  * of string, or at @ or : separators.  Returns a pointer to the unhandled
314  * part of the input string (null terminator, @, or :).  No terminator is
315  * written to dst (it is the caller's responsibility).
316  */
317 static const char *
318 fetch_pctdecode(char *dst, const char *src, size_t dlen)
319 {
320 	int d1, d2;
321 	char c;
322 	const char *s;
323 
324 	for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) {
325 		if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 &&
326 		    (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) {
327 			c = d1 << 4 | d2;
328 			s += 2;
329 		} else {
330 			c = *s;
331 		}
332 		if (dlen-- > 0)
333 			*dst++ = c;
334 	}
335 	return (s);
336 }
337 
338 /*
339  * Split an URL into components. URL syntax is:
340  * [method:/][/[user[:pwd]@]host[:port]/][document]
341  * This almost, but not quite, RFC1738 URL syntax.
342  */
343 struct url *
344 fetchParseURL(const char *URL)
345 {
346 	char *doc;
347 	const char *p, *q;
348 	struct url *u;
349 	int i, n;
350 
351 	/* allocate struct url */
352 	if ((u = calloc(1, sizeof(*u))) == NULL) {
353 		fetch_syserr();
354 		return (NULL);
355 	}
356 	u->netrcfd = -1;
357 
358 	/* scheme name */
359 	if ((p = strstr(URL, ":/"))) {
360                 if (p - URL > URL_SCHEMELEN)
361                         goto ouch;
362                 for (i = 0; URL + i < p; i++)
363                         u->scheme[i] = tolower((unsigned char)URL[i]);
364 		URL = ++p;
365 		/*
366 		 * Only one slash: no host, leave slash as part of document
367 		 * Two slashes: host follows, strip slashes
368 		 */
369 		if (URL[1] == '/')
370 			URL = (p += 2);
371 	} else {
372 		p = URL;
373 	}
374 	if (!*URL || *URL == '/' || *URL == '.' ||
375 	    (u->scheme[0] == '\0' &&
376 		strchr(URL, '/') == NULL && strchr(URL, ':') == NULL))
377 		goto nohost;
378 
379 	p = strpbrk(URL, "/@");
380 	if (p && *p == '@') {
381 		/* username */
382 		q = fetch_pctdecode(u->user, URL, URL_USERLEN);
383 
384 		/* password */
385 		if (*q == ':')
386 			q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN);
387 
388 		p++;
389 	} else {
390 		p = URL;
391 	}
392 
393 	/* hostname */
394 	if (*p == '[') {
395 		q = p + 1 + strspn(p + 1, ":0123456789ABCDEFabcdef");
396 		if (*q++ != ']')
397 			goto ouch;
398 	} else {
399 		/* valid characters in a DNS name */
400 		q = p + strspn(p, "-." "0123456789"
401 		    "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "_"
402 		    "abcdefghijklmnopqrstuvwxyz");
403 	}
404 	if ((*q != '\0' && *q != '/' && *q != ':') || q - p > MAXHOSTNAMELEN)
405 		goto ouch;
406 	for (i = 0; p + i < q; i++)
407 		u->host[i] = tolower((unsigned char)p[i]);
408 	u->host[i] = '\0';
409 	p = q;
410 
411 	/* port */
412 	if (*p == ':') {
413 		for (n = 0, q = ++p; *q && (*q != '/'); q++) {
414 			if (*q >= '0' && *q <= '9' && n < INT_MAX / 10) {
415 				n = n * 10 + (*q - '0');
416 			} else {
417 				/* invalid port */
418 				url_seterr(URL_BAD_PORT);
419 				goto ouch;
420 			}
421 		}
422 		if (n < 1 || n > IPPORT_MAX)
423 			goto ouch;
424 		u->port = n;
425 		p = q;
426 	}
427 
428 nohost:
429 	/* document */
430 	if (!*p)
431 		p = "/";
432 
433 	if (strcmp(u->scheme, SCHEME_HTTP) == 0 ||
434 	    strcmp(u->scheme, SCHEME_HTTPS) == 0) {
435 		const char hexnums[] = "0123456789abcdef";
436 
437 		/* percent-escape whitespace. */
438 		if ((doc = malloc(strlen(p) * 3 + 1)) == NULL) {
439 			fetch_syserr();
440 			goto ouch;
441 		}
442 		u->doc = doc;
443 		while (*p != '\0') {
444 			if (!isspace((unsigned char)*p)) {
445 				*doc++ = *p++;
446 			} else {
447 				*doc++ = '%';
448 				*doc++ = hexnums[((unsigned int)*p) >> 4];
449 				*doc++ = hexnums[((unsigned int)*p) & 0xf];
450 				p++;
451 			}
452 		}
453 		*doc = '\0';
454 	} else if ((u->doc = strdup(p)) == NULL) {
455 		fetch_syserr();
456 		goto ouch;
457 	}
458 
459 	DEBUGF("scheme:   \"%s\"\n"
460 	    "user:     \"%s\"\n"
461 	    "password: \"%s\"\n"
462 	    "host:     \"%s\"\n"
463 	    "port:     \"%d\"\n"
464 	    "document: \"%s\"\n",
465 	    u->scheme, u->user, u->pwd,
466 	    u->host, u->port, u->doc);
467 
468 	return (u);
469 
470 ouch:
471 	free(u);
472 	return (NULL);
473 }
474 
475 /*
476  * Free a URL
477  */
478 void
479 fetchFreeURL(struct url *u)
480 {
481 	free(u->doc);
482 	free(u);
483 }
484