xref: /freebsd/sys/kern/subr_uio.c (revision a3557ef0) 
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1991, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * (c) UNIX System Laboratories, Inc.
7  * All or some portions of this file are derived from material licensed
8  * to the University of California by American Telephone and Telegraph
9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10  * the permission of UNIX System Laboratories, Inc.
11  *
12  * Copyright (c) 2014 The FreeBSD Foundation
13  *
14  * Portions of this software were developed by Konstantin Belousov
15  * under sponsorship from the FreeBSD Foundation.
16  *
17  * Redistribution and use in source and binary forms, with or without
18  * modification, are permitted provided that the following conditions
19  * are met:
20  * 1. Redistributions of source code must retain the above copyright
21  *    notice, this list of conditions and the following disclaimer.
22  * 2. Redistributions in binary form must reproduce the above copyright
23  *    notice, this list of conditions and the following disclaimer in the
24  *    documentation and/or other materials provided with the distribution.
25  * 3. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  *
41  *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
42  */
43 
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/limits.h>
51 #include <sys/lock.h>
52 #include <sys/mman.h>
53 #include <sys/proc.h>
54 #include <sys/resourcevar.h>
55 #include <sys/rwlock.h>
56 #include <sys/sched.h>
57 #include <sys/sysctl.h>
58 #include <sys/vnode.h>
59 
60 #include <vm/vm.h>
61 #include <vm/vm_param.h>
62 #include <vm/vm_extern.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_pageout.h>
65 #include <vm/vm_map.h>
66 
67 #include <machine/bus.h>
68 
69 SYSCTL_INT(_kern, KERN_IOV_MAX, iov_max, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, UIO_MAXIOV,
70 	"Maximum number of elements in an I/O vector; sysconf(_SC_IOV_MAX)");
71 
72 static int uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault);
73 
74 int
75 copyin_nofault(const void *udaddr, void *kaddr, size_t len)
76 {
77 	int error, save;
78 
79 	save = vm_fault_disable_pagefaults();
80 	error = copyin(udaddr, kaddr, len);
81 	vm_fault_enable_pagefaults(save);
82 	return (error);
83 }
84 
85 int
86 copyout_nofault(const void *kaddr, void *udaddr, size_t len)
87 {
88 	int error, save;
89 
90 	save = vm_fault_disable_pagefaults();
91 	error = copyout(kaddr, udaddr, len);
92 	vm_fault_enable_pagefaults(save);
93 	return (error);
94 }
95 
96 #define	PHYS_PAGE_COUNT(len)	(howmany(len, PAGE_SIZE) + 1)
97 
98 int
99 physcopyin(void *src, vm_paddr_t dst, size_t len)
100 {
101 	vm_page_t m[PHYS_PAGE_COUNT(len)];
102 	struct iovec iov[1];
103 	struct uio uio;
104 	int i;
105 
106 	iov[0].iov_base = src;
107 	iov[0].iov_len = len;
108 	uio.uio_iov = iov;
109 	uio.uio_iovcnt = 1;
110 	uio.uio_offset = 0;
111 	uio.uio_resid = len;
112 	uio.uio_segflg = UIO_SYSSPACE;
113 	uio.uio_rw = UIO_WRITE;
114 	for (i = 0; i < PHYS_PAGE_COUNT(len); i++, dst += PAGE_SIZE)
115 		m[i] = PHYS_TO_VM_PAGE(dst);
116 	return (uiomove_fromphys(m, dst & PAGE_MASK, len, &uio));
117 }
118 
119 int
120 physcopyout(vm_paddr_t src, void *dst, size_t len)
121 {
122 	vm_page_t m[PHYS_PAGE_COUNT(len)];
123 	struct iovec iov[1];
124 	struct uio uio;
125 	int i;
126 
127 	iov[0].iov_base = dst;
128 	iov[0].iov_len = len;
129 	uio.uio_iov = iov;
130 	uio.uio_iovcnt = 1;
131 	uio.uio_offset = 0;
132 	uio.uio_resid = len;
133 	uio.uio_segflg = UIO_SYSSPACE;
134 	uio.uio_rw = UIO_READ;
135 	for (i = 0; i < PHYS_PAGE_COUNT(len); i++, src += PAGE_SIZE)
136 		m[i] = PHYS_TO_VM_PAGE(src);
137 	return (uiomove_fromphys(m, src & PAGE_MASK, len, &uio));
138 }
139 
140 #undef PHYS_PAGE_COUNT
141 
142 int
143 physcopyin_vlist(bus_dma_segment_t *src, off_t offset, vm_paddr_t dst,
144     size_t len)
145 {
146 	size_t seg_len;
147 	int error;
148 
149 	error = 0;
150 	while (offset >= src->ds_len) {
151 		offset -= src->ds_len;
152 		src++;
153 	}
154 
155 	while (len > 0 && error == 0) {
156 		seg_len = MIN(src->ds_len - offset, len);
157 		error = physcopyin((void *)(uintptr_t)(src->ds_addr + offset),
158 		    dst, seg_len);
159 		offset = 0;
160 		src++;
161 		len -= seg_len;
162 		dst += seg_len;
163 	}
164 
165 	return (error);
166 }
167 
168 int
169 physcopyout_vlist(vm_paddr_t src, bus_dma_segment_t *dst, off_t offset,
170     size_t len)
171 {
172 	size_t seg_len;
173 	int error;
174 
175 	error = 0;
176 	while (offset >= dst->ds_len) {
177 		offset -= dst->ds_len;
178 		dst++;
179 	}
180 
181 	while (len > 0 && error == 0) {
182 		seg_len = MIN(dst->ds_len - offset, len);
183 		error = physcopyout(src, (void *)(uintptr_t)(dst->ds_addr +
184 		    offset), seg_len);
185 		offset = 0;
186 		dst++;
187 		len -= seg_len;
188 		src += seg_len;
189 	}
190 
191 	return (error);
192 }
193 
194 int
195 uiomove(void *cp, int n, struct uio *uio)
196 {
197 
198 	return (uiomove_faultflag(cp, n, uio, 0));
199 }
200 
201 int
202 uiomove_nofault(void *cp, int n, struct uio *uio)
203 {
204 
205 	return (uiomove_faultflag(cp, n, uio, 1));
206 }
207 
208 static int
209 uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
210 {
211 	struct iovec *iov;
212 	size_t cnt;
213 	int error, newflags, save;
214 
215 	save = error = 0;
216 
217 	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
218 	    ("uiomove: mode"));
219 	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
220 	    ("uiomove proc"));
221 
222 	if (uio->uio_segflg == UIO_USERSPACE) {
223 		newflags = TDP_DEADLKTREAT;
224 		if (nofault) {
225 			/*
226 			 * Fail if a non-spurious page fault occurs.
227 			 */
228 			newflags |= TDP_NOFAULTING | TDP_RESETSPUR;
229 		} else {
230 			WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
231 			    "Calling uiomove()");
232 		}
233 		save = curthread_pflags_set(newflags);
234 	} else {
235 		KASSERT(nofault == 0, ("uiomove: nofault"));
236 	}
237 
238 	while (n > 0 && uio->uio_resid) {
239 		iov = uio->uio_iov;
240 		cnt = iov->iov_len;
241 		if (cnt == 0) {
242 			uio->uio_iov++;
243 			uio->uio_iovcnt--;
244 			continue;
245 		}
246 		if (cnt > n)
247 			cnt = n;
248 
249 		switch (uio->uio_segflg) {
250 
251 		case UIO_USERSPACE:
252 			maybe_yield();
253 			if (uio->uio_rw == UIO_READ)
254 				error = copyout(cp, iov->iov_base, cnt);
255 			else
256 				error = copyin(iov->iov_base, cp, cnt);
257 			if (error)
258 				goto out;
259 			break;
260 
261 		case UIO_SYSSPACE:
262 			if (uio->uio_rw == UIO_READ)
263 				bcopy(cp, iov->iov_base, cnt);
264 			else
265 				bcopy(iov->iov_base, cp, cnt);
266 			break;
267 		case UIO_NOCOPY:
268 			break;
269 		}
270 		iov->iov_base = (char *)iov->iov_base + cnt;
271 		iov->iov_len -= cnt;
272 		uio->uio_resid -= cnt;
273 		uio->uio_offset += cnt;
274 		cp = (char *)cp + cnt;
275 		n -= cnt;
276 	}
277 out:
278 	if (save)
279 		curthread_pflags_restore(save);
280 	return (error);
281 }
282 
283 /*
284  * Wrapper for uiomove() that validates the arguments against a known-good
285  * kernel buffer.  Currently, uiomove accepts a signed (n) argument, which
286  * is almost definitely a bad thing, so we catch that here as well.  We
287  * return a runtime failure, but it might be desirable to generate a runtime
288  * assertion failure instead.
289  */
290 int
291 uiomove_frombuf(void *buf, int buflen, struct uio *uio)
292 {
293 	size_t offset, n;
294 
295 	if (uio->uio_offset < 0 || uio->uio_resid < 0 ||
296 	    (offset = uio->uio_offset) != uio->uio_offset)
297 		return (EINVAL);
298 	if (buflen <= 0 || offset >= buflen)
299 		return (0);
300 	if ((n = buflen - offset) > IOSIZE_MAX)
301 		return (EINVAL);
302 	return (uiomove((char *)buf + offset, n, uio));
303 }
304 
305 /*
306  * Give next character to user as result of read.
307  */
308 int
309 ureadc(int c, struct uio *uio)
310 {
311 	struct iovec *iov;
312 	char *iov_base;
313 
314 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
315 	    "Calling ureadc()");
316 
317 again:
318 	if (uio->uio_iovcnt == 0 || uio->uio_resid == 0)
319 		panic("ureadc");
320 	iov = uio->uio_iov;
321 	if (iov->iov_len == 0) {
322 		uio->uio_iovcnt--;
323 		uio->uio_iov++;
324 		goto again;
325 	}
326 	switch (uio->uio_segflg) {
327 
328 	case UIO_USERSPACE:
329 		if (subyte(iov->iov_base, c) < 0)
330 			return (EFAULT);
331 		break;
332 
333 	case UIO_SYSSPACE:
334 		iov_base = iov->iov_base;
335 		*iov_base = c;
336 		break;
337 
338 	case UIO_NOCOPY:
339 		break;
340 	}
341 	iov->iov_base = (char *)iov->iov_base + 1;
342 	iov->iov_len--;
343 	uio->uio_resid--;
344 	uio->uio_offset++;
345 	return (0);
346 }
347 
348 int
349 copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov, int error)
350 {
351 	u_int iovlen;
352 
353 	*iov = NULL;
354 	if (iovcnt > UIO_MAXIOV)
355 		return (error);
356 	iovlen = iovcnt * sizeof (struct iovec);
357 	*iov = malloc(iovlen, M_IOV, M_WAITOK);
358 	error = copyin(iovp, *iov, iovlen);
359 	if (error) {
360 		free(*iov, M_IOV);
361 		*iov = NULL;
362 	}
363 	return (error);
364 }
365 
366 int
367 copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop)
368 {
369 	struct iovec *iov;
370 	struct uio *uio;
371 	u_int iovlen;
372 	int error, i;
373 
374 	*uiop = NULL;
375 	if (iovcnt > UIO_MAXIOV)
376 		return (EINVAL);
377 	iovlen = iovcnt * sizeof (struct iovec);
378 	uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
379 	iov = (struct iovec *)(uio + 1);
380 	error = copyin(iovp, iov, iovlen);
381 	if (error) {
382 		free(uio, M_IOV);
383 		return (error);
384 	}
385 	uio->uio_iov = iov;
386 	uio->uio_iovcnt = iovcnt;
387 	uio->uio_segflg = UIO_USERSPACE;
388 	uio->uio_offset = -1;
389 	uio->uio_resid = 0;
390 	for (i = 0; i < iovcnt; i++) {
391 		if (iov->iov_len > IOSIZE_MAX - uio->uio_resid) {
392 			free(uio, M_IOV);
393 			return (EINVAL);
394 		}
395 		uio->uio_resid += iov->iov_len;
396 		iov++;
397 	}
398 	*uiop = uio;
399 	return (0);
400 }
401 
402 struct uio *
403 cloneuio(struct uio *uiop)
404 {
405 	struct uio *uio;
406 	int iovlen;
407 
408 	iovlen = uiop->uio_iovcnt * sizeof (struct iovec);
409 	uio = malloc(iovlen + sizeof *uio, M_IOV, M_WAITOK);
410 	*uio = *uiop;
411 	uio->uio_iov = (struct iovec *)(uio + 1);
412 	bcopy(uiop->uio_iov, uio->uio_iov, iovlen);
413 	return (uio);
414 }
415 
416 /*
417  * Map some anonymous memory in user space of size sz, rounded up to the page
418  * boundary.
419  */
420 int
421 copyout_map(struct thread *td, vm_offset_t *addr, size_t sz)
422 {
423 	struct vmspace *vms;
424 	int error;
425 	vm_size_t size;
426 
427 	vms = td->td_proc->p_vmspace;
428 
429 	/*
430 	 * Map somewhere after heap in process memory.
431 	 */
432 	*addr = round_page((vm_offset_t)vms->vm_daddr +
433 	    lim_max(td, RLIMIT_DATA));
434 
435 	/* round size up to page boundary */
436 	size = (vm_size_t)round_page(sz);
437 	if (size == 0)
438 		return (EINVAL);
439 	error = vm_mmap_object(&vms->vm_map, addr, size, VM_PROT_READ |
440 	    VM_PROT_WRITE, VM_PROT_ALL, MAP_PRIVATE | MAP_ANON, NULL, 0,
441 	    FALSE, td);
442 	return (error);
443 }
444 
445 /*
446  * Unmap memory in user space.
447  */
448 int
449 copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz)
450 {
451 	vm_map_t map;
452 	vm_size_t size;
453 
454 	if (sz == 0)
455 		return (0);
456 
457 	map = &td->td_proc->p_vmspace->vm_map;
458 	size = (vm_size_t)round_page(sz);
459 
460 	if (vm_map_remove(map, addr, addr + size) != KERN_SUCCESS)
461 		return (EINVAL);
462 
463 	return (0);
464 }
465 
466 int32_t
467 fuword32(volatile const void *addr)
468 {
469 	int rv;
470 	int32_t val;
471 
472 	rv = fueword32(addr, &val);
473 	return (rv == -1 ? -1 : val);
474 }
475 
476 #ifdef _LP64
477 int64_t
478 fuword64(volatile const void *addr)
479 {
480 	int rv;
481 	int64_t val;
482 
483 	rv = fueword64(addr, &val);
484 	return (rv == -1 ? -1 : val);
485 }
486 #endif /* _LP64 */
487 
488 long
489 fuword(volatile const void *addr)
490 {
491 	long val;
492 	int rv;
493 
494 	rv = fueword(addr, &val);
495 	return (rv == -1 ? -1 : val);
496 }
497 
498 uint32_t
499 casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
500 {
501 	int rv;
502 	uint32_t val;
503 
504 	rv = casueword32(addr, old, &val, new);
505 	return (rv == -1 ? -1 : val);
506 }
507 
508 u_long
509 casuword(volatile u_long *addr, u_long old, u_long new)
510 {
511 	int rv;
512 	u_long val;
513 
514 	rv = casueword(addr, old, &val, new);
515 	return (rv == -1 ? -1 : val);
516 }
517