1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 /* 39 * Copyright (c) 2015 by Chunwei Chen. All rights reserved. 40 */ 41 42 #ifdef _KERNEL 43 44 #include <sys/types.h> 45 #include <sys/uio_impl.h> 46 #include <sys/sysmacros.h> 47 #include <sys/string.h> 48 #include <linux/kmap_compat.h> 49 #include <linux/uaccess.h> 50 51 /* 52 * Move "n" bytes at byte address "p"; "rw" indicates the direction 53 * of the move, and the I/O parameters are provided in "uio", which is 54 * update to reflect the data which was moved. Returns 0 on success or 55 * a non-zero errno on failure. 56 */ 57 static int 58 zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) 59 { 60 const struct iovec *iov = uio->uio_iov; 61 size_t skip = uio->uio_skip; 62 ulong_t cnt; 63 64 while (n && uio->uio_resid) { 65 cnt = MIN(iov->iov_len - skip, n); 66 switch (uio->uio_segflg) { 67 case UIO_USERSPACE: 68 /* 69 * p = kernel data pointer 70 * iov->iov_base = user data pointer 71 */ 72 if (rw == UIO_READ) { 73 if (copy_to_user(iov->iov_base+skip, p, cnt)) 74 return (EFAULT); 75 } else { 76 unsigned long b_left = 0; 77 if (uio->uio_fault_disable) { 78 if (!zfs_access_ok(VERIFY_READ, 79 (iov->iov_base + skip), cnt)) { 80 return (EFAULT); 81 } 82 pagefault_disable(); 83 b_left = 84 __copy_from_user_inatomic(p, 85 (iov->iov_base + skip), cnt); 86 pagefault_enable(); 87 } else { 88 b_left = 89 copy_from_user(p, 90 (iov->iov_base + skip), cnt); 91 } 92 if (b_left > 0) { 93 unsigned long c_bytes = 94 cnt - b_left; 95 uio->uio_skip += c_bytes; 96 ASSERT3U(uio->uio_skip, <, 97 iov->iov_len); 98 uio->uio_resid -= c_bytes; 99 uio->uio_loffset += c_bytes; 100 return (EFAULT); 101 } 102 } 103 break; 104 case UIO_SYSSPACE: 105 if (rw == UIO_READ) 106 memcpy(iov->iov_base + skip, p, cnt); 107 else 108 memcpy(p, iov->iov_base + skip, cnt); 109 break; 110 default: 111 ASSERT(0); 112 } 113 skip += cnt; 114 if (skip == iov->iov_len) { 115 skip = 0; 116 uio->uio_iov = (++iov); 117 uio->uio_iovcnt--; 118 } 119 uio->uio_skip = skip; 120 uio->uio_resid -= cnt; 121 uio->uio_loffset += cnt; 122 p = (caddr_t)p + cnt; 123 n -= cnt; 124 } 125 return (0); 126 } 127 128 static int 129 zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) 130 { 131 const struct bio_vec *bv = uio->uio_bvec; 132 size_t skip = uio->uio_skip; 133 ulong_t cnt; 134 135 while (n && uio->uio_resid) { 136 void *paddr; 137 cnt = MIN(bv->bv_len - skip, n); 138 139 paddr = zfs_kmap_atomic(bv->bv_page); 140 if (rw == UIO_READ) 141 memcpy(paddr + bv->bv_offset + skip, p, cnt); 142 else 143 memcpy(p, paddr + bv->bv_offset + skip, cnt); 144 zfs_kunmap_atomic(paddr); 145 146 skip += cnt; 147 if (skip == bv->bv_len) { 148 skip = 0; 149 uio->uio_bvec = (++bv); 150 uio->uio_iovcnt--; 151 } 152 uio->uio_skip = skip; 153 uio->uio_resid -= cnt; 154 uio->uio_loffset += cnt; 155 p = (caddr_t)p + cnt; 156 n -= cnt; 157 } 158 return (0); 159 } 160 161 #if defined(HAVE_VFS_IOV_ITER) 162 static int 163 zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, 164 boolean_t revert) 165 { 166 size_t cnt = MIN(n, uio->uio_resid); 167 168 if (uio->uio_skip) 169 iov_iter_advance(uio->uio_iter, uio->uio_skip); 170 171 if (rw == UIO_READ) 172 cnt = copy_to_iter(p, cnt, uio->uio_iter); 173 else 174 cnt = copy_from_iter(p, cnt, uio->uio_iter); 175 176 /* 177 * When operating on a full pipe no bytes are processed. 178 * In which case return EFAULT which is converted to EAGAIN 179 * by the kernel's generic_file_splice_read() function. 180 */ 181 if (cnt == 0) 182 return (EFAULT); 183 184 /* 185 * Revert advancing the uio_iter. This is set by zfs_uiocopy() 186 * to avoid consuming the uio and its iov_iter structure. 187 */ 188 if (revert) 189 iov_iter_revert(uio->uio_iter, cnt); 190 191 uio->uio_resid -= cnt; 192 uio->uio_loffset += cnt; 193 194 return (0); 195 } 196 #endif 197 198 int 199 zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio) 200 { 201 if (uio->uio_segflg == UIO_BVEC) 202 return (zfs_uiomove_bvec(p, n, rw, uio)); 203 #if defined(HAVE_VFS_IOV_ITER) 204 else if (uio->uio_segflg == UIO_ITER) 205 return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE)); 206 #endif 207 else 208 return (zfs_uiomove_iov(p, n, rw, uio)); 209 } 210 EXPORT_SYMBOL(zfs_uiomove); 211 212 /* 213 * Fault in the pages of the first n bytes specified by the uio structure. 214 * 1 byte in each page is touched and the uio struct is unmodified. Any 215 * error will terminate the process as this is only a best attempt to get 216 * the pages resident. 217 */ 218 int 219 zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio) 220 { 221 if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) { 222 /* There's never a need to fault in kernel pages */ 223 return (0); 224 #if defined(HAVE_VFS_IOV_ITER) 225 } else if (uio->uio_segflg == UIO_ITER) { 226 /* 227 * At least a Linux 4.9 kernel, iov_iter_fault_in_readable() 228 * can be relied on to fault in user pages when referenced. 229 */ 230 if (iov_iter_fault_in_readable(uio->uio_iter, n)) 231 return (EFAULT); 232 #endif 233 } else { 234 /* Fault in all user pages */ 235 ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE); 236 const struct iovec *iov = uio->uio_iov; 237 int iovcnt = uio->uio_iovcnt; 238 size_t skip = uio->uio_skip; 239 uint8_t tmp; 240 caddr_t p; 241 242 for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) { 243 ulong_t cnt = MIN(iov->iov_len - skip, n); 244 /* empty iov */ 245 if (cnt == 0) 246 continue; 247 n -= cnt; 248 /* touch each page in this segment. */ 249 p = iov->iov_base + skip; 250 while (cnt) { 251 if (get_user(tmp, (uint8_t *)p)) 252 return (EFAULT); 253 ulong_t incr = MIN(cnt, PAGESIZE); 254 p += incr; 255 cnt -= incr; 256 } 257 /* touch the last byte in case it straddles a page. */ 258 p--; 259 if (get_user(tmp, (uint8_t *)p)) 260 return (EFAULT); 261 } 262 } 263 264 return (0); 265 } 266 EXPORT_SYMBOL(zfs_uio_prefaultpages); 267 268 /* 269 * The same as zfs_uiomove() but doesn't modify uio structure. 270 * return in cbytes how many bytes were copied. 271 */ 272 int 273 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes) 274 { 275 zfs_uio_t uio_copy; 276 int ret; 277 278 memcpy(&uio_copy, uio, sizeof (zfs_uio_t)); 279 280 if (uio->uio_segflg == UIO_BVEC) 281 ret = zfs_uiomove_bvec(p, n, rw, &uio_copy); 282 #if defined(HAVE_VFS_IOV_ITER) 283 else if (uio->uio_segflg == UIO_ITER) 284 ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE); 285 #endif 286 else 287 ret = zfs_uiomove_iov(p, n, rw, &uio_copy); 288 289 *cbytes = uio->uio_resid - uio_copy.uio_resid; 290 291 return (ret); 292 } 293 EXPORT_SYMBOL(zfs_uiocopy); 294 295 /* 296 * Drop the next n chars out of *uio. 297 */ 298 void 299 zfs_uioskip(zfs_uio_t *uio, size_t n) 300 { 301 if (n > uio->uio_resid) 302 return; 303 304 if (uio->uio_segflg == UIO_BVEC) { 305 uio->uio_skip += n; 306 while (uio->uio_iovcnt && 307 uio->uio_skip >= uio->uio_bvec->bv_len) { 308 uio->uio_skip -= uio->uio_bvec->bv_len; 309 uio->uio_bvec++; 310 uio->uio_iovcnt--; 311 } 312 #if defined(HAVE_VFS_IOV_ITER) 313 } else if (uio->uio_segflg == UIO_ITER) { 314 iov_iter_advance(uio->uio_iter, n); 315 #endif 316 } else { 317 uio->uio_skip += n; 318 while (uio->uio_iovcnt && 319 uio->uio_skip >= uio->uio_iov->iov_len) { 320 uio->uio_skip -= uio->uio_iov->iov_len; 321 uio->uio_iov++; 322 uio->uio_iovcnt--; 323 } 324 } 325 uio->uio_loffset += n; 326 uio->uio_resid -= n; 327 } 328 EXPORT_SYMBOL(zfs_uioskip); 329 330 #endif /* _KERNEL */ 331