1 /* 2 * Copyright (c) 2003,2004 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Kernel XIO interface. An initialized XIO is basically a collection of 36 * appropriately held vm_page_t's. XIO buffers are vmspace agnostic and 37 * can represent userspace or kernelspace buffers, and can be passed to 38 * foreign threads outside of the originating vmspace. XIO buffers are 39 * not mapped into KVM and thus can be manipulated and passed around with 40 * very low overheads. 41 * 42 * The intent is for XIO to be used in the I/O path, VFS, CAPS, and other 43 * places that need to pass (possibly userspace) data between threads. 44 * 45 * TODO: check for busy page when modifying, check writeable. 46 */ 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/uio.h> 51 #include <sys/malloc.h> 52 #include <sys/proc.h> 53 #include <sys/vmmeter.h> 54 #include <sys/vnode.h> 55 #include <sys/xio.h> 56 57 #include <cpu/lwbuf.h> 58 59 #include <vm/vm.h> 60 #include <vm/vm_param.h> 61 #include <sys/lock.h> 62 #include <vm/vm_kern.h> 63 #include <vm/pmap.h> 64 #include <vm/vm_map.h> 65 #include <vm/vm_object.h> 66 #include <vm/vm_page.h> 67 #include <vm/vm_pageout.h> 68 #include <vm/vm_pager.h> 69 #include <vm/vm_extern.h> 70 #include <vm/vm_page2.h> 71 72 /* 73 * Just do basic initialization of an empty XIO 74 */ 75 void 76 xio_init(xio_t xio) 77 { 78 xio->xio_flags = 0; 79 xio->xio_bytes = 0; 80 xio->xio_error = 0; 81 xio->xio_offset = 0; 82 xio->xio_npages = 0; 83 xio->xio_pages = xio->xio_internal_pages; 84 } 85 86 /* 87 * Initialize an XIO given a kernelspace buffer. 0 is returned on success, 88 * an error code on failure. The actual number of bytes that could be 89 * accomodated in the XIO will be stored in xio_bytes and the page offset 90 * will be stored in xio_offset. 91 * 92 * WARNING! We cannot map user memory directly into an xio unless we also 93 * make the mapping use managed pages, otherwise modifications to 94 * the memory will race against pageouts and flushes. 95 */ 96 int 97 xio_init_kbuf(xio_t xio, void *kbase, size_t kbytes) 98 { 99 vm_offset_t addr; 100 vm_paddr_t paddr; 101 vm_page_t m; 102 int i; 103 int n; 104 105 addr = trunc_page((vm_offset_t)kbase); 106 xio->xio_flags = 0; 107 xio->xio_offset = (vm_offset_t)kbase & PAGE_MASK; 108 xio->xio_bytes = 0; 109 xio->xio_pages = xio->xio_internal_pages; 110 xio->xio_error = 0; 111 if ((n = PAGE_SIZE - xio->xio_offset) > kbytes) 112 n = kbytes; 113 for (i = 0; n && i < XIO_INTERNAL_PAGES; ++i) { 114 if ((paddr = pmap_kextract(addr)) == 0) 115 break; 116 m = PHYS_TO_VM_PAGE(paddr); 117 vm_page_hold(m); 118 xio->xio_pages[i] = m; 119 kbytes -= n; 120 xio->xio_bytes += n; 121 if ((n = kbytes) > PAGE_SIZE) 122 n = PAGE_SIZE; 123 addr += PAGE_SIZE; 124 } 125 xio->xio_npages = i; 126 127 /* 128 * If a failure occured clean out what we loaded and return EFAULT. 129 * Return 0 on success. 130 */ 131 if (i < XIO_INTERNAL_PAGES && n) { 132 xio_release(xio); 133 xio->xio_error = EFAULT; 134 } 135 return(xio->xio_error); 136 } 137 138 /* 139 * Initialize an XIO given an array of vm_page pointers. The caller is 140 * responsible for any modified state changes for the pages. 141 */ 142 int 143 xio_init_pages(xio_t xio, struct vm_page **mbase, int npages, int xflags) 144 { 145 int i; 146 147 KKASSERT(npages <= XIO_INTERNAL_PAGES); 148 149 xio->xio_flags = xflags; 150 xio->xio_offset = 0; 151 xio->xio_bytes = npages * PAGE_SIZE; 152 xio->xio_pages = xio->xio_internal_pages; 153 xio->xio_npages = npages; 154 xio->xio_error = 0; 155 for (i = 0; i < npages; ++i) { 156 vm_page_hold(mbase[i]); 157 xio->xio_pages[i] = mbase[i]; 158 } 159 return(0); 160 } 161 162 /* 163 * Cleanup an XIO so it can be destroyed. The pages associated with the 164 * XIO are released. 165 */ 166 void 167 xio_release(xio_t xio) 168 { 169 int i; 170 vm_page_t m; 171 172 for (i = 0; i < xio->xio_npages; ++i) { 173 m = xio->xio_pages[i]; 174 if (xio->xio_flags & XIOF_WRITE) 175 vm_page_dirty(m); 176 vm_page_unhold(m); 177 } 178 xio->xio_offset = 0; 179 xio->xio_npages = 0; 180 xio->xio_bytes = 0; 181 xio->xio_error = ENOBUFS; 182 } 183 184 /* 185 * Copy data between an XIO and a UIO. If the UIO represents userspace it 186 * must be relative to the current context. 187 * 188 * uoffset is the abstracted starting offset in the XIO, not the actual 189 * offset, and usually starts at 0. 190 * 191 * The XIO is not modified. The UIO is updated to reflect the copy. 192 * 193 * UIO_READ xio -> uio 194 * UIO_WRITE uio -> xio 195 */ 196 int 197 xio_uio_copy(xio_t xio, int uoffset, struct uio *uio, size_t *sizep) 198 { 199 size_t bytes; 200 int error; 201 202 bytes = xio->xio_bytes - uoffset; 203 if (bytes > uio->uio_resid) 204 bytes = uio->uio_resid; 205 KKASSERT(bytes >= 0); 206 error = uiomove_fromphys(xio->xio_pages, xio->xio_offset + uoffset, 207 bytes, uio); 208 if (error == 0) 209 *sizep = bytes; 210 else 211 *sizep = 0; 212 return(error); 213 } 214 215 /* 216 * Copy the specified number of bytes from the xio to a userland 217 * buffer. Return an error code or 0 on success. 218 * 219 * uoffset is the abstracted starting offset in the XIO, not the actual 220 * offset, and usually starts at 0. 221 * 222 * The XIO is not modified. 223 */ 224 int 225 xio_copy_xtou(xio_t xio, int uoffset, void *uptr, int bytes) 226 { 227 int i; 228 int n; 229 int error; 230 int offset; 231 vm_page_t m; 232 struct lwbuf *lwb; 233 struct lwbuf lwb_cache; 234 235 if (uoffset + bytes > xio->xio_bytes) 236 return(EFAULT); 237 238 offset = (xio->xio_offset + uoffset) & PAGE_MASK; 239 if ((n = PAGE_SIZE - offset) > bytes) 240 n = bytes; 241 242 error = 0; 243 for (i = (xio->xio_offset + uoffset) >> PAGE_SHIFT; 244 i < xio->xio_npages; 245 ++i 246 ) { 247 m = xio->xio_pages[i]; 248 lwb = lwbuf_alloc(m, &lwb_cache); 249 error = copyout((char *)lwbuf_kva(lwb) + offset, uptr, n); 250 lwbuf_free(lwb); 251 if (error) 252 break; 253 bytes -= n; 254 uptr = (char *)uptr + n; 255 if (bytes == 0) 256 break; 257 if ((n = bytes) > PAGE_SIZE) 258 n = PAGE_SIZE; 259 offset = 0; 260 } 261 return(error); 262 } 263 264 /* 265 * Copy the specified number of bytes from the xio to a kernel 266 * buffer. Return an error code or 0 on success. 267 * 268 * uoffset is the abstracted starting offset in the XIO, not the actual 269 * offset, and usually starts at 0. 270 * 271 * The XIO is not modified. 272 */ 273 int 274 xio_copy_xtok(xio_t xio, int uoffset, void *kptr, int bytes) 275 { 276 int i; 277 int n; 278 int error; 279 int offset; 280 vm_page_t m; 281 struct lwbuf *lwb; 282 struct lwbuf lwb_cache; 283 284 if (bytes + uoffset > xio->xio_bytes) 285 return(EFAULT); 286 287 offset = (xio->xio_offset + uoffset) & PAGE_MASK; 288 if ((n = PAGE_SIZE - offset) > bytes) 289 n = bytes; 290 291 error = 0; 292 for (i = (xio->xio_offset + uoffset) >> PAGE_SHIFT; 293 i < xio->xio_npages; 294 ++i 295 ) { 296 m = xio->xio_pages[i]; 297 lwb = lwbuf_alloc(m, &lwb_cache); 298 bcopy((char *)lwbuf_kva(lwb) + offset, kptr, n); 299 lwbuf_free(lwb); 300 bytes -= n; 301 kptr = (char *)kptr + n; 302 if (bytes == 0) 303 break; 304 if ((n = bytes) > PAGE_SIZE) 305 n = PAGE_SIZE; 306 offset = 0; 307 } 308 return(error); 309 } 310 311 /* 312 * Copy the specified number of bytes from userland to the xio. 313 * Return an error code or 0 on success. 314 * 315 * uoffset is the abstracted starting offset in the XIO, not the actual 316 * offset, and usually starts at 0. 317 * 318 * Data in pages backing the XIO will be modified. 319 */ 320 int 321 xio_copy_utox(xio_t xio, int uoffset, const void *uptr, int bytes) 322 { 323 int i; 324 int n; 325 int error; 326 int offset; 327 vm_page_t m; 328 struct lwbuf *lwb; 329 struct lwbuf lwb_cache; 330 331 if (uoffset + bytes > xio->xio_bytes) 332 return(EFAULT); 333 334 offset = (xio->xio_offset + uoffset) & PAGE_MASK; 335 if ((n = PAGE_SIZE - offset) > bytes) 336 n = bytes; 337 338 error = 0; 339 for (i = (xio->xio_offset + uoffset) >> PAGE_SHIFT; 340 i < xio->xio_npages; 341 ++i 342 ) { 343 m = xio->xio_pages[i]; 344 lwb = lwbuf_alloc(m, &lwb_cache); 345 error = copyin(uptr, (char *)lwbuf_kva(lwb) + offset, n); 346 lwbuf_free(lwb); 347 if (error) 348 break; 349 bytes -= n; 350 uptr = (const char *)uptr + n; 351 if (bytes == 0) 352 break; 353 if ((n = bytes) > PAGE_SIZE) 354 n = PAGE_SIZE; 355 offset = 0; 356 } 357 return(error); 358 } 359 360 /* 361 * Copy the specified number of bytes from the kernel to the xio. 362 * Return an error code or 0 on success. 363 * 364 * uoffset is the abstracted starting offset in the XIO, not the actual 365 * offset, and usually starts at 0. 366 * 367 * Data in pages backing the XIO will be modified. 368 */ 369 int 370 xio_copy_ktox(xio_t xio, int uoffset, const void *kptr, int bytes) 371 { 372 int i; 373 int n; 374 int error; 375 int offset; 376 vm_page_t m; 377 struct lwbuf *lwb; 378 struct lwbuf lwb_cache; 379 380 if (uoffset + bytes > xio->xio_bytes) 381 return(EFAULT); 382 383 offset = (xio->xio_offset + uoffset) & PAGE_MASK; 384 if ((n = PAGE_SIZE - offset) > bytes) 385 n = bytes; 386 387 error = 0; 388 for (i = (xio->xio_offset + uoffset) >> PAGE_SHIFT; 389 i < xio->xio_npages; 390 ++i 391 ) { 392 m = xio->xio_pages[i]; 393 lwb = lwbuf_alloc(m, &lwb_cache); 394 bcopy(kptr, (char *)lwbuf_kva(lwb) + offset, n); 395 lwbuf_free(lwb); 396 bytes -= n; 397 kptr = (const char *)kptr + n; 398 if (bytes == 0) 399 break; 400 if ((n = bytes) > PAGE_SIZE) 401 n = PAGE_SIZE; 402 offset = 0; 403 } 404 return(error); 405 } 406