1 /*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * $FreeBSD$ 10 * 11 */ 12 13 /*- 14 * The following functions are based in the vn(4) driver: mdstart_swap(), 15 * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(), 16 * and as such under the following copyright: 17 * 18 * Copyright (c) 1988 University of Utah. 19 * Copyright (c) 1990, 1993 20 * The Regents of the University of California. All rights reserved. 21 * Copyright (c) 2013 The FreeBSD Foundation 22 * All rights reserved. 23 * 24 * This code is derived from software contributed to Berkeley by 25 * the Systems Programming Group of the University of Utah Computer 26 * Science Department. 27 * 28 * Portions of this software were developed by Konstantin Belousov 29 * under sponsorship from the FreeBSD Foundation. 30 * 31 * Redistribution and use in source and binary forms, with or without 32 * modification, are permitted provided that the following conditions 33 * are met: 34 * 1. Redistributions of source code must retain the above copyright 35 * notice, this list of conditions and the following disclaimer. 36 * 2. Redistributions in binary form must reproduce the above copyright 37 * notice, this list of conditions and the following disclaimer in the 38 * documentation and/or other materials provided with the distribution. 39 * 3. Neither the name of the University nor the names of its contributors 40 * may be used to endorse or promote products derived from this software 41 * without specific prior written permission. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 53 * SUCH DAMAGE. 54 * 55 * from: Utah Hdr: vn.c 1.13 94/04/02 56 * 57 * from: @(#)vn.c 8.6 (Berkeley) 4/1/94 58 * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03 59 */ 60 61 #include "opt_rootdevname.h" 62 #include "opt_geom.h" 63 #include "opt_md.h" 64 65 #include <sys/param.h> 66 #include <sys/systm.h> 67 #include <sys/bio.h> 68 #include <sys/buf.h> 69 #include <sys/conf.h> 70 #include <sys/devicestat.h> 71 #include <sys/fcntl.h> 72 #include <sys/kernel.h> 73 #include <sys/kthread.h> 74 #include <sys/limits.h> 75 #include <sys/linker.h> 76 #include <sys/lock.h> 77 #include <sys/malloc.h> 78 #include <sys/mdioctl.h> 79 #include <sys/mount.h> 80 #include <sys/mutex.h> 81 #include <sys/sx.h> 82 #include <sys/namei.h> 83 #include <sys/proc.h> 84 #include <sys/queue.h> 85 #include <sys/rwlock.h> 86 #include <sys/sbuf.h> 87 #include <sys/sched.h> 88 #include <sys/sf_buf.h> 89 #include <sys/sysctl.h> 90 #include <sys/vnode.h> 91 92 #include <geom/geom.h> 93 #include <geom/geom_int.h> 94 95 #include <vm/vm.h> 96 #include <vm/vm_param.h> 97 #include <vm/vm_object.h> 98 #include <vm/vm_page.h> 99 #include <vm/vm_pager.h> 100 #include <vm/swap_pager.h> 101 #include <vm/uma.h> 102 103 #include <machine/bus.h> 104 105 #define MD_MODVER 1 106 107 #define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */ 108 #define MD_EXITING 0x20000 /* Worker thread is exiting. */ 109 110 #ifndef MD_NSECT 111 #define MD_NSECT (10000 * 2) 112 #endif 113 114 static MALLOC_DEFINE(M_MD, "md_disk", "Memory Disk"); 115 static MALLOC_DEFINE(M_MDSECT, "md_sectors", "Memory Disk Sectors"); 116 117 static int md_debug; 118 SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0, 119 "Enable md(4) debug messages"); 120 static int md_malloc_wait; 121 SYSCTL_INT(_vm, OID_AUTO, md_malloc_wait, CTLFLAG_RW, &md_malloc_wait, 0, 122 "Allow malloc to wait for memory allocations"); 123 124 #if defined(MD_ROOT) && !defined(MD_ROOT_FSTYPE) 125 #define MD_ROOT_FSTYPE "ufs" 126 #endif 127 128 #if defined(MD_ROOT) 129 /* 130 * Preloaded image gets put here. 131 */ 132 #if defined(MD_ROOT_SIZE) 133 /* 134 * We put the mfs_root symbol into the oldmfs section of the kernel object file. 135 * Applications that patch the object with the image can determine 136 * the size looking at the oldmfs section size within the kernel. 137 */ 138 u_char mfs_root[MD_ROOT_SIZE*1024] __attribute__ ((section ("oldmfs"))); 139 const int mfs_root_size = sizeof(mfs_root); 140 #else 141 extern volatile u_char __weak_symbol mfs_root; 142 extern volatile u_char __weak_symbol mfs_root_end; 143 __GLOBL(mfs_root); 144 __GLOBL(mfs_root_end); 145 #define mfs_root_size ((uintptr_t)(&mfs_root_end - &mfs_root)) 146 #endif 147 #endif 148 149 static g_init_t g_md_init; 150 static g_fini_t g_md_fini; 151 static g_start_t g_md_start; 152 static g_access_t g_md_access; 153 static void g_md_dumpconf(struct sbuf *sb, const char *indent, 154 struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp); 155 156 static struct cdev *status_dev = NULL; 157 static struct sx md_sx; 158 static struct unrhdr *md_uh; 159 160 static d_ioctl_t mdctlioctl; 161 162 static struct cdevsw mdctl_cdevsw = { 163 .d_version = D_VERSION, 164 .d_ioctl = mdctlioctl, 165 .d_name = MD_NAME, 166 }; 167 168 struct g_class g_md_class = { 169 .name = "MD", 170 .version = G_VERSION, 171 .init = g_md_init, 172 .fini = g_md_fini, 173 .start = g_md_start, 174 .access = g_md_access, 175 .dumpconf = g_md_dumpconf, 176 }; 177 178 DECLARE_GEOM_CLASS(g_md_class, g_md); 179 180 181 static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(md_softc_list); 182 183 #define NINDIR (PAGE_SIZE / sizeof(uintptr_t)) 184 #define NMASK (NINDIR-1) 185 static int nshift; 186 187 static int md_vnode_pbuf_freecnt; 188 189 struct indir { 190 uintptr_t *array; 191 u_int total; 192 u_int used; 193 u_int shift; 194 }; 195 196 struct md_s { 197 int unit; 198 LIST_ENTRY(md_s) list; 199 struct bio_queue_head bio_queue; 200 struct mtx queue_mtx; 201 struct mtx stat_mtx; 202 struct cdev *dev; 203 enum md_types type; 204 off_t mediasize; 205 unsigned sectorsize; 206 unsigned opencount; 207 unsigned fwheads; 208 unsigned fwsectors; 209 unsigned flags; 210 char name[20]; 211 struct proc *procp; 212 struct g_geom *gp; 213 struct g_provider *pp; 214 int (*start)(struct md_s *sc, struct bio *bp); 215 struct devstat *devstat; 216 217 /* MD_MALLOC related fields */ 218 struct indir *indir; 219 uma_zone_t uma; 220 221 /* MD_PRELOAD related fields */ 222 u_char *pl_ptr; 223 size_t pl_len; 224 225 /* MD_VNODE related fields */ 226 struct vnode *vnode; 227 char file[PATH_MAX]; 228 struct ucred *cred; 229 230 /* MD_SWAP related fields */ 231 vm_object_t object; 232 }; 233 234 static struct indir * 235 new_indir(u_int shift) 236 { 237 struct indir *ip; 238 239 ip = malloc(sizeof *ip, M_MD, (md_malloc_wait ? M_WAITOK : M_NOWAIT) 240 | M_ZERO); 241 if (ip == NULL) 242 return (NULL); 243 ip->array = malloc(sizeof(uintptr_t) * NINDIR, 244 M_MDSECT, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); 245 if (ip->array == NULL) { 246 free(ip, M_MD); 247 return (NULL); 248 } 249 ip->total = NINDIR; 250 ip->shift = shift; 251 return (ip); 252 } 253 254 static void 255 del_indir(struct indir *ip) 256 { 257 258 free(ip->array, M_MDSECT); 259 free(ip, M_MD); 260 } 261 262 static void 263 destroy_indir(struct md_s *sc, struct indir *ip) 264 { 265 int i; 266 267 for (i = 0; i < NINDIR; i++) { 268 if (!ip->array[i]) 269 continue; 270 if (ip->shift) 271 destroy_indir(sc, (struct indir*)(ip->array[i])); 272 else if (ip->array[i] > 255) 273 uma_zfree(sc->uma, (void *)(ip->array[i])); 274 } 275 del_indir(ip); 276 } 277 278 /* 279 * This function does the math and allocates the top level "indir" structure 280 * for a device of "size" sectors. 281 */ 282 283 static struct indir * 284 dimension(off_t size) 285 { 286 off_t rcnt; 287 struct indir *ip; 288 int layer; 289 290 rcnt = size; 291 layer = 0; 292 while (rcnt > NINDIR) { 293 rcnt /= NINDIR; 294 layer++; 295 } 296 297 /* 298 * XXX: the top layer is probably not fully populated, so we allocate 299 * too much space for ip->array in here. 300 */ 301 ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO); 302 ip->array = malloc(sizeof(uintptr_t) * NINDIR, 303 M_MDSECT, M_WAITOK | M_ZERO); 304 ip->total = NINDIR; 305 ip->shift = layer * nshift; 306 return (ip); 307 } 308 309 /* 310 * Read a given sector 311 */ 312 313 static uintptr_t 314 s_read(struct indir *ip, off_t offset) 315 { 316 struct indir *cip; 317 int idx; 318 uintptr_t up; 319 320 if (md_debug > 1) 321 printf("s_read(%jd)\n", (intmax_t)offset); 322 up = 0; 323 for (cip = ip; cip != NULL;) { 324 if (cip->shift) { 325 idx = (offset >> cip->shift) & NMASK; 326 up = cip->array[idx]; 327 cip = (struct indir *)up; 328 continue; 329 } 330 idx = offset & NMASK; 331 return (cip->array[idx]); 332 } 333 return (0); 334 } 335 336 /* 337 * Write a given sector, prune the tree if the value is 0 338 */ 339 340 static int 341 s_write(struct indir *ip, off_t offset, uintptr_t ptr) 342 { 343 struct indir *cip, *lip[10]; 344 int idx, li; 345 uintptr_t up; 346 347 if (md_debug > 1) 348 printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr); 349 up = 0; 350 li = 0; 351 cip = ip; 352 for (;;) { 353 lip[li++] = cip; 354 if (cip->shift) { 355 idx = (offset >> cip->shift) & NMASK; 356 up = cip->array[idx]; 357 if (up != 0) { 358 cip = (struct indir *)up; 359 continue; 360 } 361 /* Allocate branch */ 362 cip->array[idx] = 363 (uintptr_t)new_indir(cip->shift - nshift); 364 if (cip->array[idx] == 0) 365 return (ENOSPC); 366 cip->used++; 367 up = cip->array[idx]; 368 cip = (struct indir *)up; 369 continue; 370 } 371 /* leafnode */ 372 idx = offset & NMASK; 373 up = cip->array[idx]; 374 if (up != 0) 375 cip->used--; 376 cip->array[idx] = ptr; 377 if (ptr != 0) 378 cip->used++; 379 break; 380 } 381 if (cip->used != 0 || li == 1) 382 return (0); 383 li--; 384 while (cip->used == 0 && cip != ip) { 385 li--; 386 idx = (offset >> lip[li]->shift) & NMASK; 387 up = lip[li]->array[idx]; 388 KASSERT(up == (uintptr_t)cip, ("md screwed up")); 389 del_indir(cip); 390 lip[li]->array[idx] = 0; 391 lip[li]->used--; 392 cip = lip[li]; 393 } 394 return (0); 395 } 396 397 398 static int 399 g_md_access(struct g_provider *pp, int r, int w, int e) 400 { 401 struct md_s *sc; 402 403 sc = pp->geom->softc; 404 if (sc == NULL) { 405 if (r <= 0 && w <= 0 && e <= 0) 406 return (0); 407 return (ENXIO); 408 } 409 r += pp->acr; 410 w += pp->acw; 411 e += pp->ace; 412 if ((sc->flags & MD_READONLY) != 0 && w > 0) 413 return (EROFS); 414 if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) { 415 sc->opencount = 1; 416 } else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) { 417 sc->opencount = 0; 418 } 419 return (0); 420 } 421 422 static void 423 g_md_start(struct bio *bp) 424 { 425 struct md_s *sc; 426 427 sc = bp->bio_to->geom->softc; 428 if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE)) { 429 mtx_lock(&sc->stat_mtx); 430 devstat_start_transaction_bio(sc->devstat, bp); 431 mtx_unlock(&sc->stat_mtx); 432 } 433 mtx_lock(&sc->queue_mtx); 434 bioq_disksort(&sc->bio_queue, bp); 435 mtx_unlock(&sc->queue_mtx); 436 wakeup(sc); 437 } 438 439 #define MD_MALLOC_MOVE_ZERO 1 440 #define MD_MALLOC_MOVE_FILL 2 441 #define MD_MALLOC_MOVE_READ 3 442 #define MD_MALLOC_MOVE_WRITE 4 443 #define MD_MALLOC_MOVE_CMP 5 444 445 static int 446 md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize, 447 void *ptr, u_char fill, int op) 448 { 449 struct sf_buf *sf; 450 vm_page_t m, *mp1; 451 char *p, first; 452 off_t *uc; 453 unsigned n; 454 int error, i, ma_offs1, sz, first_read; 455 456 m = NULL; 457 error = 0; 458 sf = NULL; 459 /* if (op == MD_MALLOC_MOVE_CMP) { gcc */ 460 first = 0; 461 first_read = 0; 462 uc = ptr; 463 mp1 = *mp; 464 ma_offs1 = *ma_offs; 465 /* } */ 466 sched_pin(); 467 for (n = sectorsize; n != 0; n -= sz) { 468 sz = imin(PAGE_SIZE - *ma_offs, n); 469 if (m != **mp) { 470 if (sf != NULL) 471 sf_buf_free(sf); 472 m = **mp; 473 sf = sf_buf_alloc(m, SFB_CPUPRIVATE | 474 (md_malloc_wait ? 0 : SFB_NOWAIT)); 475 if (sf == NULL) { 476 error = ENOMEM; 477 break; 478 } 479 } 480 p = (char *)sf_buf_kva(sf) + *ma_offs; 481 switch (op) { 482 case MD_MALLOC_MOVE_ZERO: 483 bzero(p, sz); 484 break; 485 case MD_MALLOC_MOVE_FILL: 486 memset(p, fill, sz); 487 break; 488 case MD_MALLOC_MOVE_READ: 489 bcopy(ptr, p, sz); 490 cpu_flush_dcache(p, sz); 491 break; 492 case MD_MALLOC_MOVE_WRITE: 493 bcopy(p, ptr, sz); 494 break; 495 case MD_MALLOC_MOVE_CMP: 496 for (i = 0; i < sz; i++, p++) { 497 if (!first_read) { 498 *uc = (u_char)*p; 499 first = *p; 500 first_read = 1; 501 } else if (*p != first) { 502 error = EDOOFUS; 503 break; 504 } 505 } 506 break; 507 default: 508 KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op)); 509 break; 510 } 511 if (error != 0) 512 break; 513 *ma_offs += sz; 514 *ma_offs %= PAGE_SIZE; 515 if (*ma_offs == 0) 516 (*mp)++; 517 ptr = (char *)ptr + sz; 518 } 519 520 if (sf != NULL) 521 sf_buf_free(sf); 522 sched_unpin(); 523 if (op == MD_MALLOC_MOVE_CMP && error != 0) { 524 *mp = mp1; 525 *ma_offs = ma_offs1; 526 } 527 return (error); 528 } 529 530 static int 531 md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs, 532 unsigned len, void *ptr, u_char fill, int op) 533 { 534 bus_dma_segment_t *vlist; 535 uint8_t *p, *end, first; 536 off_t *uc; 537 int ma_offs, seg_len; 538 539 vlist = *pvlist; 540 ma_offs = *pma_offs; 541 uc = ptr; 542 543 for (; len != 0; len -= seg_len) { 544 seg_len = imin(vlist->ds_len - ma_offs, len); 545 p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs; 546 switch (op) { 547 case MD_MALLOC_MOVE_ZERO: 548 bzero(p, seg_len); 549 break; 550 case MD_MALLOC_MOVE_FILL: 551 memset(p, fill, seg_len); 552 break; 553 case MD_MALLOC_MOVE_READ: 554 bcopy(ptr, p, seg_len); 555 cpu_flush_dcache(p, seg_len); 556 break; 557 case MD_MALLOC_MOVE_WRITE: 558 bcopy(p, ptr, seg_len); 559 break; 560 case MD_MALLOC_MOVE_CMP: 561 end = p + seg_len; 562 first = *uc = *p; 563 /* Confirm all following bytes match the first */ 564 while (++p < end) { 565 if (*p != first) 566 return (EDOOFUS); 567 } 568 break; 569 default: 570 KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op)); 571 break; 572 } 573 574 ma_offs += seg_len; 575 if (ma_offs == vlist->ds_len) { 576 ma_offs = 0; 577 vlist++; 578 } 579 ptr = (uint8_t *)ptr + seg_len; 580 } 581 *pvlist = vlist; 582 *pma_offs = ma_offs; 583 584 return (0); 585 } 586 587 static int 588 mdstart_malloc(struct md_s *sc, struct bio *bp) 589 { 590 u_char *dst; 591 vm_page_t *m; 592 bus_dma_segment_t *vlist; 593 int i, error, error1, ma_offs, notmapped; 594 off_t secno, nsec, uc; 595 uintptr_t sp, osp; 596 597 switch (bp->bio_cmd) { 598 case BIO_READ: 599 case BIO_WRITE: 600 case BIO_DELETE: 601 break; 602 default: 603 return (EOPNOTSUPP); 604 } 605 606 notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0; 607 vlist = (bp->bio_flags & BIO_VLIST) != 0 ? 608 (bus_dma_segment_t *)bp->bio_data : NULL; 609 if (notmapped) { 610 m = bp->bio_ma; 611 ma_offs = bp->bio_ma_offset; 612 dst = NULL; 613 KASSERT(vlist == NULL, ("vlists cannot be unmapped")); 614 } else if (vlist != NULL) { 615 ma_offs = bp->bio_ma_offset; 616 dst = NULL; 617 } else { 618 dst = bp->bio_data; 619 } 620 621 nsec = bp->bio_length / sc->sectorsize; 622 secno = bp->bio_offset / sc->sectorsize; 623 error = 0; 624 while (nsec--) { 625 osp = s_read(sc->indir, secno); 626 if (bp->bio_cmd == BIO_DELETE) { 627 if (osp != 0) 628 error = s_write(sc->indir, secno, 0); 629 } else if (bp->bio_cmd == BIO_READ) { 630 if (osp == 0) { 631 if (notmapped) { 632 error = md_malloc_move_ma(&m, &ma_offs, 633 sc->sectorsize, NULL, 0, 634 MD_MALLOC_MOVE_ZERO); 635 } else if (vlist != NULL) { 636 error = md_malloc_move_vlist(&vlist, 637 &ma_offs, sc->sectorsize, NULL, 0, 638 MD_MALLOC_MOVE_ZERO); 639 } else 640 bzero(dst, sc->sectorsize); 641 } else if (osp <= 255) { 642 if (notmapped) { 643 error = md_malloc_move_ma(&m, &ma_offs, 644 sc->sectorsize, NULL, osp, 645 MD_MALLOC_MOVE_FILL); 646 } else if (vlist != NULL) { 647 error = md_malloc_move_vlist(&vlist, 648 &ma_offs, sc->sectorsize, NULL, osp, 649 MD_MALLOC_MOVE_FILL); 650 } else 651 memset(dst, osp, sc->sectorsize); 652 } else { 653 if (notmapped) { 654 error = md_malloc_move_ma(&m, &ma_offs, 655 sc->sectorsize, (void *)osp, 0, 656 MD_MALLOC_MOVE_READ); 657 } else if (vlist != NULL) { 658 error = md_malloc_move_vlist(&vlist, 659 &ma_offs, sc->sectorsize, 660 (void *)osp, 0, 661 MD_MALLOC_MOVE_READ); 662 } else { 663 bcopy((void *)osp, dst, sc->sectorsize); 664 cpu_flush_dcache(dst, sc->sectorsize); 665 } 666 } 667 osp = 0; 668 } else if (bp->bio_cmd == BIO_WRITE) { 669 if (sc->flags & MD_COMPRESS) { 670 if (notmapped) { 671 error1 = md_malloc_move_ma(&m, &ma_offs, 672 sc->sectorsize, &uc, 0, 673 MD_MALLOC_MOVE_CMP); 674 i = error1 == 0 ? sc->sectorsize : 0; 675 } else if (vlist != NULL) { 676 error1 = md_malloc_move_vlist(&vlist, 677 &ma_offs, sc->sectorsize, &uc, 0, 678 MD_MALLOC_MOVE_CMP); 679 i = error1 == 0 ? sc->sectorsize : 0; 680 } else { 681 uc = dst[0]; 682 for (i = 1; i < sc->sectorsize; i++) { 683 if (dst[i] != uc) 684 break; 685 } 686 } 687 } else { 688 i = 0; 689 uc = 0; 690 } 691 if (i == sc->sectorsize) { 692 if (osp != uc) 693 error = s_write(sc->indir, secno, uc); 694 } else { 695 if (osp <= 255) { 696 sp = (uintptr_t)uma_zalloc(sc->uma, 697 md_malloc_wait ? M_WAITOK : 698 M_NOWAIT); 699 if (sp == 0) { 700 error = ENOSPC; 701 break; 702 } 703 if (notmapped) { 704 error = md_malloc_move_ma(&m, 705 &ma_offs, sc->sectorsize, 706 (void *)sp, 0, 707 MD_MALLOC_MOVE_WRITE); 708 } else if (vlist != NULL) { 709 error = md_malloc_move_vlist( 710 &vlist, &ma_offs, 711 sc->sectorsize, (void *)sp, 712 0, MD_MALLOC_MOVE_WRITE); 713 } else { 714 bcopy(dst, (void *)sp, 715 sc->sectorsize); 716 } 717 error = s_write(sc->indir, secno, sp); 718 } else { 719 if (notmapped) { 720 error = md_malloc_move_ma(&m, 721 &ma_offs, sc->sectorsize, 722 (void *)osp, 0, 723 MD_MALLOC_MOVE_WRITE); 724 } else if (vlist != NULL) { 725 error = md_malloc_move_vlist( 726 &vlist, &ma_offs, 727 sc->sectorsize, (void *)osp, 728 0, MD_MALLOC_MOVE_WRITE); 729 } else { 730 bcopy(dst, (void *)osp, 731 sc->sectorsize); 732 } 733 osp = 0; 734 } 735 } 736 } else { 737 error = EOPNOTSUPP; 738 } 739 if (osp > 255) 740 uma_zfree(sc->uma, (void*)osp); 741 if (error != 0) 742 break; 743 secno++; 744 if (!notmapped && vlist == NULL) 745 dst += sc->sectorsize; 746 } 747 bp->bio_resid = 0; 748 return (error); 749 } 750 751 static void 752 mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len) 753 { 754 off_t seg_len; 755 756 while (offset >= vlist->ds_len) { 757 offset -= vlist->ds_len; 758 vlist++; 759 } 760 761 while (len != 0) { 762 seg_len = omin(len, vlist->ds_len - offset); 763 bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset), 764 seg_len); 765 offset = 0; 766 src = (uint8_t *)src + seg_len; 767 len -= seg_len; 768 vlist++; 769 } 770 } 771 772 static void 773 mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len) 774 { 775 off_t seg_len; 776 777 while (offset >= vlist->ds_len) { 778 offset -= vlist->ds_len; 779 vlist++; 780 } 781 782 while (len != 0) { 783 seg_len = omin(len, vlist->ds_len - offset); 784 bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst, 785 seg_len); 786 offset = 0; 787 dst = (uint8_t *)dst + seg_len; 788 len -= seg_len; 789 vlist++; 790 } 791 } 792 793 static int 794 mdstart_preload(struct md_s *sc, struct bio *bp) 795 { 796 uint8_t *p; 797 798 p = sc->pl_ptr + bp->bio_offset; 799 switch (bp->bio_cmd) { 800 case BIO_READ: 801 if ((bp->bio_flags & BIO_VLIST) != 0) { 802 mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data, 803 bp->bio_ma_offset, bp->bio_length); 804 } else { 805 bcopy(p, bp->bio_data, bp->bio_length); 806 } 807 cpu_flush_dcache(bp->bio_data, bp->bio_length); 808 break; 809 case BIO_WRITE: 810 if ((bp->bio_flags & BIO_VLIST) != 0) { 811 mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data, 812 bp->bio_ma_offset, p, bp->bio_length); 813 } else { 814 bcopy(bp->bio_data, p, bp->bio_length); 815 } 816 break; 817 } 818 bp->bio_resid = 0; 819 return (0); 820 } 821 822 static int 823 mdstart_vnode(struct md_s *sc, struct bio *bp) 824 { 825 int error; 826 struct uio auio; 827 struct iovec aiov; 828 struct iovec *piov; 829 struct mount *mp; 830 struct vnode *vp; 831 struct buf *pb; 832 bus_dma_segment_t *vlist; 833 struct thread *td; 834 off_t iolen, len, zerosize; 835 int ma_offs, npages; 836 837 switch (bp->bio_cmd) { 838 case BIO_READ: 839 auio.uio_rw = UIO_READ; 840 break; 841 case BIO_WRITE: 842 case BIO_DELETE: 843 auio.uio_rw = UIO_WRITE; 844 break; 845 case BIO_FLUSH: 846 break; 847 default: 848 return (EOPNOTSUPP); 849 } 850 851 td = curthread; 852 vp = sc->vnode; 853 pb = NULL; 854 piov = NULL; 855 ma_offs = bp->bio_ma_offset; 856 len = bp->bio_length; 857 858 /* 859 * VNODE I/O 860 * 861 * If an error occurs, we set BIO_ERROR but we do not set 862 * B_INVAL because (for a write anyway), the buffer is 863 * still valid. 864 */ 865 866 if (bp->bio_cmd == BIO_FLUSH) { 867 (void) vn_start_write(vp, &mp, V_WAIT); 868 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 869 error = VOP_FSYNC(vp, MNT_WAIT, td); 870 VOP_UNLOCK(vp, 0); 871 vn_finished_write(mp); 872 return (error); 873 } 874 875 auio.uio_offset = (vm_ooffset_t)bp->bio_offset; 876 auio.uio_resid = bp->bio_length; 877 auio.uio_segflg = UIO_SYSSPACE; 878 auio.uio_td = td; 879 880 if (bp->bio_cmd == BIO_DELETE) { 881 /* 882 * Emulate BIO_DELETE by writing zeros. 883 */ 884 zerosize = ZERO_REGION_SIZE - 885 (ZERO_REGION_SIZE % sc->sectorsize); 886 auio.uio_iovcnt = howmany(bp->bio_length, zerosize); 887 piov = malloc(sizeof(*piov) * auio.uio_iovcnt, M_MD, M_WAITOK); 888 auio.uio_iov = piov; 889 while (len > 0) { 890 piov->iov_base = __DECONST(void *, zero_region); 891 piov->iov_len = len; 892 if (len > zerosize) 893 piov->iov_len = zerosize; 894 len -= piov->iov_len; 895 piov++; 896 } 897 piov = auio.uio_iov; 898 } else if ((bp->bio_flags & BIO_VLIST) != 0) { 899 piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK); 900 auio.uio_iov = piov; 901 vlist = (bus_dma_segment_t *)bp->bio_data; 902 while (len > 0) { 903 piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr + 904 ma_offs); 905 piov->iov_len = vlist->ds_len - ma_offs; 906 if (piov->iov_len > len) 907 piov->iov_len = len; 908 len -= piov->iov_len; 909 ma_offs = 0; 910 vlist++; 911 piov++; 912 } 913 auio.uio_iovcnt = piov - auio.uio_iov; 914 piov = auio.uio_iov; 915 } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 916 pb = getpbuf(&md_vnode_pbuf_freecnt); 917 bp->bio_resid = len; 918 unmapped_step: 919 npages = atop(min(MAXPHYS, round_page(len + (ma_offs & 920 PAGE_MASK)))); 921 iolen = min(ptoa(npages) - (ma_offs & PAGE_MASK), len); 922 KASSERT(iolen > 0, ("zero iolen")); 923 pmap_qenter((vm_offset_t)pb->b_data, 924 &bp->bio_ma[atop(ma_offs)], npages); 925 aiov.iov_base = (void *)((vm_offset_t)pb->b_data + 926 (ma_offs & PAGE_MASK)); 927 aiov.iov_len = iolen; 928 auio.uio_iov = &aiov; 929 auio.uio_iovcnt = 1; 930 auio.uio_resid = iolen; 931 } else { 932 aiov.iov_base = bp->bio_data; 933 aiov.iov_len = bp->bio_length; 934 auio.uio_iov = &aiov; 935 auio.uio_iovcnt = 1; 936 } 937 /* 938 * When reading set IO_DIRECT to try to avoid double-caching 939 * the data. When writing IO_DIRECT is not optimal. 940 */ 941 if (auio.uio_rw == UIO_READ) { 942 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 943 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 944 VOP_UNLOCK(vp, 0); 945 } else { 946 (void) vn_start_write(vp, &mp, V_WAIT); 947 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 948 error = VOP_WRITE(vp, &auio, sc->flags & MD_ASYNC ? 0 : IO_SYNC, 949 sc->cred); 950 VOP_UNLOCK(vp, 0); 951 vn_finished_write(mp); 952 } 953 954 if (pb != NULL) { 955 pmap_qremove((vm_offset_t)pb->b_data, npages); 956 if (error == 0) { 957 len -= iolen; 958 bp->bio_resid -= iolen; 959 ma_offs += iolen; 960 if (len > 0) 961 goto unmapped_step; 962 } 963 relpbuf(pb, &md_vnode_pbuf_freecnt); 964 } 965 966 free(piov, M_MD); 967 if (pb == NULL) 968 bp->bio_resid = auio.uio_resid; 969 return (error); 970 } 971 972 static int 973 mdstart_swap(struct md_s *sc, struct bio *bp) 974 { 975 vm_page_t m; 976 u_char *p; 977 vm_pindex_t i, lastp; 978 bus_dma_segment_t *vlist; 979 int rv, ma_offs, offs, len, lastend; 980 981 switch (bp->bio_cmd) { 982 case BIO_READ: 983 case BIO_WRITE: 984 case BIO_DELETE: 985 break; 986 default: 987 return (EOPNOTSUPP); 988 } 989 990 p = bp->bio_data; 991 ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ? 992 bp->bio_ma_offset : 0; 993 vlist = (bp->bio_flags & BIO_VLIST) != 0 ? 994 (bus_dma_segment_t *)bp->bio_data : NULL; 995 996 /* 997 * offs is the offset at which to start operating on the 998 * next (ie, first) page. lastp is the last page on 999 * which we're going to operate. lastend is the ending 1000 * position within that last page (ie, PAGE_SIZE if 1001 * we're operating on complete aligned pages). 1002 */ 1003 offs = bp->bio_offset % PAGE_SIZE; 1004 lastp = (bp->bio_offset + bp->bio_length - 1) / PAGE_SIZE; 1005 lastend = (bp->bio_offset + bp->bio_length - 1) % PAGE_SIZE + 1; 1006 1007 rv = VM_PAGER_OK; 1008 VM_OBJECT_WLOCK(sc->object); 1009 vm_object_pip_add(sc->object, 1); 1010 for (i = bp->bio_offset / PAGE_SIZE; i <= lastp; i++) { 1011 len = ((i == lastp) ? lastend : PAGE_SIZE) - offs; 1012 m = vm_page_grab(sc->object, i, VM_ALLOC_SYSTEM); 1013 if (bp->bio_cmd == BIO_READ) { 1014 if (m->valid == VM_PAGE_BITS_ALL) 1015 rv = VM_PAGER_OK; 1016 else 1017 rv = vm_pager_get_pages(sc->object, &m, 1, 1018 NULL, NULL); 1019 if (rv == VM_PAGER_ERROR) { 1020 vm_page_xunbusy(m); 1021 break; 1022 } else if (rv == VM_PAGER_FAIL) { 1023 /* 1024 * Pager does not have the page. Zero 1025 * the allocated page, and mark it as 1026 * valid. Do not set dirty, the page 1027 * can be recreated if thrown out. 1028 */ 1029 pmap_zero_page(m); 1030 m->valid = VM_PAGE_BITS_ALL; 1031 } 1032 if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 1033 pmap_copy_pages(&m, offs, bp->bio_ma, 1034 ma_offs, len); 1035 } else if ((bp->bio_flags & BIO_VLIST) != 0) { 1036 physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs, 1037 vlist, ma_offs, len); 1038 cpu_flush_dcache(p, len); 1039 } else { 1040 physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len); 1041 cpu_flush_dcache(p, len); 1042 } 1043 } else if (bp->bio_cmd == BIO_WRITE) { 1044 if (len != PAGE_SIZE && m->valid != VM_PAGE_BITS_ALL) 1045 rv = vm_pager_get_pages(sc->object, &m, 1, 1046 NULL, NULL); 1047 else 1048 rv = VM_PAGER_OK; 1049 if (rv == VM_PAGER_ERROR) { 1050 vm_page_xunbusy(m); 1051 break; 1052 } 1053 if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 1054 pmap_copy_pages(bp->bio_ma, ma_offs, &m, 1055 offs, len); 1056 } else if ((bp->bio_flags & BIO_VLIST) != 0) { 1057 physcopyin_vlist(vlist, ma_offs, 1058 VM_PAGE_TO_PHYS(m) + offs, len); 1059 } else { 1060 physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); 1061 } 1062 m->valid = VM_PAGE_BITS_ALL; 1063 } else if (bp->bio_cmd == BIO_DELETE) { 1064 if (len != PAGE_SIZE && m->valid != VM_PAGE_BITS_ALL) 1065 rv = vm_pager_get_pages(sc->object, &m, 1, 1066 NULL, NULL); 1067 else 1068 rv = VM_PAGER_OK; 1069 if (rv == VM_PAGER_ERROR) { 1070 vm_page_xunbusy(m); 1071 break; 1072 } 1073 if (len != PAGE_SIZE) { 1074 pmap_zero_page_area(m, offs, len); 1075 vm_page_clear_dirty(m, offs, len); 1076 m->valid = VM_PAGE_BITS_ALL; 1077 } else 1078 vm_pager_page_unswapped(m); 1079 } 1080 vm_page_xunbusy(m); 1081 vm_page_lock(m); 1082 if (bp->bio_cmd == BIO_DELETE && len == PAGE_SIZE) 1083 vm_page_free(m); 1084 else 1085 vm_page_activate(m); 1086 vm_page_unlock(m); 1087 if (bp->bio_cmd == BIO_WRITE) { 1088 vm_page_dirty(m); 1089 vm_pager_page_unswapped(m); 1090 } 1091 1092 /* Actions on further pages start at offset 0 */ 1093 p += PAGE_SIZE - offs; 1094 offs = 0; 1095 ma_offs += len; 1096 } 1097 vm_object_pip_wakeup(sc->object); 1098 VM_OBJECT_WUNLOCK(sc->object); 1099 return (rv != VM_PAGER_ERROR ? 0 : ENOSPC); 1100 } 1101 1102 static int 1103 mdstart_null(struct md_s *sc, struct bio *bp) 1104 { 1105 1106 switch (bp->bio_cmd) { 1107 case BIO_READ: 1108 bzero(bp->bio_data, bp->bio_length); 1109 cpu_flush_dcache(bp->bio_data, bp->bio_length); 1110 break; 1111 case BIO_WRITE: 1112 break; 1113 } 1114 bp->bio_resid = 0; 1115 return (0); 1116 } 1117 1118 static void 1119 md_kthread(void *arg) 1120 { 1121 struct md_s *sc; 1122 struct bio *bp; 1123 int error; 1124 1125 sc = arg; 1126 thread_lock(curthread); 1127 sched_prio(curthread, PRIBIO); 1128 thread_unlock(curthread); 1129 if (sc->type == MD_VNODE) 1130 curthread->td_pflags |= TDP_NORUNNINGBUF; 1131 1132 for (;;) { 1133 mtx_lock(&sc->queue_mtx); 1134 if (sc->flags & MD_SHUTDOWN) { 1135 sc->flags |= MD_EXITING; 1136 mtx_unlock(&sc->queue_mtx); 1137 kproc_exit(0); 1138 } 1139 bp = bioq_takefirst(&sc->bio_queue); 1140 if (!bp) { 1141 msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0); 1142 continue; 1143 } 1144 mtx_unlock(&sc->queue_mtx); 1145 if (bp->bio_cmd == BIO_GETATTR) { 1146 if ((sc->fwsectors && sc->fwheads && 1147 (g_handleattr_int(bp, "GEOM::fwsectors", 1148 sc->fwsectors) || 1149 g_handleattr_int(bp, "GEOM::fwheads", 1150 sc->fwheads))) || 1151 g_handleattr_int(bp, "GEOM::candelete", 1)) 1152 error = -1; 1153 else 1154 error = EOPNOTSUPP; 1155 } else { 1156 error = sc->start(sc, bp); 1157 } 1158 1159 if (error != -1) { 1160 bp->bio_completed = bp->bio_length; 1161 if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE)) 1162 devstat_end_transaction_bio(sc->devstat, bp); 1163 g_io_deliver(bp, error); 1164 } 1165 } 1166 } 1167 1168 static struct md_s * 1169 mdfind(int unit) 1170 { 1171 struct md_s *sc; 1172 1173 LIST_FOREACH(sc, &md_softc_list, list) { 1174 if (sc->unit == unit) 1175 break; 1176 } 1177 return (sc); 1178 } 1179 1180 static struct md_s * 1181 mdnew(int unit, int *errp, enum md_types type) 1182 { 1183 struct md_s *sc; 1184 int error; 1185 1186 *errp = 0; 1187 if (unit == -1) 1188 unit = alloc_unr(md_uh); 1189 else 1190 unit = alloc_unr_specific(md_uh, unit); 1191 1192 if (unit == -1) { 1193 *errp = EBUSY; 1194 return (NULL); 1195 } 1196 1197 sc = (struct md_s *)malloc(sizeof *sc, M_MD, M_WAITOK | M_ZERO); 1198 sc->type = type; 1199 bioq_init(&sc->bio_queue); 1200 mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF); 1201 mtx_init(&sc->stat_mtx, "md stat", NULL, MTX_DEF); 1202 sc->unit = unit; 1203 sprintf(sc->name, "md%d", unit); 1204 LIST_INSERT_HEAD(&md_softc_list, sc, list); 1205 error = kproc_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name); 1206 if (error == 0) 1207 return (sc); 1208 LIST_REMOVE(sc, list); 1209 mtx_destroy(&sc->stat_mtx); 1210 mtx_destroy(&sc->queue_mtx); 1211 free_unr(md_uh, sc->unit); 1212 free(sc, M_MD); 1213 *errp = error; 1214 return (NULL); 1215 } 1216 1217 static void 1218 mdinit(struct md_s *sc) 1219 { 1220 struct g_geom *gp; 1221 struct g_provider *pp; 1222 1223 g_topology_lock(); 1224 gp = g_new_geomf(&g_md_class, "md%d", sc->unit); 1225 gp->softc = sc; 1226 pp = g_new_providerf(gp, "md%d", sc->unit); 1227 pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; 1228 pp->mediasize = sc->mediasize; 1229 pp->sectorsize = sc->sectorsize; 1230 switch (sc->type) { 1231 case MD_MALLOC: 1232 case MD_VNODE: 1233 case MD_SWAP: 1234 pp->flags |= G_PF_ACCEPT_UNMAPPED; 1235 break; 1236 case MD_PRELOAD: 1237 case MD_NULL: 1238 break; 1239 } 1240 sc->gp = gp; 1241 sc->pp = pp; 1242 g_error_provider(pp, 0); 1243 g_topology_unlock(); 1244 sc->devstat = devstat_new_entry("md", sc->unit, sc->sectorsize, 1245 DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX); 1246 } 1247 1248 static int 1249 mdcreate_malloc(struct md_s *sc, struct md_ioctl *mdio) 1250 { 1251 uintptr_t sp; 1252 int error; 1253 off_t u; 1254 1255 error = 0; 1256 if (mdio->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE)) 1257 return (EINVAL); 1258 if (mdio->md_sectorsize != 0 && !powerof2(mdio->md_sectorsize)) 1259 return (EINVAL); 1260 /* Compression doesn't make sense if we have reserved space */ 1261 if (mdio->md_options & MD_RESERVE) 1262 mdio->md_options &= ~MD_COMPRESS; 1263 if (mdio->md_fwsectors != 0) 1264 sc->fwsectors = mdio->md_fwsectors; 1265 if (mdio->md_fwheads != 0) 1266 sc->fwheads = mdio->md_fwheads; 1267 sc->flags = mdio->md_options & (MD_COMPRESS | MD_FORCE); 1268 sc->indir = dimension(sc->mediasize / sc->sectorsize); 1269 sc->uma = uma_zcreate(sc->name, sc->sectorsize, NULL, NULL, NULL, NULL, 1270 0x1ff, 0); 1271 if (mdio->md_options & MD_RESERVE) { 1272 off_t nsectors; 1273 1274 nsectors = sc->mediasize / sc->sectorsize; 1275 for (u = 0; u < nsectors; u++) { 1276 sp = (uintptr_t)uma_zalloc(sc->uma, (md_malloc_wait ? 1277 M_WAITOK : M_NOWAIT) | M_ZERO); 1278 if (sp != 0) 1279 error = s_write(sc->indir, u, sp); 1280 else 1281 error = ENOMEM; 1282 if (error != 0) 1283 break; 1284 } 1285 } 1286 return (error); 1287 } 1288 1289 1290 static int 1291 mdsetcred(struct md_s *sc, struct ucred *cred) 1292 { 1293 char *tmpbuf; 1294 int error = 0; 1295 1296 /* 1297 * Set credits in our softc 1298 */ 1299 1300 if (sc->cred) 1301 crfree(sc->cred); 1302 sc->cred = crhold(cred); 1303 1304 /* 1305 * Horrible kludge to establish credentials for NFS XXX. 1306 */ 1307 1308 if (sc->vnode) { 1309 struct uio auio; 1310 struct iovec aiov; 1311 1312 tmpbuf = malloc(sc->sectorsize, M_TEMP, M_WAITOK); 1313 bzero(&auio, sizeof(auio)); 1314 1315 aiov.iov_base = tmpbuf; 1316 aiov.iov_len = sc->sectorsize; 1317 auio.uio_iov = &aiov; 1318 auio.uio_iovcnt = 1; 1319 auio.uio_offset = 0; 1320 auio.uio_rw = UIO_READ; 1321 auio.uio_segflg = UIO_SYSSPACE; 1322 auio.uio_resid = aiov.iov_len; 1323 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 1324 error = VOP_READ(sc->vnode, &auio, 0, sc->cred); 1325 VOP_UNLOCK(sc->vnode, 0); 1326 free(tmpbuf, M_TEMP); 1327 } 1328 return (error); 1329 } 1330 1331 static int 1332 mdcreate_vnode(struct md_s *sc, struct md_ioctl *mdio, struct thread *td) 1333 { 1334 struct vattr vattr; 1335 struct nameidata nd; 1336 char *fname; 1337 int error, flags; 1338 1339 /* 1340 * Kernel-originated requests must have the filename appended 1341 * to the mdio structure to protect against malicious software. 1342 */ 1343 fname = mdio->md_file; 1344 if ((void *)fname != (void *)(mdio + 1)) { 1345 error = copyinstr(fname, sc->file, sizeof(sc->file), NULL); 1346 if (error != 0) 1347 return (error); 1348 } else 1349 strlcpy(sc->file, fname, sizeof(sc->file)); 1350 1351 /* 1352 * If the user specified that this is a read only device, don't 1353 * set the FWRITE mask before trying to open the backing store. 1354 */ 1355 flags = FREAD | ((mdio->md_options & MD_READONLY) ? 0 : FWRITE); 1356 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->file, td); 1357 error = vn_open(&nd, &flags, 0, NULL); 1358 if (error != 0) 1359 return (error); 1360 NDFREE(&nd, NDF_ONLY_PNBUF); 1361 if (nd.ni_vp->v_type != VREG) { 1362 error = EINVAL; 1363 goto bad; 1364 } 1365 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 1366 if (error != 0) 1367 goto bad; 1368 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 1369 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 1370 if (nd.ni_vp->v_iflag & VI_DOOMED) { 1371 /* Forced unmount. */ 1372 error = EBADF; 1373 goto bad; 1374 } 1375 } 1376 nd.ni_vp->v_vflag |= VV_MD; 1377 VOP_UNLOCK(nd.ni_vp, 0); 1378 1379 if (mdio->md_fwsectors != 0) 1380 sc->fwsectors = mdio->md_fwsectors; 1381 if (mdio->md_fwheads != 0) 1382 sc->fwheads = mdio->md_fwheads; 1383 sc->flags = mdio->md_options & (MD_FORCE | MD_ASYNC); 1384 if (!(flags & FWRITE)) 1385 sc->flags |= MD_READONLY; 1386 sc->vnode = nd.ni_vp; 1387 1388 error = mdsetcred(sc, td->td_ucred); 1389 if (error != 0) { 1390 sc->vnode = NULL; 1391 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 1392 nd.ni_vp->v_vflag &= ~VV_MD; 1393 goto bad; 1394 } 1395 return (0); 1396 bad: 1397 VOP_UNLOCK(nd.ni_vp, 0); 1398 (void)vn_close(nd.ni_vp, flags, td->td_ucred, td); 1399 return (error); 1400 } 1401 1402 static int 1403 mddestroy(struct md_s *sc, struct thread *td) 1404 { 1405 1406 if (sc->gp) { 1407 sc->gp->softc = NULL; 1408 g_topology_lock(); 1409 g_wither_geom(sc->gp, ENXIO); 1410 g_topology_unlock(); 1411 sc->gp = NULL; 1412 sc->pp = NULL; 1413 } 1414 if (sc->devstat) { 1415 devstat_remove_entry(sc->devstat); 1416 sc->devstat = NULL; 1417 } 1418 mtx_lock(&sc->queue_mtx); 1419 sc->flags |= MD_SHUTDOWN; 1420 wakeup(sc); 1421 while (!(sc->flags & MD_EXITING)) 1422 msleep(sc->procp, &sc->queue_mtx, PRIBIO, "mddestroy", hz / 10); 1423 mtx_unlock(&sc->queue_mtx); 1424 mtx_destroy(&sc->stat_mtx); 1425 mtx_destroy(&sc->queue_mtx); 1426 if (sc->vnode != NULL) { 1427 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 1428 sc->vnode->v_vflag &= ~VV_MD; 1429 VOP_UNLOCK(sc->vnode, 0); 1430 (void)vn_close(sc->vnode, sc->flags & MD_READONLY ? 1431 FREAD : (FREAD|FWRITE), sc->cred, td); 1432 } 1433 if (sc->cred != NULL) 1434 crfree(sc->cred); 1435 if (sc->object != NULL) 1436 vm_object_deallocate(sc->object); 1437 if (sc->indir) 1438 destroy_indir(sc, sc->indir); 1439 if (sc->uma) 1440 uma_zdestroy(sc->uma); 1441 1442 LIST_REMOVE(sc, list); 1443 free_unr(md_uh, sc->unit); 1444 free(sc, M_MD); 1445 return (0); 1446 } 1447 1448 static int 1449 mdresize(struct md_s *sc, struct md_ioctl *mdio) 1450 { 1451 int error, res; 1452 vm_pindex_t oldpages, newpages; 1453 1454 switch (sc->type) { 1455 case MD_VNODE: 1456 case MD_NULL: 1457 break; 1458 case MD_SWAP: 1459 if (mdio->md_mediasize <= 0 || 1460 (mdio->md_mediasize % PAGE_SIZE) != 0) 1461 return (EDOM); 1462 oldpages = OFF_TO_IDX(round_page(sc->mediasize)); 1463 newpages = OFF_TO_IDX(round_page(mdio->md_mediasize)); 1464 if (newpages < oldpages) { 1465 VM_OBJECT_WLOCK(sc->object); 1466 vm_object_page_remove(sc->object, newpages, 0, 0); 1467 swap_pager_freespace(sc->object, newpages, 1468 oldpages - newpages); 1469 swap_release_by_cred(IDX_TO_OFF(oldpages - 1470 newpages), sc->cred); 1471 sc->object->charge = IDX_TO_OFF(newpages); 1472 sc->object->size = newpages; 1473 VM_OBJECT_WUNLOCK(sc->object); 1474 } else if (newpages > oldpages) { 1475 res = swap_reserve_by_cred(IDX_TO_OFF(newpages - 1476 oldpages), sc->cred); 1477 if (!res) 1478 return (ENOMEM); 1479 if ((mdio->md_options & MD_RESERVE) || 1480 (sc->flags & MD_RESERVE)) { 1481 error = swap_pager_reserve(sc->object, 1482 oldpages, newpages - oldpages); 1483 if (error < 0) { 1484 swap_release_by_cred( 1485 IDX_TO_OFF(newpages - oldpages), 1486 sc->cred); 1487 return (EDOM); 1488 } 1489 } 1490 VM_OBJECT_WLOCK(sc->object); 1491 sc->object->charge = IDX_TO_OFF(newpages); 1492 sc->object->size = newpages; 1493 VM_OBJECT_WUNLOCK(sc->object); 1494 } 1495 break; 1496 default: 1497 return (EOPNOTSUPP); 1498 } 1499 1500 sc->mediasize = mdio->md_mediasize; 1501 g_topology_lock(); 1502 g_resize_provider(sc->pp, sc->mediasize); 1503 g_topology_unlock(); 1504 return (0); 1505 } 1506 1507 static int 1508 mdcreate_swap(struct md_s *sc, struct md_ioctl *mdio, struct thread *td) 1509 { 1510 vm_ooffset_t npage; 1511 int error; 1512 1513 /* 1514 * Range check. Disallow negative sizes and sizes not being 1515 * multiple of page size. 1516 */ 1517 if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) 1518 return (EDOM); 1519 1520 /* 1521 * Allocate an OBJT_SWAP object. 1522 * 1523 * Note the truncation. 1524 */ 1525 1526 npage = mdio->md_mediasize / PAGE_SIZE; 1527 if (mdio->md_fwsectors != 0) 1528 sc->fwsectors = mdio->md_fwsectors; 1529 if (mdio->md_fwheads != 0) 1530 sc->fwheads = mdio->md_fwheads; 1531 sc->object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * npage, 1532 VM_PROT_DEFAULT, 0, td->td_ucred); 1533 if (sc->object == NULL) 1534 return (ENOMEM); 1535 sc->flags = mdio->md_options & (MD_FORCE | MD_RESERVE); 1536 if (mdio->md_options & MD_RESERVE) { 1537 if (swap_pager_reserve(sc->object, 0, npage) < 0) { 1538 error = EDOM; 1539 goto finish; 1540 } 1541 } 1542 error = mdsetcred(sc, td->td_ucred); 1543 finish: 1544 if (error != 0) { 1545 vm_object_deallocate(sc->object); 1546 sc->object = NULL; 1547 } 1548 return (error); 1549 } 1550 1551 static int 1552 mdcreate_null(struct md_s *sc, struct md_ioctl *mdio, struct thread *td) 1553 { 1554 1555 /* 1556 * Range check. Disallow negative sizes and sizes not being 1557 * multiple of page size. 1558 */ 1559 if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) 1560 return (EDOM); 1561 1562 return (0); 1563 } 1564 1565 static int 1566 xmdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) 1567 { 1568 struct md_ioctl *mdio; 1569 struct md_s *sc; 1570 int error, i; 1571 unsigned sectsize; 1572 1573 if (md_debug) 1574 printf("mdctlioctl(%s %lx %p %x %p)\n", 1575 devtoname(dev), cmd, addr, flags, td); 1576 1577 mdio = (struct md_ioctl *)addr; 1578 if (mdio->md_version != MDIOVERSION) 1579 return (EINVAL); 1580 1581 /* 1582 * We assert the version number in the individual ioctl 1583 * handlers instead of out here because (a) it is possible we 1584 * may add another ioctl in the future which doesn't read an 1585 * mdio, and (b) the correct return value for an unknown ioctl 1586 * is ENOIOCTL, not EINVAL. 1587 */ 1588 error = 0; 1589 switch (cmd) { 1590 case MDIOCATTACH: 1591 switch (mdio->md_type) { 1592 case MD_MALLOC: 1593 case MD_PRELOAD: 1594 case MD_VNODE: 1595 case MD_SWAP: 1596 case MD_NULL: 1597 break; 1598 default: 1599 return (EINVAL); 1600 } 1601 if (mdio->md_sectorsize == 0) 1602 sectsize = DEV_BSIZE; 1603 else 1604 sectsize = mdio->md_sectorsize; 1605 if (sectsize > MAXPHYS || mdio->md_mediasize < sectsize) 1606 return (EINVAL); 1607 if (mdio->md_options & MD_AUTOUNIT) 1608 sc = mdnew(-1, &error, mdio->md_type); 1609 else { 1610 if (mdio->md_unit > INT_MAX) 1611 return (EINVAL); 1612 sc = mdnew(mdio->md_unit, &error, mdio->md_type); 1613 } 1614 if (sc == NULL) 1615 return (error); 1616 if (mdio->md_options & MD_AUTOUNIT) 1617 mdio->md_unit = sc->unit; 1618 sc->mediasize = mdio->md_mediasize; 1619 sc->sectorsize = sectsize; 1620 error = EDOOFUS; 1621 switch (sc->type) { 1622 case MD_MALLOC: 1623 sc->start = mdstart_malloc; 1624 error = mdcreate_malloc(sc, mdio); 1625 break; 1626 case MD_PRELOAD: 1627 /* 1628 * We disallow attaching preloaded memory disks via 1629 * ioctl. Preloaded memory disks are automatically 1630 * attached in g_md_init(). 1631 */ 1632 error = EOPNOTSUPP; 1633 break; 1634 case MD_VNODE: 1635 sc->start = mdstart_vnode; 1636 error = mdcreate_vnode(sc, mdio, td); 1637 break; 1638 case MD_SWAP: 1639 sc->start = mdstart_swap; 1640 error = mdcreate_swap(sc, mdio, td); 1641 break; 1642 case MD_NULL: 1643 sc->start = mdstart_null; 1644 error = mdcreate_null(sc, mdio, td); 1645 break; 1646 } 1647 if (error != 0) { 1648 mddestroy(sc, td); 1649 return (error); 1650 } 1651 1652 /* Prune off any residual fractional sector */ 1653 i = sc->mediasize % sc->sectorsize; 1654 sc->mediasize -= i; 1655 1656 mdinit(sc); 1657 return (0); 1658 case MDIOCDETACH: 1659 if (mdio->md_mediasize != 0 || 1660 (mdio->md_options & ~MD_FORCE) != 0) 1661 return (EINVAL); 1662 1663 sc = mdfind(mdio->md_unit); 1664 if (sc == NULL) 1665 return (ENOENT); 1666 if (sc->opencount != 0 && !(sc->flags & MD_FORCE) && 1667 !(mdio->md_options & MD_FORCE)) 1668 return (EBUSY); 1669 return (mddestroy(sc, td)); 1670 case MDIOCRESIZE: 1671 if ((mdio->md_options & ~(MD_FORCE | MD_RESERVE)) != 0) 1672 return (EINVAL); 1673 1674 sc = mdfind(mdio->md_unit); 1675 if (sc == NULL) 1676 return (ENOENT); 1677 if (mdio->md_mediasize < sc->sectorsize) 1678 return (EINVAL); 1679 if (mdio->md_mediasize < sc->mediasize && 1680 !(sc->flags & MD_FORCE) && 1681 !(mdio->md_options & MD_FORCE)) 1682 return (EBUSY); 1683 return (mdresize(sc, mdio)); 1684 case MDIOCQUERY: 1685 sc = mdfind(mdio->md_unit); 1686 if (sc == NULL) 1687 return (ENOENT); 1688 mdio->md_type = sc->type; 1689 mdio->md_options = sc->flags; 1690 mdio->md_mediasize = sc->mediasize; 1691 mdio->md_sectorsize = sc->sectorsize; 1692 if (sc->type == MD_VNODE || 1693 (sc->type == MD_PRELOAD && mdio->md_file != NULL)) 1694 error = copyout(sc->file, mdio->md_file, 1695 strlen(sc->file) + 1); 1696 return (error); 1697 case MDIOCLIST: 1698 i = 1; 1699 LIST_FOREACH(sc, &md_softc_list, list) { 1700 if (i == MDNPAD - 1) 1701 mdio->md_pad[i] = -1; 1702 else 1703 mdio->md_pad[i++] = sc->unit; 1704 } 1705 mdio->md_pad[0] = i - 1; 1706 return (0); 1707 default: 1708 return (ENOIOCTL); 1709 }; 1710 } 1711 1712 static int 1713 mdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) 1714 { 1715 int error; 1716 1717 sx_xlock(&md_sx); 1718 error = xmdctlioctl(dev, cmd, addr, flags, td); 1719 sx_xunlock(&md_sx); 1720 return (error); 1721 } 1722 1723 static void 1724 md_preloaded(u_char *image, size_t length, const char *name) 1725 { 1726 struct md_s *sc; 1727 int error; 1728 1729 sc = mdnew(-1, &error, MD_PRELOAD); 1730 if (sc == NULL) 1731 return; 1732 sc->mediasize = length; 1733 sc->sectorsize = DEV_BSIZE; 1734 sc->pl_ptr = image; 1735 sc->pl_len = length; 1736 sc->start = mdstart_preload; 1737 if (name != NULL) 1738 strlcpy(sc->file, name, sizeof(sc->file)); 1739 #if defined(MD_ROOT) && !defined(ROOTDEVNAME) 1740 if (sc->unit == 0) 1741 rootdevnames[0] = MD_ROOT_FSTYPE ":/dev/md0"; 1742 #endif 1743 mdinit(sc); 1744 if (name != NULL) { 1745 printf("%s%d: Preloaded image <%s> %zd bytes at %p\n", 1746 MD_NAME, sc->unit, name, length, image); 1747 } else { 1748 printf("%s%d: Embedded image %zd bytes at %p\n", 1749 MD_NAME, sc->unit, length, image); 1750 } 1751 } 1752 1753 static void 1754 g_md_init(struct g_class *mp __unused) 1755 { 1756 caddr_t mod; 1757 u_char *ptr, *name, *type; 1758 unsigned len; 1759 int i; 1760 1761 /* figure out log2(NINDIR) */ 1762 for (i = NINDIR, nshift = -1; i; nshift++) 1763 i >>= 1; 1764 1765 mod = NULL; 1766 sx_init(&md_sx, "MD config lock"); 1767 g_topology_unlock(); 1768 md_uh = new_unrhdr(0, INT_MAX, NULL); 1769 #ifdef MD_ROOT 1770 if (mfs_root_size != 0) { 1771 sx_xlock(&md_sx); 1772 md_preloaded(__DEVOLATILE(u_char *, &mfs_root), mfs_root_size, 1773 NULL); 1774 sx_xunlock(&md_sx); 1775 } 1776 #endif 1777 /* XXX: are preload_* static or do they need Giant ? */ 1778 while ((mod = preload_search_next_name(mod)) != NULL) { 1779 name = (char *)preload_search_info(mod, MODINFO_NAME); 1780 if (name == NULL) 1781 continue; 1782 type = (char *)preload_search_info(mod, MODINFO_TYPE); 1783 if (type == NULL) 1784 continue; 1785 if (strcmp(type, "md_image") && strcmp(type, "mfs_root")) 1786 continue; 1787 ptr = preload_fetch_addr(mod); 1788 len = preload_fetch_size(mod); 1789 if (ptr != NULL && len != 0) { 1790 sx_xlock(&md_sx); 1791 md_preloaded(ptr, len, name); 1792 sx_xunlock(&md_sx); 1793 } 1794 } 1795 md_vnode_pbuf_freecnt = nswbuf / 10; 1796 status_dev = make_dev(&mdctl_cdevsw, INT_MAX, UID_ROOT, GID_WHEEL, 1797 0600, MDCTL_NAME); 1798 g_topology_lock(); 1799 } 1800 1801 static void 1802 g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, 1803 struct g_consumer *cp __unused, struct g_provider *pp) 1804 { 1805 struct md_s *mp; 1806 char *type; 1807 1808 mp = gp->softc; 1809 if (mp == NULL) 1810 return; 1811 1812 switch (mp->type) { 1813 case MD_MALLOC: 1814 type = "malloc"; 1815 break; 1816 case MD_PRELOAD: 1817 type = "preload"; 1818 break; 1819 case MD_VNODE: 1820 type = "vnode"; 1821 break; 1822 case MD_SWAP: 1823 type = "swap"; 1824 break; 1825 case MD_NULL: 1826 type = "null"; 1827 break; 1828 default: 1829 type = "unknown"; 1830 break; 1831 } 1832 1833 if (pp != NULL) { 1834 if (indent == NULL) { 1835 sbuf_printf(sb, " u %d", mp->unit); 1836 sbuf_printf(sb, " s %ju", (uintmax_t) mp->sectorsize); 1837 sbuf_printf(sb, " f %ju", (uintmax_t) mp->fwheads); 1838 sbuf_printf(sb, " fs %ju", (uintmax_t) mp->fwsectors); 1839 sbuf_printf(sb, " l %ju", (uintmax_t) mp->mediasize); 1840 sbuf_printf(sb, " t %s", type); 1841 if ((mp->type == MD_VNODE && mp->vnode != NULL) || 1842 (mp->type == MD_PRELOAD && mp->file[0] != '\0')) 1843 sbuf_printf(sb, " file %s", mp->file); 1844 } else { 1845 sbuf_printf(sb, "%s<unit>%d</unit>\n", indent, 1846 mp->unit); 1847 sbuf_printf(sb, "%s<sectorsize>%ju</sectorsize>\n", 1848 indent, (uintmax_t) mp->sectorsize); 1849 sbuf_printf(sb, "%s<fwheads>%ju</fwheads>\n", 1850 indent, (uintmax_t) mp->fwheads); 1851 sbuf_printf(sb, "%s<fwsectors>%ju</fwsectors>\n", 1852 indent, (uintmax_t) mp->fwsectors); 1853 sbuf_printf(sb, "%s<length>%ju</length>\n", 1854 indent, (uintmax_t) mp->mediasize); 1855 sbuf_printf(sb, "%s<compression>%s</compression>\n", indent, 1856 (mp->flags & MD_COMPRESS) == 0 ? "off": "on"); 1857 sbuf_printf(sb, "%s<access>%s</access>\n", indent, 1858 (mp->flags & MD_READONLY) == 0 ? "read-write": 1859 "read-only"); 1860 sbuf_printf(sb, "%s<type>%s</type>\n", indent, 1861 type); 1862 if ((mp->type == MD_VNODE && mp->vnode != NULL) || 1863 (mp->type == MD_PRELOAD && mp->file[0] != '\0')) { 1864 sbuf_printf(sb, "%s<file>", indent); 1865 g_conf_printf_escaped(sb, "%s", mp->file); 1866 sbuf_printf(sb, "</file>\n"); 1867 } 1868 } 1869 } 1870 } 1871 1872 static void 1873 g_md_fini(struct g_class *mp __unused) 1874 { 1875 1876 sx_destroy(&md_sx); 1877 if (status_dev != NULL) 1878 destroy_dev(status_dev); 1879 delete_unrhdr(md_uh); 1880 } 1881