1 /*- 2 * SPDX-License-Identifier: (Beerware AND BSD-3-Clause) 3 * 4 * ---------------------------------------------------------------------------- 5 * "THE BEER-WARE LICENSE" (Revision 42): 6 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 7 * can do whatever you want with this stuff. If we meet some day, and you think 8 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 9 * ---------------------------------------------------------------------------- 10 * 11 * $FreeBSD$ 12 * 13 */ 14 15 /*- 16 * The following functions are based on the vn(4) driver: mdstart_swap(), 17 * mdstart_vnode(), mdcreate_swap(), mdcreate_vnode() and mddestroy(), 18 * and as such under the following copyright: 19 * 20 * Copyright (c) 1988 University of Utah. 21 * Copyright (c) 1990, 1993 22 * The Regents of the University of California. All rights reserved. 23 * Copyright (c) 2013 The FreeBSD Foundation 24 * All rights reserved. 25 * 26 * This code is derived from software contributed to Berkeley by 27 * the Systems Programming Group of the University of Utah Computer 28 * Science Department. 29 * 30 * Portions of this software were developed by Konstantin Belousov 31 * under sponsorship from the FreeBSD Foundation. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. Neither the name of the University nor the names of its contributors 42 * may be used to endorse or promote products derived from this software 43 * without specific prior written permission. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 * 57 * from: Utah Hdr: vn.c 1.13 94/04/02 58 * 59 * from: @(#)vn.c 8.6 (Berkeley) 4/1/94 60 * From: src/sys/dev/vn/vn.c,v 1.122 2000/12/16 16:06:03 61 */ 62 63 #include "opt_rootdevname.h" 64 #include "opt_geom.h" 65 #include "opt_md.h" 66 67 #include <sys/param.h> 68 #include <sys/systm.h> 69 #include <sys/bio.h> 70 #include <sys/buf.h> 71 #include <sys/conf.h> 72 #include <sys/devicestat.h> 73 #include <sys/fcntl.h> 74 #include <sys/kernel.h> 75 #include <sys/kthread.h> 76 #include <sys/limits.h> 77 #include <sys/linker.h> 78 #include <sys/lock.h> 79 #include <sys/malloc.h> 80 #include <sys/mdioctl.h> 81 #include <sys/mount.h> 82 #include <sys/mutex.h> 83 #include <sys/sx.h> 84 #include <sys/namei.h> 85 #include <sys/proc.h> 86 #include <sys/queue.h> 87 #include <sys/rwlock.h> 88 #include <sys/sbuf.h> 89 #include <sys/sched.h> 90 #include <sys/sf_buf.h> 91 #include <sys/sysctl.h> 92 #include <sys/uio.h> 93 #include <sys/vnode.h> 94 #include <sys/disk.h> 95 96 #include <geom/geom.h> 97 #include <geom/geom_int.h> 98 99 #include <vm/vm.h> 100 #include <vm/vm_param.h> 101 #include <vm/vm_object.h> 102 #include <vm/vm_page.h> 103 #include <vm/vm_pager.h> 104 #include <vm/swap_pager.h> 105 #include <vm/uma.h> 106 107 #include <machine/bus.h> 108 109 #define MD_MODVER 1 110 111 #define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */ 112 #define MD_EXITING 0x20000 /* Worker thread is exiting. */ 113 #define MD_PROVIDERGONE 0x40000 /* Safe to free the softc */ 114 115 #ifndef MD_NSECT 116 #define MD_NSECT (10000 * 2) 117 #endif 118 119 struct md_req { 120 unsigned md_unit; /* unit number */ 121 enum md_types md_type; /* type of disk */ 122 off_t md_mediasize; /* size of disk in bytes */ 123 unsigned md_sectorsize; /* sectorsize */ 124 unsigned md_options; /* options */ 125 int md_fwheads; /* firmware heads */ 126 int md_fwsectors; /* firmware sectors */ 127 char *md_file; /* pathname of file to mount */ 128 enum uio_seg md_file_seg; /* location of md_file */ 129 char *md_label; /* label of the device (userspace) */ 130 int *md_units; /* pointer to units array (kernel) */ 131 size_t md_units_nitems; /* items in md_units array */ 132 }; 133 134 #ifdef COMPAT_FREEBSD32 135 struct md_ioctl32 { 136 unsigned md_version; 137 unsigned md_unit; 138 enum md_types md_type; 139 uint32_t md_file; 140 off_t md_mediasize; 141 unsigned md_sectorsize; 142 unsigned md_options; 143 uint64_t md_base; 144 int md_fwheads; 145 int md_fwsectors; 146 uint32_t md_label; 147 int md_pad[MDNPAD]; 148 } __attribute__((__packed__)); 149 CTASSERT((sizeof(struct md_ioctl32)) == 436); 150 151 #define MDIOCATTACH_32 _IOC_NEWTYPE(MDIOCATTACH, struct md_ioctl32) 152 #define MDIOCDETACH_32 _IOC_NEWTYPE(MDIOCDETACH, struct md_ioctl32) 153 #define MDIOCQUERY_32 _IOC_NEWTYPE(MDIOCQUERY, struct md_ioctl32) 154 #define MDIOCRESIZE_32 _IOC_NEWTYPE(MDIOCRESIZE, struct md_ioctl32) 155 #endif /* COMPAT_FREEBSD32 */ 156 157 static MALLOC_DEFINE(M_MD, "md_disk", "Memory Disk"); 158 static MALLOC_DEFINE(M_MDSECT, "md_sectors", "Memory Disk Sectors"); 159 160 static int md_debug; 161 SYSCTL_INT(_debug, OID_AUTO, mddebug, CTLFLAG_RW, &md_debug, 0, 162 "Enable md(4) debug messages"); 163 static int md_malloc_wait; 164 SYSCTL_INT(_vm, OID_AUTO, md_malloc_wait, CTLFLAG_RW, &md_malloc_wait, 0, 165 "Allow malloc to wait for memory allocations"); 166 167 #if defined(MD_ROOT) && !defined(MD_ROOT_FSTYPE) 168 #define MD_ROOT_FSTYPE "ufs" 169 #endif 170 171 #if defined(MD_ROOT) 172 /* 173 * Preloaded image gets put here. 174 */ 175 #if defined(MD_ROOT_SIZE) 176 /* 177 * We put the mfs_root symbol into the oldmfs section of the kernel object file. 178 * Applications that patch the object with the image can determine 179 * the size looking at the oldmfs section size within the kernel. 180 */ 181 u_char mfs_root[MD_ROOT_SIZE*1024] __attribute__ ((section ("oldmfs"))); 182 const int mfs_root_size = sizeof(mfs_root); 183 #elif defined(MD_ROOT_MEM) 184 /* MD region already mapped in the memory */ 185 u_char *mfs_root; 186 int mfs_root_size; 187 #else 188 extern volatile u_char __weak_symbol mfs_root; 189 extern volatile u_char __weak_symbol mfs_root_end; 190 #define mfs_root_size ((uintptr_t)(&mfs_root_end - &mfs_root)) 191 #endif 192 #endif 193 194 static g_init_t g_md_init; 195 static g_fini_t g_md_fini; 196 static g_start_t g_md_start; 197 static g_access_t g_md_access; 198 static void g_md_dumpconf(struct sbuf *sb, const char *indent, 199 struct g_geom *gp, struct g_consumer *cp __unused, struct g_provider *pp); 200 static g_provgone_t g_md_providergone; 201 202 static struct cdev *status_dev = NULL; 203 static struct sx md_sx; 204 static struct unrhdr *md_uh; 205 206 static d_ioctl_t mdctlioctl; 207 208 static struct cdevsw mdctl_cdevsw = { 209 .d_version = D_VERSION, 210 .d_ioctl = mdctlioctl, 211 .d_name = MD_NAME, 212 }; 213 214 struct g_class g_md_class = { 215 .name = "MD", 216 .version = G_VERSION, 217 .init = g_md_init, 218 .fini = g_md_fini, 219 .start = g_md_start, 220 .access = g_md_access, 221 .dumpconf = g_md_dumpconf, 222 .providergone = g_md_providergone, 223 }; 224 225 DECLARE_GEOM_CLASS(g_md_class, g_md); 226 227 static LIST_HEAD(, md_s) md_softc_list = LIST_HEAD_INITIALIZER(md_softc_list); 228 229 #define NINDIR (PAGE_SIZE / sizeof(uintptr_t)) 230 #define NMASK (NINDIR-1) 231 static int nshift; 232 233 static uma_zone_t md_pbuf_zone; 234 235 struct indir { 236 uintptr_t *array; 237 u_int total; 238 u_int used; 239 u_int shift; 240 }; 241 242 struct md_s { 243 int unit; 244 LIST_ENTRY(md_s) list; 245 struct bio_queue_head bio_queue; 246 struct mtx queue_mtx; 247 struct cdev *dev; 248 enum md_types type; 249 off_t mediasize; 250 unsigned sectorsize; 251 unsigned opencount; 252 unsigned fwheads; 253 unsigned fwsectors; 254 char ident[32]; 255 unsigned flags; 256 char name[20]; 257 struct proc *procp; 258 struct g_geom *gp; 259 struct g_provider *pp; 260 int (*start)(struct md_s *sc, struct bio *bp); 261 struct devstat *devstat; 262 263 /* MD_MALLOC related fields */ 264 struct indir *indir; 265 uma_zone_t uma; 266 267 /* MD_PRELOAD related fields */ 268 u_char *pl_ptr; 269 size_t pl_len; 270 271 /* MD_VNODE related fields */ 272 struct vnode *vnode; 273 char file[PATH_MAX]; 274 char label[PATH_MAX]; 275 struct ucred *cred; 276 277 /* MD_SWAP related fields */ 278 vm_object_t object; 279 }; 280 281 static struct indir * 282 new_indir(u_int shift) 283 { 284 struct indir *ip; 285 286 ip = malloc(sizeof *ip, M_MD, (md_malloc_wait ? M_WAITOK : M_NOWAIT) 287 | M_ZERO); 288 if (ip == NULL) 289 return (NULL); 290 ip->array = malloc(sizeof(uintptr_t) * NINDIR, 291 M_MDSECT, (md_malloc_wait ? M_WAITOK : M_NOWAIT) | M_ZERO); 292 if (ip->array == NULL) { 293 free(ip, M_MD); 294 return (NULL); 295 } 296 ip->total = NINDIR; 297 ip->shift = shift; 298 return (ip); 299 } 300 301 static void 302 del_indir(struct indir *ip) 303 { 304 305 free(ip->array, M_MDSECT); 306 free(ip, M_MD); 307 } 308 309 static void 310 destroy_indir(struct md_s *sc, struct indir *ip) 311 { 312 int i; 313 314 for (i = 0; i < NINDIR; i++) { 315 if (!ip->array[i]) 316 continue; 317 if (ip->shift) 318 destroy_indir(sc, (struct indir*)(ip->array[i])); 319 else if (ip->array[i] > 255) 320 uma_zfree(sc->uma, (void *)(ip->array[i])); 321 } 322 del_indir(ip); 323 } 324 325 /* 326 * This function does the math and allocates the top level "indir" structure 327 * for a device of "size" sectors. 328 */ 329 330 static struct indir * 331 dimension(off_t size) 332 { 333 off_t rcnt; 334 struct indir *ip; 335 int layer; 336 337 rcnt = size; 338 layer = 0; 339 while (rcnt > NINDIR) { 340 rcnt /= NINDIR; 341 layer++; 342 } 343 344 /* 345 * XXX: the top layer is probably not fully populated, so we allocate 346 * too much space for ip->array in here. 347 */ 348 ip = malloc(sizeof *ip, M_MD, M_WAITOK | M_ZERO); 349 ip->array = malloc(sizeof(uintptr_t) * NINDIR, 350 M_MDSECT, M_WAITOK | M_ZERO); 351 ip->total = NINDIR; 352 ip->shift = layer * nshift; 353 return (ip); 354 } 355 356 /* 357 * Read a given sector 358 */ 359 360 static uintptr_t 361 s_read(struct indir *ip, off_t offset) 362 { 363 struct indir *cip; 364 int idx; 365 uintptr_t up; 366 367 if (md_debug > 1) 368 printf("s_read(%jd)\n", (intmax_t)offset); 369 up = 0; 370 for (cip = ip; cip != NULL;) { 371 if (cip->shift) { 372 idx = (offset >> cip->shift) & NMASK; 373 up = cip->array[idx]; 374 cip = (struct indir *)up; 375 continue; 376 } 377 idx = offset & NMASK; 378 return (cip->array[idx]); 379 } 380 return (0); 381 } 382 383 /* 384 * Write a given sector, prune the tree if the value is 0 385 */ 386 387 static int 388 s_write(struct indir *ip, off_t offset, uintptr_t ptr) 389 { 390 struct indir *cip, *lip[10]; 391 int idx, li; 392 uintptr_t up; 393 394 if (md_debug > 1) 395 printf("s_write(%jd, %p)\n", (intmax_t)offset, (void *)ptr); 396 up = 0; 397 li = 0; 398 cip = ip; 399 for (;;) { 400 lip[li++] = cip; 401 if (cip->shift) { 402 idx = (offset >> cip->shift) & NMASK; 403 up = cip->array[idx]; 404 if (up != 0) { 405 cip = (struct indir *)up; 406 continue; 407 } 408 /* Allocate branch */ 409 cip->array[idx] = 410 (uintptr_t)new_indir(cip->shift - nshift); 411 if (cip->array[idx] == 0) 412 return (ENOSPC); 413 cip->used++; 414 up = cip->array[idx]; 415 cip = (struct indir *)up; 416 continue; 417 } 418 /* leafnode */ 419 idx = offset & NMASK; 420 up = cip->array[idx]; 421 if (up != 0) 422 cip->used--; 423 cip->array[idx] = ptr; 424 if (ptr != 0) 425 cip->used++; 426 break; 427 } 428 if (cip->used != 0 || li == 1) 429 return (0); 430 li--; 431 while (cip->used == 0 && cip != ip) { 432 li--; 433 idx = (offset >> lip[li]->shift) & NMASK; 434 up = lip[li]->array[idx]; 435 KASSERT(up == (uintptr_t)cip, ("md screwed up")); 436 del_indir(cip); 437 lip[li]->array[idx] = 0; 438 lip[li]->used--; 439 cip = lip[li]; 440 } 441 return (0); 442 } 443 444 static int 445 g_md_access(struct g_provider *pp, int r, int w, int e) 446 { 447 struct md_s *sc; 448 449 sc = pp->geom->softc; 450 if (sc == NULL) { 451 if (r <= 0 && w <= 0 && e <= 0) 452 return (0); 453 return (ENXIO); 454 } 455 r += pp->acr; 456 w += pp->acw; 457 e += pp->ace; 458 if ((sc->flags & MD_READONLY) != 0 && w > 0) 459 return (EROFS); 460 if ((pp->acr + pp->acw + pp->ace) == 0 && (r + w + e) > 0) { 461 sc->opencount = 1; 462 } else if ((pp->acr + pp->acw + pp->ace) > 0 && (r + w + e) == 0) { 463 sc->opencount = 0; 464 } 465 return (0); 466 } 467 468 static void 469 g_md_start(struct bio *bp) 470 { 471 struct md_s *sc; 472 473 sc = bp->bio_to->geom->softc; 474 if ((bp->bio_cmd == BIO_READ) || (bp->bio_cmd == BIO_WRITE)) { 475 devstat_start_transaction_bio(sc->devstat, bp); 476 } 477 mtx_lock(&sc->queue_mtx); 478 bioq_disksort(&sc->bio_queue, bp); 479 wakeup(sc); 480 mtx_unlock(&sc->queue_mtx); 481 } 482 483 #define MD_MALLOC_MOVE_ZERO 1 484 #define MD_MALLOC_MOVE_FILL 2 485 #define MD_MALLOC_MOVE_READ 3 486 #define MD_MALLOC_MOVE_WRITE 4 487 #define MD_MALLOC_MOVE_CMP 5 488 489 static int 490 md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize, 491 void *ptr, u_char fill, int op) 492 { 493 struct sf_buf *sf; 494 vm_page_t m, *mp1; 495 char *p, first; 496 off_t *uc; 497 unsigned n; 498 int error, i, ma_offs1, sz, first_read; 499 500 m = NULL; 501 error = 0; 502 sf = NULL; 503 /* if (op == MD_MALLOC_MOVE_CMP) { gcc */ 504 first = 0; 505 first_read = 0; 506 uc = ptr; 507 mp1 = *mp; 508 ma_offs1 = *ma_offs; 509 /* } */ 510 sched_pin(); 511 for (n = sectorsize; n != 0; n -= sz) { 512 sz = imin(PAGE_SIZE - *ma_offs, n); 513 if (m != **mp) { 514 if (sf != NULL) 515 sf_buf_free(sf); 516 m = **mp; 517 sf = sf_buf_alloc(m, SFB_CPUPRIVATE | 518 (md_malloc_wait ? 0 : SFB_NOWAIT)); 519 if (sf == NULL) { 520 error = ENOMEM; 521 break; 522 } 523 } 524 p = (char *)sf_buf_kva(sf) + *ma_offs; 525 switch (op) { 526 case MD_MALLOC_MOVE_ZERO: 527 bzero(p, sz); 528 break; 529 case MD_MALLOC_MOVE_FILL: 530 memset(p, fill, sz); 531 break; 532 case MD_MALLOC_MOVE_READ: 533 bcopy(ptr, p, sz); 534 cpu_flush_dcache(p, sz); 535 break; 536 case MD_MALLOC_MOVE_WRITE: 537 bcopy(p, ptr, sz); 538 break; 539 case MD_MALLOC_MOVE_CMP: 540 for (i = 0; i < sz; i++, p++) { 541 if (!first_read) { 542 *uc = (u_char)*p; 543 first = *p; 544 first_read = 1; 545 } else if (*p != first) { 546 error = EDOOFUS; 547 break; 548 } 549 } 550 break; 551 default: 552 KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op)); 553 break; 554 } 555 if (error != 0) 556 break; 557 *ma_offs += sz; 558 *ma_offs %= PAGE_SIZE; 559 if (*ma_offs == 0) 560 (*mp)++; 561 ptr = (char *)ptr + sz; 562 } 563 564 if (sf != NULL) 565 sf_buf_free(sf); 566 sched_unpin(); 567 if (op == MD_MALLOC_MOVE_CMP && error != 0) { 568 *mp = mp1; 569 *ma_offs = ma_offs1; 570 } 571 return (error); 572 } 573 574 static int 575 md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs, 576 unsigned len, void *ptr, u_char fill, int op) 577 { 578 bus_dma_segment_t *vlist; 579 uint8_t *p, *end, first; 580 off_t *uc; 581 int ma_offs, seg_len; 582 583 vlist = *pvlist; 584 ma_offs = *pma_offs; 585 uc = ptr; 586 587 for (; len != 0; len -= seg_len) { 588 seg_len = imin(vlist->ds_len - ma_offs, len); 589 p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs; 590 switch (op) { 591 case MD_MALLOC_MOVE_ZERO: 592 bzero(p, seg_len); 593 break; 594 case MD_MALLOC_MOVE_FILL: 595 memset(p, fill, seg_len); 596 break; 597 case MD_MALLOC_MOVE_READ: 598 bcopy(ptr, p, seg_len); 599 cpu_flush_dcache(p, seg_len); 600 break; 601 case MD_MALLOC_MOVE_WRITE: 602 bcopy(p, ptr, seg_len); 603 break; 604 case MD_MALLOC_MOVE_CMP: 605 end = p + seg_len; 606 first = *uc = *p; 607 /* Confirm all following bytes match the first */ 608 while (++p < end) { 609 if (*p != first) 610 return (EDOOFUS); 611 } 612 break; 613 default: 614 KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op)); 615 break; 616 } 617 618 ma_offs += seg_len; 619 if (ma_offs == vlist->ds_len) { 620 ma_offs = 0; 621 vlist++; 622 } 623 ptr = (uint8_t *)ptr + seg_len; 624 } 625 *pvlist = vlist; 626 *pma_offs = ma_offs; 627 628 return (0); 629 } 630 631 static int 632 mdstart_malloc(struct md_s *sc, struct bio *bp) 633 { 634 u_char *dst; 635 vm_page_t *m; 636 bus_dma_segment_t *vlist; 637 int i, error, error1, ma_offs, notmapped; 638 off_t secno, nsec, uc; 639 uintptr_t sp, osp; 640 641 switch (bp->bio_cmd) { 642 case BIO_READ: 643 case BIO_WRITE: 644 case BIO_DELETE: 645 break; 646 default: 647 return (EOPNOTSUPP); 648 } 649 650 notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0; 651 vlist = (bp->bio_flags & BIO_VLIST) != 0 ? 652 (bus_dma_segment_t *)bp->bio_data : NULL; 653 if (notmapped) { 654 m = bp->bio_ma; 655 ma_offs = bp->bio_ma_offset; 656 dst = NULL; 657 KASSERT(vlist == NULL, ("vlists cannot be unmapped")); 658 } else if (vlist != NULL) { 659 ma_offs = bp->bio_ma_offset; 660 dst = NULL; 661 } else { 662 dst = bp->bio_data; 663 } 664 665 nsec = bp->bio_length / sc->sectorsize; 666 secno = bp->bio_offset / sc->sectorsize; 667 error = 0; 668 while (nsec--) { 669 osp = s_read(sc->indir, secno); 670 if (bp->bio_cmd == BIO_DELETE) { 671 if (osp != 0) 672 error = s_write(sc->indir, secno, 0); 673 } else if (bp->bio_cmd == BIO_READ) { 674 if (osp == 0) { 675 if (notmapped) { 676 error = md_malloc_move_ma(&m, &ma_offs, 677 sc->sectorsize, NULL, 0, 678 MD_MALLOC_MOVE_ZERO); 679 } else if (vlist != NULL) { 680 error = md_malloc_move_vlist(&vlist, 681 &ma_offs, sc->sectorsize, NULL, 0, 682 MD_MALLOC_MOVE_ZERO); 683 } else 684 bzero(dst, sc->sectorsize); 685 } else if (osp <= 255) { 686 if (notmapped) { 687 error = md_malloc_move_ma(&m, &ma_offs, 688 sc->sectorsize, NULL, osp, 689 MD_MALLOC_MOVE_FILL); 690 } else if (vlist != NULL) { 691 error = md_malloc_move_vlist(&vlist, 692 &ma_offs, sc->sectorsize, NULL, osp, 693 MD_MALLOC_MOVE_FILL); 694 } else 695 memset(dst, osp, sc->sectorsize); 696 } else { 697 if (notmapped) { 698 error = md_malloc_move_ma(&m, &ma_offs, 699 sc->sectorsize, (void *)osp, 0, 700 MD_MALLOC_MOVE_READ); 701 } else if (vlist != NULL) { 702 error = md_malloc_move_vlist(&vlist, 703 &ma_offs, sc->sectorsize, 704 (void *)osp, 0, 705 MD_MALLOC_MOVE_READ); 706 } else { 707 bcopy((void *)osp, dst, sc->sectorsize); 708 cpu_flush_dcache(dst, sc->sectorsize); 709 } 710 } 711 osp = 0; 712 } else if (bp->bio_cmd == BIO_WRITE) { 713 if (sc->flags & MD_COMPRESS) { 714 if (notmapped) { 715 error1 = md_malloc_move_ma(&m, &ma_offs, 716 sc->sectorsize, &uc, 0, 717 MD_MALLOC_MOVE_CMP); 718 i = error1 == 0 ? sc->sectorsize : 0; 719 } else if (vlist != NULL) { 720 error1 = md_malloc_move_vlist(&vlist, 721 &ma_offs, sc->sectorsize, &uc, 0, 722 MD_MALLOC_MOVE_CMP); 723 i = error1 == 0 ? sc->sectorsize : 0; 724 } else { 725 uc = dst[0]; 726 for (i = 1; i < sc->sectorsize; i++) { 727 if (dst[i] != uc) 728 break; 729 } 730 } 731 } else { 732 i = 0; 733 uc = 0; 734 } 735 if (i == sc->sectorsize) { 736 if (osp != uc) 737 error = s_write(sc->indir, secno, uc); 738 } else { 739 if (osp <= 255) { 740 sp = (uintptr_t)uma_zalloc(sc->uma, 741 md_malloc_wait ? M_WAITOK : 742 M_NOWAIT); 743 if (sp == 0) { 744 error = ENOSPC; 745 break; 746 } 747 if (notmapped) { 748 error = md_malloc_move_ma(&m, 749 &ma_offs, sc->sectorsize, 750 (void *)sp, 0, 751 MD_MALLOC_MOVE_WRITE); 752 } else if (vlist != NULL) { 753 error = md_malloc_move_vlist( 754 &vlist, &ma_offs, 755 sc->sectorsize, (void *)sp, 756 0, MD_MALLOC_MOVE_WRITE); 757 } else { 758 bcopy(dst, (void *)sp, 759 sc->sectorsize); 760 } 761 error = s_write(sc->indir, secno, sp); 762 } else { 763 if (notmapped) { 764 error = md_malloc_move_ma(&m, 765 &ma_offs, sc->sectorsize, 766 (void *)osp, 0, 767 MD_MALLOC_MOVE_WRITE); 768 } else if (vlist != NULL) { 769 error = md_malloc_move_vlist( 770 &vlist, &ma_offs, 771 sc->sectorsize, (void *)osp, 772 0, MD_MALLOC_MOVE_WRITE); 773 } else { 774 bcopy(dst, (void *)osp, 775 sc->sectorsize); 776 } 777 osp = 0; 778 } 779 } 780 } else { 781 error = EOPNOTSUPP; 782 } 783 if (osp > 255) 784 uma_zfree(sc->uma, (void*)osp); 785 if (error != 0) 786 break; 787 secno++; 788 if (!notmapped && vlist == NULL) 789 dst += sc->sectorsize; 790 } 791 bp->bio_resid = 0; 792 return (error); 793 } 794 795 static void 796 mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len) 797 { 798 off_t seg_len; 799 800 while (offset >= vlist->ds_len) { 801 offset -= vlist->ds_len; 802 vlist++; 803 } 804 805 while (len != 0) { 806 seg_len = omin(len, vlist->ds_len - offset); 807 bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset), 808 seg_len); 809 offset = 0; 810 src = (uint8_t *)src + seg_len; 811 len -= seg_len; 812 vlist++; 813 } 814 } 815 816 static void 817 mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len) 818 { 819 off_t seg_len; 820 821 while (offset >= vlist->ds_len) { 822 offset -= vlist->ds_len; 823 vlist++; 824 } 825 826 while (len != 0) { 827 seg_len = omin(len, vlist->ds_len - offset); 828 bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst, 829 seg_len); 830 offset = 0; 831 dst = (uint8_t *)dst + seg_len; 832 len -= seg_len; 833 vlist++; 834 } 835 } 836 837 static int 838 mdstart_preload(struct md_s *sc, struct bio *bp) 839 { 840 uint8_t *p; 841 842 p = sc->pl_ptr + bp->bio_offset; 843 switch (bp->bio_cmd) { 844 case BIO_READ: 845 if ((bp->bio_flags & BIO_VLIST) != 0) { 846 mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data, 847 bp->bio_ma_offset, bp->bio_length); 848 } else { 849 bcopy(p, bp->bio_data, bp->bio_length); 850 } 851 cpu_flush_dcache(bp->bio_data, bp->bio_length); 852 break; 853 case BIO_WRITE: 854 if ((bp->bio_flags & BIO_VLIST) != 0) { 855 mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data, 856 bp->bio_ma_offset, p, bp->bio_length); 857 } else { 858 bcopy(bp->bio_data, p, bp->bio_length); 859 } 860 break; 861 } 862 bp->bio_resid = 0; 863 return (0); 864 } 865 866 static int 867 mdstart_vnode(struct md_s *sc, struct bio *bp) 868 { 869 int error; 870 struct uio auio; 871 struct iovec aiov; 872 struct iovec *piov; 873 struct mount *mp; 874 struct vnode *vp; 875 struct buf *pb; 876 bus_dma_segment_t *vlist; 877 struct thread *td; 878 off_t iolen, iostart, off, len; 879 int ma_offs, npages; 880 881 switch (bp->bio_cmd) { 882 case BIO_READ: 883 auio.uio_rw = UIO_READ; 884 break; 885 case BIO_WRITE: 886 auio.uio_rw = UIO_WRITE; 887 break; 888 case BIO_DELETE: 889 case BIO_FLUSH: 890 break; 891 default: 892 return (EOPNOTSUPP); 893 } 894 895 td = curthread; 896 vp = sc->vnode; 897 pb = NULL; 898 piov = NULL; 899 ma_offs = bp->bio_ma_offset; 900 off = bp->bio_offset; 901 len = bp->bio_length; 902 903 /* 904 * VNODE I/O 905 * 906 * If an error occurs, we set BIO_ERROR but we do not set 907 * B_INVAL because (for a write anyway), the buffer is 908 * still valid. 909 */ 910 911 if (bp->bio_cmd == BIO_FLUSH) { 912 (void) vn_start_write(vp, &mp, V_WAIT); 913 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 914 error = VOP_FSYNC(vp, MNT_WAIT, td); 915 VOP_UNLOCK(vp); 916 vn_finished_write(mp); 917 return (error); 918 } else if (bp->bio_cmd == BIO_DELETE) { 919 error = vn_deallocate(vp, &off, &len, 0, 920 sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred, NOCRED); 921 bp->bio_resid = len; 922 return (error); 923 } 924 925 auio.uio_offset = (vm_ooffset_t)bp->bio_offset; 926 auio.uio_resid = bp->bio_length; 927 auio.uio_segflg = UIO_SYSSPACE; 928 auio.uio_td = td; 929 930 if ((bp->bio_flags & BIO_VLIST) != 0) { 931 piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK); 932 auio.uio_iov = piov; 933 vlist = (bus_dma_segment_t *)bp->bio_data; 934 while (len > 0) { 935 piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr + 936 ma_offs); 937 piov->iov_len = vlist->ds_len - ma_offs; 938 if (piov->iov_len > len) 939 piov->iov_len = len; 940 len -= piov->iov_len; 941 ma_offs = 0; 942 vlist++; 943 piov++; 944 } 945 auio.uio_iovcnt = piov - auio.uio_iov; 946 piov = auio.uio_iov; 947 } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 948 pb = uma_zalloc(md_pbuf_zone, M_WAITOK); 949 MPASS((pb->b_flags & B_MAXPHYS) != 0); 950 bp->bio_resid = len; 951 unmapped_step: 952 npages = atop(min(maxphys, round_page(len + (ma_offs & 953 PAGE_MASK)))); 954 iolen = min(ptoa(npages) - (ma_offs & PAGE_MASK), len); 955 KASSERT(iolen > 0, ("zero iolen")); 956 pmap_qenter((vm_offset_t)pb->b_data, 957 &bp->bio_ma[atop(ma_offs)], npages); 958 aiov.iov_base = (void *)((vm_offset_t)pb->b_data + 959 (ma_offs & PAGE_MASK)); 960 aiov.iov_len = iolen; 961 auio.uio_iov = &aiov; 962 auio.uio_iovcnt = 1; 963 auio.uio_resid = iolen; 964 } else { 965 aiov.iov_base = bp->bio_data; 966 aiov.iov_len = bp->bio_length; 967 auio.uio_iov = &aiov; 968 auio.uio_iovcnt = 1; 969 } 970 iostart = auio.uio_offset; 971 if (auio.uio_rw == UIO_READ) { 972 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 973 error = VOP_READ(vp, &auio, 0, sc->cred); 974 VOP_UNLOCK(vp); 975 } else { 976 (void) vn_start_write(vp, &mp, V_WAIT); 977 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 978 error = VOP_WRITE(vp, &auio, sc->flags & MD_ASYNC ? 0 : IO_SYNC, 979 sc->cred); 980 VOP_UNLOCK(vp); 981 vn_finished_write(mp); 982 if (error == 0) 983 sc->flags &= ~MD_VERIFY; 984 } 985 986 /* When MD_CACHE is set, try to avoid double-caching the data. */ 987 if (error == 0 && (sc->flags & MD_CACHE) == 0) 988 VOP_ADVISE(vp, iostart, auio.uio_offset - 1, 989 POSIX_FADV_DONTNEED); 990 991 if (pb != NULL) { 992 pmap_qremove((vm_offset_t)pb->b_data, npages); 993 if (error == 0) { 994 len -= iolen; 995 bp->bio_resid -= iolen; 996 ma_offs += iolen; 997 if (len > 0) 998 goto unmapped_step; 999 } 1000 uma_zfree(md_pbuf_zone, pb); 1001 } else { 1002 bp->bio_resid = auio.uio_resid; 1003 } 1004 1005 free(piov, M_MD); 1006 return (error); 1007 } 1008 1009 static int 1010 mdstart_swap(struct md_s *sc, struct bio *bp) 1011 { 1012 vm_page_t m; 1013 u_char *p; 1014 vm_pindex_t i, lastp; 1015 bus_dma_segment_t *vlist; 1016 int rv, ma_offs, offs, len, lastend; 1017 1018 switch (bp->bio_cmd) { 1019 case BIO_READ: 1020 case BIO_WRITE: 1021 case BIO_DELETE: 1022 break; 1023 default: 1024 return (EOPNOTSUPP); 1025 } 1026 1027 p = bp->bio_data; 1028 ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ? 1029 bp->bio_ma_offset : 0; 1030 vlist = (bp->bio_flags & BIO_VLIST) != 0 ? 1031 (bus_dma_segment_t *)bp->bio_data : NULL; 1032 1033 /* 1034 * offs is the offset at which to start operating on the 1035 * next (ie, first) page. lastp is the last page on 1036 * which we're going to operate. lastend is the ending 1037 * position within that last page (ie, PAGE_SIZE if 1038 * we're operating on complete aligned pages). 1039 */ 1040 offs = bp->bio_offset % PAGE_SIZE; 1041 lastp = (bp->bio_offset + bp->bio_length - 1) / PAGE_SIZE; 1042 lastend = (bp->bio_offset + bp->bio_length - 1) % PAGE_SIZE + 1; 1043 1044 rv = VM_PAGER_OK; 1045 vm_object_pip_add(sc->object, 1); 1046 for (i = bp->bio_offset / PAGE_SIZE; i <= lastp; i++) { 1047 len = ((i == lastp) ? lastend : PAGE_SIZE) - offs; 1048 m = vm_page_grab_unlocked(sc->object, i, VM_ALLOC_SYSTEM); 1049 if (bp->bio_cmd == BIO_READ) { 1050 if (vm_page_all_valid(m)) 1051 rv = VM_PAGER_OK; 1052 else 1053 rv = vm_pager_get_pages(sc->object, &m, 1, 1054 NULL, NULL); 1055 if (rv == VM_PAGER_ERROR) { 1056 VM_OBJECT_WLOCK(sc->object); 1057 vm_page_free(m); 1058 VM_OBJECT_WUNLOCK(sc->object); 1059 break; 1060 } else if (rv == VM_PAGER_FAIL) { 1061 /* 1062 * Pager does not have the page. Zero 1063 * the allocated page, and mark it as 1064 * valid. Do not set dirty, the page 1065 * can be recreated if thrown out. 1066 */ 1067 pmap_zero_page(m); 1068 vm_page_valid(m); 1069 } 1070 if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 1071 pmap_copy_pages(&m, offs, bp->bio_ma, 1072 ma_offs, len); 1073 } else if ((bp->bio_flags & BIO_VLIST) != 0) { 1074 physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs, 1075 vlist, ma_offs, len); 1076 cpu_flush_dcache(p, len); 1077 } else { 1078 physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len); 1079 cpu_flush_dcache(p, len); 1080 } 1081 } else if (bp->bio_cmd == BIO_WRITE) { 1082 if (len == PAGE_SIZE || vm_page_all_valid(m)) 1083 rv = VM_PAGER_OK; 1084 else 1085 rv = vm_pager_get_pages(sc->object, &m, 1, 1086 NULL, NULL); 1087 if (rv == VM_PAGER_ERROR) { 1088 VM_OBJECT_WLOCK(sc->object); 1089 vm_page_free(m); 1090 VM_OBJECT_WUNLOCK(sc->object); 1091 break; 1092 } else if (rv == VM_PAGER_FAIL) 1093 pmap_zero_page(m); 1094 1095 if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 1096 pmap_copy_pages(bp->bio_ma, ma_offs, &m, 1097 offs, len); 1098 } else if ((bp->bio_flags & BIO_VLIST) != 0) { 1099 physcopyin_vlist(vlist, ma_offs, 1100 VM_PAGE_TO_PHYS(m) + offs, len); 1101 } else { 1102 physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); 1103 } 1104 1105 vm_page_valid(m); 1106 vm_page_set_dirty(m); 1107 } else if (bp->bio_cmd == BIO_DELETE) { 1108 if (len == PAGE_SIZE || vm_page_all_valid(m)) 1109 rv = VM_PAGER_OK; 1110 else 1111 rv = vm_pager_get_pages(sc->object, &m, 1, 1112 NULL, NULL); 1113 VM_OBJECT_WLOCK(sc->object); 1114 if (rv == VM_PAGER_ERROR) { 1115 vm_page_free(m); 1116 VM_OBJECT_WUNLOCK(sc->object); 1117 break; 1118 } else if (rv == VM_PAGER_FAIL) { 1119 vm_page_free(m); 1120 m = NULL; 1121 } else { 1122 /* Page is valid. */ 1123 if (len != PAGE_SIZE) { 1124 pmap_zero_page_area(m, offs, len); 1125 vm_page_set_dirty(m); 1126 } else { 1127 vm_pager_page_unswapped(m); 1128 vm_page_free(m); 1129 m = NULL; 1130 } 1131 } 1132 VM_OBJECT_WUNLOCK(sc->object); 1133 } 1134 if (m != NULL) { 1135 /* 1136 * The page may be deactivated prior to setting 1137 * PGA_REFERENCED, but in this case it will be 1138 * reactivated by the page daemon. 1139 */ 1140 if (vm_page_active(m)) 1141 vm_page_reference(m); 1142 else 1143 vm_page_activate(m); 1144 vm_page_xunbusy(m); 1145 } 1146 1147 /* Actions on further pages start at offset 0 */ 1148 p += PAGE_SIZE - offs; 1149 offs = 0; 1150 ma_offs += len; 1151 } 1152 vm_object_pip_wakeup(sc->object); 1153 return (rv != VM_PAGER_ERROR ? 0 : ENOSPC); 1154 } 1155 1156 static int 1157 mdstart_null(struct md_s *sc, struct bio *bp) 1158 { 1159 1160 switch (bp->bio_cmd) { 1161 case BIO_READ: 1162 bzero(bp->bio_data, bp->bio_length); 1163 cpu_flush_dcache(bp->bio_data, bp->bio_length); 1164 break; 1165 case BIO_WRITE: 1166 break; 1167 } 1168 bp->bio_resid = 0; 1169 return (0); 1170 } 1171 1172 static void 1173 md_handleattr(struct md_s *sc, struct bio *bp) 1174 { 1175 if (sc->fwsectors && sc->fwheads && 1176 (g_handleattr_int(bp, "GEOM::fwsectors", sc->fwsectors) != 0 || 1177 g_handleattr_int(bp, "GEOM::fwheads", sc->fwheads) != 0)) 1178 return; 1179 if (g_handleattr_int(bp, "GEOM::candelete", 1) != 0) 1180 return; 1181 if (sc->ident[0] != '\0' && 1182 g_handleattr_str(bp, "GEOM::ident", sc->ident) != 0) 1183 return; 1184 if (g_handleattr_int(bp, "MNT::verified", (sc->flags & MD_VERIFY) != 0)) 1185 return; 1186 g_io_deliver(bp, EOPNOTSUPP); 1187 } 1188 1189 static void 1190 md_kthread(void *arg) 1191 { 1192 struct md_s *sc; 1193 struct bio *bp; 1194 int error; 1195 1196 sc = arg; 1197 thread_lock(curthread); 1198 sched_prio(curthread, PRIBIO); 1199 thread_unlock(curthread); 1200 if (sc->type == MD_VNODE) 1201 curthread->td_pflags |= TDP_NORUNNINGBUF; 1202 1203 for (;;) { 1204 mtx_lock(&sc->queue_mtx); 1205 if (sc->flags & MD_SHUTDOWN) { 1206 sc->flags |= MD_EXITING; 1207 mtx_unlock(&sc->queue_mtx); 1208 kproc_exit(0); 1209 } 1210 bp = bioq_takefirst(&sc->bio_queue); 1211 if (!bp) { 1212 msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, "mdwait", 0); 1213 continue; 1214 } 1215 mtx_unlock(&sc->queue_mtx); 1216 if (bp->bio_cmd == BIO_GETATTR) { 1217 md_handleattr(sc, bp); 1218 } else { 1219 error = sc->start(sc, bp); 1220 if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { 1221 /* 1222 * Devstat uses (bio_bcount, bio_resid) for 1223 * determining the length of the completed part 1224 * of the i/o. g_io_deliver() will translate 1225 * from bio_completed to that, but it also 1226 * destroys the bio so we must do our own 1227 * translation. 1228 */ 1229 bp->bio_bcount = bp->bio_length; 1230 devstat_end_transaction_bio(sc->devstat, bp); 1231 } 1232 bp->bio_completed = bp->bio_length - bp->bio_resid; 1233 g_io_deliver(bp, error); 1234 } 1235 } 1236 } 1237 1238 static struct md_s * 1239 mdfind(int unit) 1240 { 1241 struct md_s *sc; 1242 1243 LIST_FOREACH(sc, &md_softc_list, list) { 1244 if (sc->unit == unit) 1245 break; 1246 } 1247 return (sc); 1248 } 1249 1250 static struct md_s * 1251 mdnew(int unit, int *errp, enum md_types type) 1252 { 1253 struct md_s *sc; 1254 int error; 1255 1256 *errp = 0; 1257 if (unit == -1) 1258 unit = alloc_unr(md_uh); 1259 else 1260 unit = alloc_unr_specific(md_uh, unit); 1261 1262 if (unit == -1) { 1263 *errp = EBUSY; 1264 return (NULL); 1265 } 1266 1267 sc = (struct md_s *)malloc(sizeof *sc, M_MD, M_WAITOK | M_ZERO); 1268 sc->type = type; 1269 bioq_init(&sc->bio_queue); 1270 mtx_init(&sc->queue_mtx, "md bio queue", NULL, MTX_DEF); 1271 sc->unit = unit; 1272 sprintf(sc->name, "md%d", unit); 1273 LIST_INSERT_HEAD(&md_softc_list, sc, list); 1274 error = kproc_create(md_kthread, sc, &sc->procp, 0, 0,"%s", sc->name); 1275 if (error == 0) 1276 return (sc); 1277 LIST_REMOVE(sc, list); 1278 mtx_destroy(&sc->queue_mtx); 1279 free_unr(md_uh, sc->unit); 1280 free(sc, M_MD); 1281 *errp = error; 1282 return (NULL); 1283 } 1284 1285 static void 1286 mdinit(struct md_s *sc) 1287 { 1288 struct g_geom *gp; 1289 struct g_provider *pp; 1290 1291 g_topology_lock(); 1292 gp = g_new_geomf(&g_md_class, "md%d", sc->unit); 1293 gp->softc = sc; 1294 pp = g_new_providerf(gp, "md%d", sc->unit); 1295 devstat_remove_entry(pp->stat); 1296 pp->stat = NULL; 1297 pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE; 1298 pp->mediasize = sc->mediasize; 1299 pp->sectorsize = sc->sectorsize; 1300 switch (sc->type) { 1301 case MD_MALLOC: 1302 case MD_VNODE: 1303 case MD_SWAP: 1304 pp->flags |= G_PF_ACCEPT_UNMAPPED; 1305 break; 1306 case MD_PRELOAD: 1307 case MD_NULL: 1308 break; 1309 } 1310 sc->gp = gp; 1311 sc->pp = pp; 1312 sc->devstat = devstat_new_entry("md", sc->unit, sc->sectorsize, 1313 DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, DEVSTAT_PRIORITY_MAX); 1314 sc->devstat->id = pp; 1315 g_error_provider(pp, 0); 1316 g_topology_unlock(); 1317 } 1318 1319 static int 1320 mdcreate_malloc(struct md_s *sc, struct md_req *mdr) 1321 { 1322 uintptr_t sp; 1323 int error; 1324 off_t u; 1325 1326 error = 0; 1327 if (mdr->md_options & ~(MD_AUTOUNIT | MD_COMPRESS | MD_RESERVE)) 1328 return (EINVAL); 1329 if (mdr->md_sectorsize != 0 && !powerof2(mdr->md_sectorsize)) 1330 return (EINVAL); 1331 /* Compression doesn't make sense if we have reserved space */ 1332 if (mdr->md_options & MD_RESERVE) 1333 mdr->md_options &= ~MD_COMPRESS; 1334 if (mdr->md_fwsectors != 0) 1335 sc->fwsectors = mdr->md_fwsectors; 1336 if (mdr->md_fwheads != 0) 1337 sc->fwheads = mdr->md_fwheads; 1338 sc->flags = mdr->md_options & (MD_COMPRESS | MD_FORCE); 1339 sc->indir = dimension(sc->mediasize / sc->sectorsize); 1340 sc->uma = uma_zcreate(sc->name, sc->sectorsize, NULL, NULL, NULL, NULL, 1341 0x1ff, 0); 1342 if (mdr->md_options & MD_RESERVE) { 1343 off_t nsectors; 1344 1345 nsectors = sc->mediasize / sc->sectorsize; 1346 for (u = 0; u < nsectors; u++) { 1347 sp = (uintptr_t)uma_zalloc(sc->uma, (md_malloc_wait ? 1348 M_WAITOK : M_NOWAIT) | M_ZERO); 1349 if (sp != 0) 1350 error = s_write(sc->indir, u, sp); 1351 else 1352 error = ENOMEM; 1353 if (error != 0) 1354 break; 1355 } 1356 } 1357 return (error); 1358 } 1359 1360 static int 1361 mdsetcred(struct md_s *sc, struct ucred *cred) 1362 { 1363 char *tmpbuf; 1364 int error = 0; 1365 1366 /* 1367 * Set credits in our softc 1368 */ 1369 1370 if (sc->cred) 1371 crfree(sc->cred); 1372 sc->cred = crhold(cred); 1373 1374 /* 1375 * Horrible kludge to establish credentials for NFS XXX. 1376 */ 1377 1378 if (sc->vnode) { 1379 struct uio auio; 1380 struct iovec aiov; 1381 1382 tmpbuf = malloc(sc->sectorsize, M_TEMP, M_WAITOK); 1383 bzero(&auio, sizeof(auio)); 1384 1385 aiov.iov_base = tmpbuf; 1386 aiov.iov_len = sc->sectorsize; 1387 auio.uio_iov = &aiov; 1388 auio.uio_iovcnt = 1; 1389 auio.uio_offset = 0; 1390 auio.uio_rw = UIO_READ; 1391 auio.uio_segflg = UIO_SYSSPACE; 1392 auio.uio_resid = aiov.iov_len; 1393 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 1394 error = VOP_READ(sc->vnode, &auio, 0, sc->cred); 1395 VOP_UNLOCK(sc->vnode); 1396 free(tmpbuf, M_TEMP); 1397 } 1398 return (error); 1399 } 1400 1401 static int 1402 mdcreate_vnode(struct md_s *sc, struct md_req *mdr, struct thread *td) 1403 { 1404 struct vattr vattr; 1405 struct nameidata nd; 1406 char *fname; 1407 int error, flags; 1408 1409 fname = mdr->md_file; 1410 if (mdr->md_file_seg == UIO_USERSPACE) { 1411 error = copyinstr(fname, sc->file, sizeof(sc->file), NULL); 1412 if (error != 0) 1413 return (error); 1414 } else if (mdr->md_file_seg == UIO_SYSSPACE) 1415 strlcpy(sc->file, fname, sizeof(sc->file)); 1416 else 1417 return (EDOOFUS); 1418 1419 /* 1420 * If the user specified that this is a read only device, don't 1421 * set the FWRITE mask before trying to open the backing store. 1422 */ 1423 flags = FREAD | ((mdr->md_options & MD_READONLY) ? 0 : FWRITE) \ 1424 | ((mdr->md_options & MD_VERIFY) ? O_VERIFY : 0); 1425 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->file, td); 1426 error = vn_open(&nd, &flags, 0, NULL); 1427 if (error != 0) 1428 return (error); 1429 NDFREE(&nd, NDF_ONLY_PNBUF); 1430 if (nd.ni_vp->v_type != VREG) { 1431 error = EINVAL; 1432 goto bad; 1433 } 1434 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 1435 if (error != 0) 1436 goto bad; 1437 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 1438 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 1439 if (VN_IS_DOOMED(nd.ni_vp)) { 1440 /* Forced unmount. */ 1441 error = EBADF; 1442 goto bad; 1443 } 1444 } 1445 nd.ni_vp->v_vflag |= VV_MD; 1446 VOP_UNLOCK(nd.ni_vp); 1447 1448 if (mdr->md_fwsectors != 0) 1449 sc->fwsectors = mdr->md_fwsectors; 1450 if (mdr->md_fwheads != 0) 1451 sc->fwheads = mdr->md_fwheads; 1452 snprintf(sc->ident, sizeof(sc->ident), "MD-DEV%ju-INO%ju", 1453 (uintmax_t)vattr.va_fsid, (uintmax_t)vattr.va_fileid); 1454 sc->flags = mdr->md_options & (MD_ASYNC | MD_CACHE | MD_FORCE | 1455 MD_VERIFY); 1456 if (!(flags & FWRITE)) 1457 sc->flags |= MD_READONLY; 1458 sc->vnode = nd.ni_vp; 1459 1460 error = mdsetcred(sc, td->td_ucred); 1461 if (error != 0) { 1462 sc->vnode = NULL; 1463 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 1464 nd.ni_vp->v_vflag &= ~VV_MD; 1465 goto bad; 1466 } 1467 return (0); 1468 bad: 1469 VOP_UNLOCK(nd.ni_vp); 1470 (void)vn_close(nd.ni_vp, flags, td->td_ucred, td); 1471 return (error); 1472 } 1473 1474 static void 1475 g_md_providergone(struct g_provider *pp) 1476 { 1477 struct md_s *sc = pp->geom->softc; 1478 1479 mtx_lock(&sc->queue_mtx); 1480 sc->flags |= MD_PROVIDERGONE; 1481 wakeup(&sc->flags); 1482 mtx_unlock(&sc->queue_mtx); 1483 } 1484 1485 static int 1486 mddestroy(struct md_s *sc, struct thread *td) 1487 { 1488 1489 if (sc->gp) { 1490 g_topology_lock(); 1491 g_wither_geom(sc->gp, ENXIO); 1492 g_topology_unlock(); 1493 1494 mtx_lock(&sc->queue_mtx); 1495 while (!(sc->flags & MD_PROVIDERGONE)) 1496 msleep(&sc->flags, &sc->queue_mtx, PRIBIO, "mddestroy", 0); 1497 mtx_unlock(&sc->queue_mtx); 1498 } 1499 if (sc->devstat) { 1500 devstat_remove_entry(sc->devstat); 1501 sc->devstat = NULL; 1502 } 1503 mtx_lock(&sc->queue_mtx); 1504 sc->flags |= MD_SHUTDOWN; 1505 wakeup(sc); 1506 while (!(sc->flags & MD_EXITING)) 1507 msleep(sc->procp, &sc->queue_mtx, PRIBIO, "mddestroy", hz / 10); 1508 mtx_unlock(&sc->queue_mtx); 1509 mtx_destroy(&sc->queue_mtx); 1510 if (sc->vnode != NULL) { 1511 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 1512 sc->vnode->v_vflag &= ~VV_MD; 1513 VOP_UNLOCK(sc->vnode); 1514 (void)vn_close(sc->vnode, sc->flags & MD_READONLY ? 1515 FREAD : (FREAD|FWRITE), sc->cred, td); 1516 } 1517 if (sc->cred != NULL) 1518 crfree(sc->cred); 1519 if (sc->object != NULL) 1520 vm_object_deallocate(sc->object); 1521 if (sc->indir) 1522 destroy_indir(sc, sc->indir); 1523 if (sc->uma) 1524 uma_zdestroy(sc->uma); 1525 1526 LIST_REMOVE(sc, list); 1527 free_unr(md_uh, sc->unit); 1528 free(sc, M_MD); 1529 return (0); 1530 } 1531 1532 static int 1533 mdresize(struct md_s *sc, struct md_req *mdr) 1534 { 1535 int error, res; 1536 vm_pindex_t oldpages, newpages; 1537 1538 switch (sc->type) { 1539 case MD_VNODE: 1540 case MD_NULL: 1541 break; 1542 case MD_SWAP: 1543 if (mdr->md_mediasize <= 0 || 1544 (mdr->md_mediasize % PAGE_SIZE) != 0) 1545 return (EDOM); 1546 oldpages = OFF_TO_IDX(sc->mediasize); 1547 newpages = OFF_TO_IDX(mdr->md_mediasize); 1548 if (newpages < oldpages) { 1549 VM_OBJECT_WLOCK(sc->object); 1550 vm_object_page_remove(sc->object, newpages, 0, 0); 1551 swap_release_by_cred(IDX_TO_OFF(oldpages - 1552 newpages), sc->cred); 1553 sc->object->charge = IDX_TO_OFF(newpages); 1554 sc->object->size = newpages; 1555 VM_OBJECT_WUNLOCK(sc->object); 1556 } else if (newpages > oldpages) { 1557 res = swap_reserve_by_cred(IDX_TO_OFF(newpages - 1558 oldpages), sc->cred); 1559 if (!res) 1560 return (ENOMEM); 1561 if ((mdr->md_options & MD_RESERVE) || 1562 (sc->flags & MD_RESERVE)) { 1563 error = swap_pager_reserve(sc->object, 1564 oldpages, newpages - oldpages); 1565 if (error < 0) { 1566 swap_release_by_cred( 1567 IDX_TO_OFF(newpages - oldpages), 1568 sc->cred); 1569 return (EDOM); 1570 } 1571 } 1572 VM_OBJECT_WLOCK(sc->object); 1573 sc->object->charge = IDX_TO_OFF(newpages); 1574 sc->object->size = newpages; 1575 VM_OBJECT_WUNLOCK(sc->object); 1576 } 1577 break; 1578 default: 1579 return (EOPNOTSUPP); 1580 } 1581 1582 sc->mediasize = mdr->md_mediasize; 1583 1584 g_topology_lock(); 1585 g_resize_provider(sc->pp, sc->mediasize); 1586 g_topology_unlock(); 1587 return (0); 1588 } 1589 1590 static int 1591 mdcreate_swap(struct md_s *sc, struct md_req *mdr, struct thread *td) 1592 { 1593 vm_ooffset_t npage; 1594 int error; 1595 1596 /* 1597 * Range check. Disallow negative sizes and sizes not being 1598 * multiple of page size. 1599 */ 1600 if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) 1601 return (EDOM); 1602 1603 /* 1604 * Allocate an OBJT_SWAP object. 1605 * 1606 * Note the truncation. 1607 */ 1608 1609 if ((mdr->md_options & MD_VERIFY) != 0) 1610 return (EINVAL); 1611 npage = mdr->md_mediasize / PAGE_SIZE; 1612 if (mdr->md_fwsectors != 0) 1613 sc->fwsectors = mdr->md_fwsectors; 1614 if (mdr->md_fwheads != 0) 1615 sc->fwheads = mdr->md_fwheads; 1616 sc->object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * npage, 1617 VM_PROT_DEFAULT, 0, td->td_ucred); 1618 if (sc->object == NULL) 1619 return (ENOMEM); 1620 sc->flags = mdr->md_options & (MD_FORCE | MD_RESERVE); 1621 if (mdr->md_options & MD_RESERVE) { 1622 if (swap_pager_reserve(sc->object, 0, npage) < 0) { 1623 error = EDOM; 1624 goto finish; 1625 } 1626 } 1627 error = mdsetcred(sc, td->td_ucred); 1628 finish: 1629 if (error != 0) { 1630 vm_object_deallocate(sc->object); 1631 sc->object = NULL; 1632 } 1633 return (error); 1634 } 1635 1636 static int 1637 mdcreate_null(struct md_s *sc, struct md_req *mdr, struct thread *td) 1638 { 1639 1640 /* 1641 * Range check. Disallow negative sizes and sizes not being 1642 * multiple of page size. 1643 */ 1644 if (sc->mediasize <= 0 || (sc->mediasize % PAGE_SIZE) != 0) 1645 return (EDOM); 1646 1647 return (0); 1648 } 1649 1650 static int 1651 kern_mdattach_locked(struct thread *td, struct md_req *mdr) 1652 { 1653 struct md_s *sc; 1654 unsigned sectsize; 1655 int error, i; 1656 1657 sx_assert(&md_sx, SA_XLOCKED); 1658 1659 switch (mdr->md_type) { 1660 case MD_MALLOC: 1661 case MD_PRELOAD: 1662 case MD_VNODE: 1663 case MD_SWAP: 1664 case MD_NULL: 1665 break; 1666 default: 1667 return (EINVAL); 1668 } 1669 if (mdr->md_sectorsize == 0) 1670 sectsize = DEV_BSIZE; 1671 else 1672 sectsize = mdr->md_sectorsize; 1673 if (sectsize > maxphys || mdr->md_mediasize < sectsize) 1674 return (EINVAL); 1675 if (mdr->md_options & MD_AUTOUNIT) 1676 sc = mdnew(-1, &error, mdr->md_type); 1677 else { 1678 if (mdr->md_unit > INT_MAX) 1679 return (EINVAL); 1680 sc = mdnew(mdr->md_unit, &error, mdr->md_type); 1681 } 1682 if (sc == NULL) 1683 return (error); 1684 if (mdr->md_label != NULL) 1685 error = copyinstr(mdr->md_label, sc->label, 1686 sizeof(sc->label), NULL); 1687 if (error != 0) 1688 goto err_after_new; 1689 if (mdr->md_options & MD_AUTOUNIT) 1690 mdr->md_unit = sc->unit; 1691 sc->mediasize = mdr->md_mediasize; 1692 sc->sectorsize = sectsize; 1693 error = EDOOFUS; 1694 switch (sc->type) { 1695 case MD_MALLOC: 1696 sc->start = mdstart_malloc; 1697 error = mdcreate_malloc(sc, mdr); 1698 break; 1699 case MD_PRELOAD: 1700 /* 1701 * We disallow attaching preloaded memory disks via 1702 * ioctl. Preloaded memory disks are automatically 1703 * attached in g_md_init(). 1704 */ 1705 error = EOPNOTSUPP; 1706 break; 1707 case MD_VNODE: 1708 sc->start = mdstart_vnode; 1709 error = mdcreate_vnode(sc, mdr, td); 1710 break; 1711 case MD_SWAP: 1712 sc->start = mdstart_swap; 1713 error = mdcreate_swap(sc, mdr, td); 1714 break; 1715 case MD_NULL: 1716 sc->start = mdstart_null; 1717 error = mdcreate_null(sc, mdr, td); 1718 break; 1719 } 1720 err_after_new: 1721 if (error != 0) { 1722 mddestroy(sc, td); 1723 return (error); 1724 } 1725 1726 /* Prune off any residual fractional sector */ 1727 i = sc->mediasize % sc->sectorsize; 1728 sc->mediasize -= i; 1729 1730 mdinit(sc); 1731 return (0); 1732 } 1733 1734 static int 1735 kern_mdattach(struct thread *td, struct md_req *mdr) 1736 { 1737 int error; 1738 1739 sx_xlock(&md_sx); 1740 error = kern_mdattach_locked(td, mdr); 1741 sx_xunlock(&md_sx); 1742 return (error); 1743 } 1744 1745 static int 1746 kern_mddetach_locked(struct thread *td, struct md_req *mdr) 1747 { 1748 struct md_s *sc; 1749 1750 sx_assert(&md_sx, SA_XLOCKED); 1751 1752 if (mdr->md_mediasize != 0 || 1753 (mdr->md_options & ~MD_FORCE) != 0) 1754 return (EINVAL); 1755 1756 sc = mdfind(mdr->md_unit); 1757 if (sc == NULL) 1758 return (ENOENT); 1759 if (sc->opencount != 0 && !(sc->flags & MD_FORCE) && 1760 !(mdr->md_options & MD_FORCE)) 1761 return (EBUSY); 1762 return (mddestroy(sc, td)); 1763 } 1764 1765 static int 1766 kern_mddetach(struct thread *td, struct md_req *mdr) 1767 { 1768 int error; 1769 1770 sx_xlock(&md_sx); 1771 error = kern_mddetach_locked(td, mdr); 1772 sx_xunlock(&md_sx); 1773 return (error); 1774 } 1775 1776 static int 1777 kern_mdresize_locked(struct md_req *mdr) 1778 { 1779 struct md_s *sc; 1780 1781 sx_assert(&md_sx, SA_XLOCKED); 1782 1783 if ((mdr->md_options & ~(MD_FORCE | MD_RESERVE)) != 0) 1784 return (EINVAL); 1785 1786 sc = mdfind(mdr->md_unit); 1787 if (sc == NULL) 1788 return (ENOENT); 1789 if (mdr->md_mediasize < sc->sectorsize) 1790 return (EINVAL); 1791 mdr->md_mediasize -= mdr->md_mediasize % sc->sectorsize; 1792 if (mdr->md_mediasize < sc->mediasize && 1793 !(sc->flags & MD_FORCE) && 1794 !(mdr->md_options & MD_FORCE)) 1795 return (EBUSY); 1796 return (mdresize(sc, mdr)); 1797 } 1798 1799 static int 1800 kern_mdresize(struct md_req *mdr) 1801 { 1802 int error; 1803 1804 sx_xlock(&md_sx); 1805 error = kern_mdresize_locked(mdr); 1806 sx_xunlock(&md_sx); 1807 return (error); 1808 } 1809 1810 static int 1811 kern_mdquery_locked(struct md_req *mdr) 1812 { 1813 struct md_s *sc; 1814 int error; 1815 1816 sx_assert(&md_sx, SA_XLOCKED); 1817 1818 sc = mdfind(mdr->md_unit); 1819 if (sc == NULL) 1820 return (ENOENT); 1821 mdr->md_type = sc->type; 1822 mdr->md_options = sc->flags; 1823 mdr->md_mediasize = sc->mediasize; 1824 mdr->md_sectorsize = sc->sectorsize; 1825 error = 0; 1826 if (mdr->md_label != NULL) { 1827 error = copyout(sc->label, mdr->md_label, 1828 strlen(sc->label) + 1); 1829 if (error != 0) 1830 return (error); 1831 } 1832 if (sc->type == MD_VNODE || 1833 (sc->type == MD_PRELOAD && mdr->md_file != NULL)) 1834 error = copyout(sc->file, mdr->md_file, 1835 strlen(sc->file) + 1); 1836 return (error); 1837 } 1838 1839 static int 1840 kern_mdquery(struct md_req *mdr) 1841 { 1842 int error; 1843 1844 sx_xlock(&md_sx); 1845 error = kern_mdquery_locked(mdr); 1846 sx_xunlock(&md_sx); 1847 return (error); 1848 } 1849 1850 /* Copy members that are not userspace pointers. */ 1851 #define MD_IOCTL2REQ(mdio, mdr) do { \ 1852 (mdr)->md_unit = (mdio)->md_unit; \ 1853 (mdr)->md_type = (mdio)->md_type; \ 1854 (mdr)->md_mediasize = (mdio)->md_mediasize; \ 1855 (mdr)->md_sectorsize = (mdio)->md_sectorsize; \ 1856 (mdr)->md_options = (mdio)->md_options; \ 1857 (mdr)->md_fwheads = (mdio)->md_fwheads; \ 1858 (mdr)->md_fwsectors = (mdio)->md_fwsectors; \ 1859 (mdr)->md_units = &(mdio)->md_pad[0]; \ 1860 (mdr)->md_units_nitems = nitems((mdio)->md_pad); \ 1861 } while(0) 1862 1863 /* Copy members that might have been updated */ 1864 #define MD_REQ2IOCTL(mdr, mdio) do { \ 1865 (mdio)->md_unit = (mdr)->md_unit; \ 1866 (mdio)->md_type = (mdr)->md_type; \ 1867 (mdio)->md_mediasize = (mdr)->md_mediasize; \ 1868 (mdio)->md_sectorsize = (mdr)->md_sectorsize; \ 1869 (mdio)->md_options = (mdr)->md_options; \ 1870 (mdio)->md_fwheads = (mdr)->md_fwheads; \ 1871 (mdio)->md_fwsectors = (mdr)->md_fwsectors; \ 1872 } while(0) 1873 1874 static int 1875 mdctlioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, 1876 struct thread *td) 1877 { 1878 struct md_req mdr; 1879 int error; 1880 1881 if (md_debug) 1882 printf("mdctlioctl(%s %lx %p %x %p)\n", 1883 devtoname(dev), cmd, addr, flags, td); 1884 1885 bzero(&mdr, sizeof(mdr)); 1886 switch (cmd) { 1887 case MDIOCATTACH: 1888 case MDIOCDETACH: 1889 case MDIOCRESIZE: 1890 case MDIOCQUERY: { 1891 struct md_ioctl *mdio = (struct md_ioctl *)addr; 1892 if (mdio->md_version != MDIOVERSION) 1893 return (EINVAL); 1894 MD_IOCTL2REQ(mdio, &mdr); 1895 mdr.md_file = mdio->md_file; 1896 mdr.md_file_seg = UIO_USERSPACE; 1897 /* If the file is adjacent to the md_ioctl it's in kernel. */ 1898 if ((void *)mdio->md_file == (void *)(mdio + 1)) 1899 mdr.md_file_seg = UIO_SYSSPACE; 1900 mdr.md_label = mdio->md_label; 1901 break; 1902 } 1903 #ifdef COMPAT_FREEBSD32 1904 case MDIOCATTACH_32: 1905 case MDIOCDETACH_32: 1906 case MDIOCRESIZE_32: 1907 case MDIOCQUERY_32: { 1908 struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; 1909 if (mdio->md_version != MDIOVERSION) 1910 return (EINVAL); 1911 MD_IOCTL2REQ(mdio, &mdr); 1912 mdr.md_file = (void *)(uintptr_t)mdio->md_file; 1913 mdr.md_file_seg = UIO_USERSPACE; 1914 mdr.md_label = (void *)(uintptr_t)mdio->md_label; 1915 break; 1916 } 1917 #endif 1918 default: 1919 /* Fall through to handler switch. */ 1920 break; 1921 } 1922 1923 error = 0; 1924 switch (cmd) { 1925 case MDIOCATTACH: 1926 #ifdef COMPAT_FREEBSD32 1927 case MDIOCATTACH_32: 1928 #endif 1929 error = kern_mdattach(td, &mdr); 1930 break; 1931 case MDIOCDETACH: 1932 #ifdef COMPAT_FREEBSD32 1933 case MDIOCDETACH_32: 1934 #endif 1935 error = kern_mddetach(td, &mdr); 1936 break; 1937 case MDIOCRESIZE: 1938 #ifdef COMPAT_FREEBSD32 1939 case MDIOCRESIZE_32: 1940 #endif 1941 error = kern_mdresize(&mdr); 1942 break; 1943 case MDIOCQUERY: 1944 #ifdef COMPAT_FREEBSD32 1945 case MDIOCQUERY_32: 1946 #endif 1947 error = kern_mdquery(&mdr); 1948 break; 1949 default: 1950 error = ENOIOCTL; 1951 } 1952 1953 switch (cmd) { 1954 case MDIOCATTACH: 1955 case MDIOCQUERY: { 1956 struct md_ioctl *mdio = (struct md_ioctl *)addr; 1957 MD_REQ2IOCTL(&mdr, mdio); 1958 break; 1959 } 1960 #ifdef COMPAT_FREEBSD32 1961 case MDIOCATTACH_32: 1962 case MDIOCQUERY_32: { 1963 struct md_ioctl32 *mdio = (struct md_ioctl32 *)addr; 1964 MD_REQ2IOCTL(&mdr, mdio); 1965 break; 1966 } 1967 #endif 1968 default: 1969 /* Other commands to not alter mdr. */ 1970 break; 1971 } 1972 1973 return (error); 1974 } 1975 1976 static void 1977 md_preloaded(u_char *image, size_t length, const char *name) 1978 { 1979 struct md_s *sc; 1980 int error; 1981 1982 sc = mdnew(-1, &error, MD_PRELOAD); 1983 if (sc == NULL) 1984 return; 1985 sc->mediasize = length; 1986 sc->sectorsize = DEV_BSIZE; 1987 sc->pl_ptr = image; 1988 sc->pl_len = length; 1989 sc->start = mdstart_preload; 1990 if (name != NULL) 1991 strlcpy(sc->file, name, sizeof(sc->file)); 1992 #ifdef MD_ROOT 1993 if (sc->unit == 0) { 1994 #ifndef ROOTDEVNAME 1995 rootdevnames[0] = MD_ROOT_FSTYPE ":/dev/md0"; 1996 #endif 1997 #ifdef MD_ROOT_READONLY 1998 sc->flags |= MD_READONLY; 1999 #endif 2000 } 2001 #endif 2002 mdinit(sc); 2003 if (name != NULL) { 2004 printf("%s%d: Preloaded image <%s> %zd bytes at %p\n", 2005 MD_NAME, sc->unit, name, length, image); 2006 } else { 2007 printf("%s%d: Embedded image %zd bytes at %p\n", 2008 MD_NAME, sc->unit, length, image); 2009 } 2010 } 2011 2012 static void 2013 g_md_init(struct g_class *mp __unused) 2014 { 2015 caddr_t mod; 2016 u_char *ptr, *name, *type; 2017 unsigned len; 2018 int i; 2019 2020 /* figure out log2(NINDIR) */ 2021 for (i = NINDIR, nshift = -1; i; nshift++) 2022 i >>= 1; 2023 2024 mod = NULL; 2025 sx_init(&md_sx, "MD config lock"); 2026 g_topology_unlock(); 2027 md_uh = new_unrhdr(0, INT_MAX, NULL); 2028 #ifdef MD_ROOT 2029 if (mfs_root_size != 0) { 2030 sx_xlock(&md_sx); 2031 #ifdef MD_ROOT_MEM 2032 md_preloaded(mfs_root, mfs_root_size, NULL); 2033 #else 2034 md_preloaded(__DEVOLATILE(u_char *, &mfs_root), mfs_root_size, 2035 NULL); 2036 #endif 2037 sx_xunlock(&md_sx); 2038 } 2039 #endif 2040 /* XXX: are preload_* static or do they need Giant ? */ 2041 while ((mod = preload_search_next_name(mod)) != NULL) { 2042 name = (char *)preload_search_info(mod, MODINFO_NAME); 2043 if (name == NULL) 2044 continue; 2045 type = (char *)preload_search_info(mod, MODINFO_TYPE); 2046 if (type == NULL) 2047 continue; 2048 if (strcmp(type, "md_image") && strcmp(type, "mfs_root")) 2049 continue; 2050 ptr = preload_fetch_addr(mod); 2051 len = preload_fetch_size(mod); 2052 if (ptr != NULL && len != 0) { 2053 sx_xlock(&md_sx); 2054 md_preloaded(ptr, len, name); 2055 sx_xunlock(&md_sx); 2056 } 2057 } 2058 md_pbuf_zone = pbuf_zsecond_create("mdpbuf", nswbuf / 10); 2059 status_dev = make_dev(&mdctl_cdevsw, INT_MAX, UID_ROOT, GID_WHEEL, 2060 0600, MDCTL_NAME); 2061 g_topology_lock(); 2062 } 2063 2064 static void 2065 g_md_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, 2066 struct g_consumer *cp __unused, struct g_provider *pp) 2067 { 2068 struct md_s *mp; 2069 char *type; 2070 2071 mp = gp->softc; 2072 if (mp == NULL) 2073 return; 2074 2075 switch (mp->type) { 2076 case MD_MALLOC: 2077 type = "malloc"; 2078 break; 2079 case MD_PRELOAD: 2080 type = "preload"; 2081 break; 2082 case MD_VNODE: 2083 type = "vnode"; 2084 break; 2085 case MD_SWAP: 2086 type = "swap"; 2087 break; 2088 case MD_NULL: 2089 type = "null"; 2090 break; 2091 default: 2092 type = "unknown"; 2093 break; 2094 } 2095 2096 if (pp != NULL) { 2097 if (indent == NULL) { 2098 sbuf_printf(sb, " u %d", mp->unit); 2099 sbuf_printf(sb, " s %ju", (uintmax_t) mp->sectorsize); 2100 sbuf_printf(sb, " f %ju", (uintmax_t) mp->fwheads); 2101 sbuf_printf(sb, " fs %ju", (uintmax_t) mp->fwsectors); 2102 sbuf_printf(sb, " l %ju", (uintmax_t) mp->mediasize); 2103 sbuf_printf(sb, " t %s", type); 2104 if ((mp->type == MD_VNODE && mp->vnode != NULL) || 2105 (mp->type == MD_PRELOAD && mp->file[0] != '\0')) 2106 sbuf_printf(sb, " file %s", mp->file); 2107 sbuf_printf(sb, " label %s", mp->label); 2108 } else { 2109 sbuf_printf(sb, "%s<unit>%d</unit>\n", indent, 2110 mp->unit); 2111 sbuf_printf(sb, "%s<sectorsize>%ju</sectorsize>\n", 2112 indent, (uintmax_t) mp->sectorsize); 2113 sbuf_printf(sb, "%s<fwheads>%ju</fwheads>\n", 2114 indent, (uintmax_t) mp->fwheads); 2115 sbuf_printf(sb, "%s<fwsectors>%ju</fwsectors>\n", 2116 indent, (uintmax_t) mp->fwsectors); 2117 if (mp->ident[0] != '\0') { 2118 sbuf_printf(sb, "%s<ident>", indent); 2119 g_conf_printf_escaped(sb, "%s", mp->ident); 2120 sbuf_printf(sb, "</ident>\n"); 2121 } 2122 sbuf_printf(sb, "%s<length>%ju</length>\n", 2123 indent, (uintmax_t) mp->mediasize); 2124 sbuf_printf(sb, "%s<compression>%s</compression>\n", indent, 2125 (mp->flags & MD_COMPRESS) == 0 ? "off": "on"); 2126 sbuf_printf(sb, "%s<access>%s</access>\n", indent, 2127 (mp->flags & MD_READONLY) == 0 ? "read-write": 2128 "read-only"); 2129 sbuf_printf(sb, "%s<type>%s</type>\n", indent, 2130 type); 2131 if ((mp->type == MD_VNODE && mp->vnode != NULL) || 2132 (mp->type == MD_PRELOAD && mp->file[0] != '\0')) { 2133 sbuf_printf(sb, "%s<file>", indent); 2134 g_conf_printf_escaped(sb, "%s", mp->file); 2135 sbuf_printf(sb, "</file>\n"); 2136 } 2137 if (mp->type == MD_VNODE) 2138 sbuf_printf(sb, "%s<cache>%s</cache>\n", indent, 2139 (mp->flags & MD_CACHE) == 0 ? "off": "on"); 2140 sbuf_printf(sb, "%s<label>", indent); 2141 g_conf_printf_escaped(sb, "%s", mp->label); 2142 sbuf_printf(sb, "</label>\n"); 2143 } 2144 } 2145 } 2146 2147 static void 2148 g_md_fini(struct g_class *mp __unused) 2149 { 2150 2151 sx_destroy(&md_sx); 2152 if (status_dev != NULL) 2153 destroy_dev(status_dev); 2154 uma_zdestroy(md_pbuf_zone); 2155 delete_unrhdr(md_uh); 2156 } 2157