1 /* 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 #include <sys/buf.h> 40 #include <sys/conf.h> 41 #include <sys/disklabel.h> 42 #include <sys/disklabel32.h> 43 #include <sys/disklabel64.h> 44 #include <sys/diskslice.h> 45 #include <sys/diskmbr.h> 46 #include <sys/disk.h> 47 #include <sys/malloc.h> 48 #include <sys/device.h> 49 #include <sys/devfs.h> 50 #include <sys/thread.h> 51 #include <sys/queue.h> 52 #include <sys/lock.h> 53 #include <sys/stat.h> 54 #include <sys/uuid.h> 55 #include <sys/dmsg.h> 56 57 #include <sys/buf2.h> 58 #include <sys/msgport2.h> 59 60 struct dios_open { 61 int openrd; 62 int openwr; 63 }; 64 65 struct dios_io { 66 int count; 67 int eof; 68 kdmsg_data_t data; 69 }; 70 71 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg"); 72 73 static int blk_active; 74 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0, 75 "Number of active iocom IOs"); 76 77 static int disk_iocom_reconnect(struct disk *dp, struct file *fp); 78 static int disk_rcvdmsg(kdmsg_msg_t *msg); 79 80 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg); 81 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg); 82 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg); 83 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg); 84 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg); 85 static void diskiodone(struct bio *bio); 86 87 void 88 disk_iocom_init(struct disk *dp) 89 { 90 kdmsg_iocom_init(&dp->d_iocom, dp, 91 KDMSG_IOCOMF_AUTOCONN | 92 KDMSG_IOCOMF_AUTORXSPAN | 93 KDMSG_IOCOMF_AUTOTXSPAN, 94 M_DMSG_DISK, disk_rcvdmsg); 95 } 96 97 void 98 disk_iocom_update(struct disk *dp) 99 { 100 } 101 102 void 103 disk_iocom_uninit(struct disk *dp) 104 { 105 kdmsg_iocom_uninit(&dp->d_iocom); 106 } 107 108 int 109 disk_iocom_ioctl(struct disk *dp, u_long cmd, void *data) 110 { 111 struct file *fp; 112 struct disk_ioc_recluster *recl; 113 int error; 114 115 switch(cmd) { 116 case DIOCRECLUSTER: 117 recl = data; 118 fp = holdfp(curthread, recl->fd, -1); 119 if (fp) { 120 error = disk_iocom_reconnect(dp, fp); 121 } else { 122 error = EINVAL; 123 } 124 break; 125 default: 126 error = EOPNOTSUPP; 127 break; 128 } 129 return error; 130 } 131 132 static 133 int 134 disk_iocom_reconnect(struct disk *dp, struct file *fp) 135 { 136 char devname[64]; 137 138 ksnprintf(devname, sizeof(devname), "%s%d", 139 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev)); 140 141 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname); 142 143 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1; 144 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK; 145 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK; 146 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1; 147 #if 0 148 if (dp->d_info.d_serialno) { 149 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 150 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 151 "%s/%s", hostname, dp->d_info.d_serialno); 152 } else { 153 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 154 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 155 "%s/%s", hostname, devname); 156 } 157 #endif 158 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 159 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 160 "%s/%s", hostname, devname); 161 162 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1; 163 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK; 164 dp->d_iocom.auto_lnk_span.media.block.bytes = 165 dp->d_info.d_media_size; 166 dp->d_iocom.auto_lnk_span.media.block.blksize = 167 dp->d_info.d_media_blksize; 168 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label, 169 sizeof(dp->d_iocom.auto_lnk_span.peer_label), 170 "%s", dp->d_iocom.auto_lnk_conn.peer_label); 171 if (dp->d_info.d_serialno) { 172 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 173 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 174 "%s", dp->d_info.d_serialno); 175 } else { 176 /* 177 * If no serial number is available generate a dummy serial 178 * number from the host and device name and pray. This will 179 * allow e.g. /dev/vn* to look meaningful on a remote machine. 180 */ 181 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 182 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 183 "%s.%s", hostname, devname); 184 } 185 186 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL); 187 188 return (0); 189 } 190 191 static int 192 disk_rcvdmsg(kdmsg_msg_t *msg) 193 { 194 struct disk *dp = msg->state->iocom->handle; 195 196 /* 197 * Handle debug messages (these might not be in transactions) 198 */ 199 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) { 200 case DMSG_DBG_SHELL: 201 /* 202 * Execute shell command (not supported atm) 203 */ 204 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 205 return(0); 206 case DMSG_DBG_SHELL | DMSGF_REPLY: 207 if (msg->aux_data) { 208 msg->aux_data[msg->aux_size - 1] = 0; 209 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data); 210 } 211 return(0); 212 } 213 214 /* 215 * All remaining messages must be in a transaction. 216 * 217 * NOTE! We currently don't care if the transaction is just 218 * the span transaction (for disk probes) or if it is the 219 * BLK_OPEN transaction. 220 * 221 * NOTE! We are switching on the first message's command. The 222 * actual message command within the transaction may be 223 * different (if streaming within a transaction). 224 */ 225 if (msg->state == &msg->state->iocom->state0) { 226 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 227 return(0); 228 } 229 230 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) { 231 case DMSG_BLK_OPEN: 232 disk_blk_open(dp, msg); 233 break; 234 case DMSG_BLK_READ: 235 /* 236 * not reached normally but leave in for completeness 237 */ 238 disk_blk_read(dp, msg); 239 break; 240 case DMSG_BLK_WRITE: 241 disk_blk_write(dp, msg); 242 break; 243 case DMSG_BLK_FLUSH: 244 disk_blk_flush(dp, msg); 245 break; 246 case DMSG_BLK_FREEBLKS: 247 disk_blk_freeblks(dp, msg); 248 break; 249 default: 250 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) { 251 if (msg->any.head.cmd & DMSGF_DELETE) 252 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 253 else 254 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP); 255 } 256 break; 257 } 258 return (0); 259 } 260 261 static 262 void 263 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg) 264 { 265 struct dios_open *openst; 266 int error = DMSG_ERR_NOSUPP; 267 int fflags; 268 269 openst = msg->state->any.any; 270 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) { 271 if (openst == NULL) { 272 openst = kmalloc(sizeof(*openst), M_DEVBUF, 273 M_WAITOK | M_ZERO); 274 msg->state->any.any = openst; 275 } 276 fflags = 0; 277 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 278 fflags = FREAD; 279 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 280 fflags |= FWRITE; 281 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, 282 proc0.p_ucred, NULL, NULL); 283 if (error) { 284 error = DMSG_ERR_IO; 285 } else { 286 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 287 ++openst->openrd; 288 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 289 ++openst->openwr; 290 } 291 } 292 #if 0 293 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE && 294 openst) { 295 fflags = 0; 296 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) && 297 openst->openrd) { 298 fflags = FREAD; 299 } 300 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) && 301 openst->openwr) { 302 fflags |= FWRITE; 303 } 304 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL); 305 if (error) { 306 error = DMSG_ERR_IO; 307 } else { 308 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 309 --openst->openrd; 310 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 311 --openst->openwr; 312 } 313 } 314 #endif 315 if (msg->any.head.cmd & DMSGF_DELETE) { 316 if (openst) { 317 while (openst->openrd && openst->openwr) { 318 --openst->openrd; 319 --openst->openwr; 320 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL); 321 } 322 while (openst->openrd) { 323 --openst->openrd; 324 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL); 325 } 326 while (openst->openwr) { 327 --openst->openwr; 328 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL); 329 } 330 kfree(openst, M_DEVBUF); 331 msg->state->any.any = NULL; 332 } 333 kdmsg_msg_reply(msg, error); 334 } else { 335 kdmsg_msg_result(msg, error); 336 } 337 } 338 339 static 340 void 341 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg) 342 { 343 struct dios_io *iost; 344 struct buf *bp; 345 struct bio *bio; 346 int error = DMSG_ERR_NOSUPP; 347 int reterr = 1; 348 349 /* 350 * Only DMSG_BLK_READ commands imply read ops. 351 */ 352 iost = msg->state->any.any; 353 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) { 354 if (msg->any.blk_read.bytes < DEV_BSIZE || 355 msg->any.blk_read.bytes > MAXPHYS) { 356 error = DMSG_ERR_PARAM; 357 goto done; 358 } 359 if (iost == NULL) { 360 iost = kmalloc(sizeof(*iost), M_DEVBUF, 361 M_WAITOK | M_ZERO); 362 msg->state->any.any = iost; 363 } 364 reterr = 0; 365 bp = getpbuf_mem(NULL); 366 KKASSERT(msg->any.blk_read.bytes <= bp->b_bufsize); 367 bio = &bp->b_bio1; 368 bp->b_cmd = BUF_CMD_READ; 369 bp->b_bcount = msg->any.blk_read.bytes; 370 bp->b_resid = bp->b_bcount; 371 bio->bio_offset = msg->any.blk_read.offset; 372 bio->bio_caller_info1.ptr = msg->state; 373 bio->bio_done = diskiodone; 374 375 /* kdmsg_state_hold(msg->state); */ 376 atomic_add_int(&blk_active, 1); 377 atomic_add_int(&iost->count, 1); 378 if (msg->any.head.cmd & DMSGF_DELETE) 379 iost->eof = 1; 380 BUF_KERNPROC(bp); 381 dev_dstrategy(dp->d_rawdev, bio); 382 } 383 done: 384 if (reterr) { 385 if (msg->any.head.cmd & DMSGF_DELETE) { 386 if (iost && iost->count == 0) { 387 kfree(iost, M_DEVBUF); 388 msg->state->any.any = NULL; 389 } 390 kdmsg_msg_reply(msg, error); 391 } else { 392 kdmsg_msg_result(msg, error); 393 } 394 } 395 } 396 397 static 398 void 399 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg) 400 { 401 struct dios_io *iost; 402 struct buf *bp; 403 struct bio *bio; 404 int error = DMSG_ERR_NOSUPP; 405 int reterr = 1; 406 407 /* 408 * Only DMSG_BLK_WRITE commands imply read ops. 409 */ 410 iost = msg->state->any.any; 411 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) { 412 if (msg->any.blk_write.bytes < DEV_BSIZE || 413 msg->any.blk_write.bytes > MAXPHYS) { 414 error = DMSG_ERR_PARAM; 415 goto done; 416 } 417 if (iost == NULL) { 418 iost = kmalloc(sizeof(*iost), M_DEVBUF, 419 M_WAITOK | M_ZERO); 420 msg->state->any.any = iost; 421 } 422 423 /* 424 * Issue WRITE. Short data implies zeros. Try to optimize 425 * the buffer cache buffer for the case where we can just 426 * use the message's data pointer. 427 */ 428 reterr = 0; 429 if (msg->aux_size >= msg->any.blk_write.bytes) 430 bp = getpbuf(NULL); 431 else 432 bp = getpbuf_mem(NULL); 433 KKASSERT(msg->any.blk_write.bytes <= bp->b_bufsize); 434 bio = &bp->b_bio1; 435 bp->b_cmd = BUF_CMD_WRITE; 436 bp->b_bcount = msg->any.blk_write.bytes; 437 bp->b_resid = bp->b_bcount; 438 if (msg->aux_size >= msg->any.blk_write.bytes) { 439 bp->b_data = msg->aux_data; 440 kdmsg_detach_aux_data(msg, &iost->data); 441 } else { 442 bcopy(msg->aux_data, bp->b_data, msg->aux_size); 443 bzero(bp->b_data + msg->aux_size, 444 msg->any.blk_write.bytes - msg->aux_size); 445 bzero(&iost->data, sizeof(iost->data)); 446 } 447 bio->bio_offset = msg->any.blk_write.offset; 448 bio->bio_caller_info1.ptr = msg->state; 449 bio->bio_done = diskiodone; 450 451 /* kdmsg_state_hold(msg->state); */ 452 atomic_add_int(&blk_active, 1); 453 atomic_add_int(&iost->count, 1); 454 if (msg->any.head.cmd & DMSGF_DELETE) 455 iost->eof = 1; 456 BUF_KERNPROC(bp); 457 dev_dstrategy(dp->d_rawdev, bio); 458 } 459 done: 460 if (reterr) { 461 if (msg->any.head.cmd & DMSGF_DELETE) { 462 if (iost && iost->count == 0) { 463 kfree(iost, M_DEVBUF); 464 msg->state->any.any = NULL; 465 } 466 kdmsg_msg_reply(msg, error); 467 } else { 468 kdmsg_msg_result(msg, error); 469 } 470 } 471 } 472 473 static 474 void 475 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg) 476 { 477 struct dios_io *iost; 478 struct buf *bp; 479 struct bio *bio; 480 int error = DMSG_ERR_NOSUPP; 481 int reterr = 1; 482 483 /* 484 * Only DMSG_BLK_FLUSH commands imply read ops. 485 */ 486 iost = msg->state->any.any; 487 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) { 488 if (iost == NULL) { 489 iost = kmalloc(sizeof(*iost), M_DEVBUF, 490 M_WAITOK | M_ZERO); 491 msg->state->any.any = iost; 492 } 493 reterr = 0; 494 bp = getpbuf(NULL); 495 bio = &bp->b_bio1; 496 bp->b_cmd = BUF_CMD_FLUSH; 497 bp->b_bcount = msg->any.blk_flush.bytes; 498 bp->b_resid = 0; 499 bio->bio_offset = msg->any.blk_flush.offset; 500 bio->bio_caller_info1.ptr = msg->state; 501 bio->bio_done = diskiodone; 502 503 /* kdmsg_state_hold(msg->state); */ 504 atomic_add_int(&blk_active, 1); 505 atomic_add_int(&iost->count, 1); 506 if (msg->any.head.cmd & DMSGF_DELETE) 507 iost->eof = 1; 508 BUF_KERNPROC(bp); 509 dev_dstrategy(dp->d_rawdev, bio); 510 } 511 if (reterr) { 512 if (msg->any.head.cmd & DMSGF_DELETE) { 513 if (iost && iost->count == 0) { 514 kfree(iost, M_DEVBUF); 515 msg->state->any.any = NULL; 516 } 517 kdmsg_msg_reply(msg, error); 518 } else { 519 kdmsg_msg_result(msg, error); 520 } 521 } 522 } 523 524 static 525 void 526 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg) 527 { 528 struct dios_io *iost; 529 struct buf *bp; 530 struct bio *bio; 531 int error = DMSG_ERR_NOSUPP; 532 int reterr = 1; 533 534 /* 535 * Only DMSG_BLK_FREEBLKS commands imply read ops. 536 */ 537 iost = msg->state->any.any; 538 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) { 539 if (iost == NULL) { 540 iost = kmalloc(sizeof(*iost), M_DEVBUF, 541 M_WAITOK | M_ZERO); 542 msg->state->any.any = iost; 543 } 544 reterr = 0; 545 bp = getpbuf(NULL); 546 bio = &bp->b_bio1; 547 bp->b_cmd = BUF_CMD_FREEBLKS; 548 bp->b_bcount = msg->any.blk_freeblks.bytes; 549 bp->b_resid = 0; 550 bio->bio_offset = msg->any.blk_freeblks.offset; 551 bio->bio_caller_info1.ptr = msg->state; 552 bio->bio_done = diskiodone; 553 554 /* kdmsg_state_hold(msg->state); */ 555 atomic_add_int(&blk_active, 1); 556 atomic_add_int(&iost->count, 1); 557 if (msg->any.head.cmd & DMSGF_DELETE) 558 iost->eof = 1; 559 BUF_KERNPROC(bp); 560 dev_dstrategy(dp->d_rawdev, bio); 561 } 562 if (reterr) { 563 if (msg->any.head.cmd & DMSGF_DELETE) { 564 if (iost && iost->count == 0) { 565 kfree(iost, M_DEVBUF); 566 msg->state->any.any = NULL; 567 } 568 kdmsg_msg_reply(msg, error); 569 } else { 570 kdmsg_msg_result(msg, error); 571 } 572 } 573 } 574 575 static 576 void 577 diskiodone(struct bio *bio) 578 { 579 struct buf *bp = bio->bio_buf; 580 kdmsg_state_t *state = bio->bio_caller_info1.ptr; 581 kdmsg_msg_t *rmsg; 582 struct dios_io *iost = state->any.any; 583 int error; 584 int resid = 0; 585 int bytes; 586 uint32_t cmd; 587 void *data; 588 589 cmd = DMSG_LNK_ERROR; 590 data = NULL; 591 bytes = 0; 592 593 switch(bp->b_cmd) { 594 case BUF_CMD_READ: 595 cmd = DMSG_LNK_ERROR; 596 data = bp->b_data; 597 bytes = bp->b_bcount; 598 /* fall through */ 599 case BUF_CMD_WRITE: 600 if (bp->b_flags & B_ERROR) { 601 error = bp->b_error; 602 } else { 603 error = 0; 604 resid = bp->b_resid; 605 } 606 kdmsg_free_aux_data(&iost->data); 607 break; 608 case BUF_CMD_FLUSH: 609 case BUF_CMD_FREEBLKS: 610 if (bp->b_flags & B_ERROR) 611 error = bp->b_error; 612 else 613 error = 0; 614 break; 615 default: 616 panic("diskiodone: Unknown bio cmd = %d\n", 617 bio->bio_buf->b_cmd); 618 error = 0; /* avoid compiler warning */ 619 break; /* NOT REACHED */ 620 } 621 622 /* 623 * Convert error to DMSG_ERR_* code. 624 */ 625 if (error) 626 error = DMSG_ERR_IO; 627 628 /* 629 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will 630 * have already converted cmd to BLK_ERROR and set up data to return. 631 */ 632 if (resid && cmd == DMSG_LNK_ERROR) 633 cmd = DMSG_BLK_ERROR; 634 /* XXX txcmd is delayed so this won't work for streaming */ 635 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */ 636 cmd |= DMSGF_CREATE; 637 if (iost->eof) { 638 if (atomic_fetchadd_int(&iost->count, -1) == 1) 639 cmd |= DMSGF_DELETE; 640 } else { 641 atomic_add_int(&iost->count, -1); 642 } 643 atomic_add_int(&blk_active, -1); 644 cmd |= DMSGF_REPLY; 645 646 /* 647 * Allocate a basic or extended reply. Be careful not to populate 648 * extended header fields unless we allocated an extended reply. 649 */ 650 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0); 651 if (data) { 652 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT); 653 rmsg->aux_size = bytes; 654 rmsg->flags |= KDMSG_FLAG_AUXALLOC; 655 bcopy(data, rmsg->aux_data, bytes); 656 } 657 rmsg->any.blk_error.head.error = error; 658 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR) 659 rmsg->any.blk_error.resid = resid; 660 bio->bio_caller_info1.ptr = NULL; 661 /* kdmsg_state_drop(state); */ 662 kdmsg_msg_write(rmsg); 663 if (bp->b_flags & B_PAGING) { 664 relpbuf(bp, NULL); 665 } else { 666 bp->b_flags |= B_INVAL | B_AGE; 667 relpbuf(bp, NULL); 668 } 669 } 670