1 /* 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 #include <sys/buf.h> 40 #include <sys/conf.h> 41 #include <sys/disklabel.h> 42 #include <sys/disklabel32.h> 43 #include <sys/disklabel64.h> 44 #include <sys/diskslice.h> 45 #include <sys/diskmbr.h> 46 #include <sys/disk.h> 47 #include <sys/malloc.h> 48 #include <sys/device.h> 49 #include <sys/devfs.h> 50 #include <sys/thread.h> 51 #include <sys/queue.h> 52 #include <sys/lock.h> 53 #include <sys/stat.h> 54 #include <sys/uuid.h> 55 #include <sys/dmsg.h> 56 57 #include <sys/buf2.h> 58 #include <sys/msgport2.h> 59 60 struct dios_open { 61 int openrd; 62 int openwr; 63 }; 64 65 struct dios_io { 66 int count; 67 int eof; 68 kdmsg_data_t data; 69 }; 70 71 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg"); 72 73 static int blk_active; 74 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0, 75 "Number of active iocom IOs"); 76 77 static int disk_iocom_reconnect(struct disk *dp, struct file *fp); 78 static int disk_rcvdmsg(kdmsg_msg_t *msg); 79 80 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg); 81 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg); 82 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg); 83 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg); 84 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg); 85 static void diskiodone(struct bio *bio); 86 87 void 88 disk_iocom_init(struct disk *dp) 89 { 90 kdmsg_iocom_init(&dp->d_iocom, dp, 91 KDMSG_IOCOMF_AUTOCONN | 92 KDMSG_IOCOMF_AUTORXSPAN | 93 KDMSG_IOCOMF_AUTOTXSPAN, 94 M_DMSG_DISK, disk_rcvdmsg); 95 } 96 97 void 98 disk_iocom_update(struct disk *dp) 99 { 100 } 101 102 void 103 disk_iocom_uninit(struct disk *dp) 104 { 105 kdmsg_iocom_uninit(&dp->d_iocom); 106 } 107 108 int 109 disk_iocom_ioctl(struct disk *dp, u_long cmd, void *data) 110 { 111 struct file *fp; 112 struct disk_ioc_recluster *recl; 113 int error; 114 115 switch(cmd) { 116 case DIOCRECLUSTER: 117 recl = data; 118 fp = holdfp(curthread, recl->fd, -1); 119 if (fp) { 120 error = disk_iocom_reconnect(dp, fp); 121 } else { 122 error = EINVAL; 123 } 124 break; 125 default: 126 error = EOPNOTSUPP; 127 break; 128 } 129 return error; 130 } 131 132 static 133 int 134 disk_iocom_reconnect(struct disk *dp, struct file *fp) 135 { 136 char devname[64]; 137 138 ksnprintf(devname, sizeof(devname), "%s%d", 139 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev)); 140 141 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname); 142 143 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1; 144 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK; 145 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK; 146 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1; 147 #if 0 148 if (dp->d_info.d_serialno) { 149 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 150 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 151 "%s/%s", hostname, dp->d_info.d_serialno); 152 } else { 153 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 154 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 155 "%s/%s", hostname, devname); 156 } 157 #endif 158 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 159 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 160 "%s/%s", hostname, devname); 161 162 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1; 163 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK; 164 dp->d_iocom.auto_lnk_span.media.block.bytes = 165 dp->d_info.d_media_size; 166 dp->d_iocom.auto_lnk_span.media.block.blksize = 167 dp->d_info.d_media_blksize; 168 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label, 169 sizeof(dp->d_iocom.auto_lnk_span.peer_label), 170 "%s", dp->d_iocom.auto_lnk_conn.peer_label); 171 if (dp->d_info.d_serialno) { 172 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 173 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 174 "%s", dp->d_info.d_serialno); 175 } else { 176 /* 177 * If no serial number is available generate a dummy serial 178 * number from the host and device name and pray. This will 179 * allow e.g. /dev/vn* to look meaningful on a remote machine. 180 */ 181 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 182 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 183 "%s.%s", hostname, devname); 184 } 185 186 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL); 187 188 return (0); 189 } 190 191 static int 192 disk_rcvdmsg(kdmsg_msg_t *msg) 193 { 194 struct disk *dp = msg->state->iocom->handle; 195 196 /* 197 * Handle debug messages (these might not be in transactions) 198 */ 199 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) { 200 case DMSG_DBG_SHELL: 201 /* 202 * Execute shell command (not supported atm) 203 */ 204 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 205 return(0); 206 case DMSG_DBG_SHELL | DMSGF_REPLY: 207 if (msg->aux_data) { 208 msg->aux_data[msg->aux_size - 1] = 0; 209 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data); 210 } 211 return(0); 212 } 213 214 /* 215 * All remaining messages must be in a transaction. 216 * 217 * NOTE! We currently don't care if the transaction is just 218 * the span transaction (for disk probes) or if it is the 219 * BLK_OPEN transaction. 220 * 221 * NOTE! We are switching on the first message's command. The 222 * actual message command within the transaction may be 223 * different (if streaming within a transaction). 224 */ 225 if (msg->state == &msg->state->iocom->state0) { 226 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 227 return(0); 228 } 229 230 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) { 231 case DMSG_BLK_OPEN: 232 disk_blk_open(dp, msg); 233 break; 234 case DMSG_BLK_READ: 235 /* 236 * not reached normally but leave in for completeness 237 */ 238 disk_blk_read(dp, msg); 239 break; 240 case DMSG_BLK_WRITE: 241 disk_blk_write(dp, msg); 242 break; 243 case DMSG_BLK_FLUSH: 244 disk_blk_flush(dp, msg); 245 break; 246 case DMSG_BLK_FREEBLKS: 247 disk_blk_freeblks(dp, msg); 248 break; 249 default: 250 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) { 251 if (msg->any.head.cmd & DMSGF_DELETE) 252 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 253 else 254 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP); 255 } 256 break; 257 } 258 return (0); 259 } 260 261 static 262 void 263 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg) 264 { 265 struct dios_open *openst; 266 int error = DMSG_ERR_NOSUPP; 267 int fflags; 268 269 openst = msg->state->any.any; 270 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) { 271 if (openst == NULL) { 272 openst = kmalloc(sizeof(*openst), M_DEVBUF, 273 M_WAITOK | M_ZERO); 274 msg->state->any.any = openst; 275 } 276 fflags = 0; 277 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 278 fflags = FREAD; 279 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 280 fflags |= FWRITE; 281 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL); 282 if (error) { 283 error = DMSG_ERR_IO; 284 } else { 285 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 286 ++openst->openrd; 287 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 288 ++openst->openwr; 289 } 290 } 291 #if 0 292 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE && 293 openst) { 294 fflags = 0; 295 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) && 296 openst->openrd) { 297 fflags = FREAD; 298 } 299 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) && 300 openst->openwr) { 301 fflags |= FWRITE; 302 } 303 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL); 304 if (error) { 305 error = DMSG_ERR_IO; 306 } else { 307 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 308 --openst->openrd; 309 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 310 --openst->openwr; 311 } 312 } 313 #endif 314 if (msg->any.head.cmd & DMSGF_DELETE) { 315 if (openst) { 316 while (openst->openrd && openst->openwr) { 317 --openst->openrd; 318 --openst->openwr; 319 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL); 320 } 321 while (openst->openrd) { 322 --openst->openrd; 323 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL); 324 } 325 while (openst->openwr) { 326 --openst->openwr; 327 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL); 328 } 329 kfree(openst, M_DEVBUF); 330 msg->state->any.any = NULL; 331 } 332 kdmsg_msg_reply(msg, error); 333 } else { 334 kdmsg_msg_result(msg, error); 335 } 336 } 337 338 static 339 void 340 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg) 341 { 342 struct dios_io *iost; 343 struct buf *bp; 344 struct bio *bio; 345 int error = DMSG_ERR_NOSUPP; 346 int reterr = 1; 347 348 /* 349 * Only DMSG_BLK_READ commands imply read ops. 350 */ 351 iost = msg->state->any.any; 352 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) { 353 if (msg->any.blk_read.bytes < DEV_BSIZE || 354 msg->any.blk_read.bytes > MAXPHYS) { 355 error = DMSG_ERR_PARAM; 356 goto done; 357 } 358 if (iost == NULL) { 359 iost = kmalloc(sizeof(*iost), M_DEVBUF, 360 M_WAITOK | M_ZERO); 361 msg->state->any.any = iost; 362 } 363 reterr = 0; 364 bp = getpbuf_mem(NULL); 365 KKASSERT(msg->any.blk_read.bytes <= bp->b_bufsize); 366 bio = &bp->b_bio1; 367 bp->b_cmd = BUF_CMD_READ; 368 bp->b_bcount = msg->any.blk_read.bytes; 369 bp->b_resid = bp->b_bcount; 370 bio->bio_offset = msg->any.blk_read.offset; 371 bio->bio_caller_info1.ptr = msg->state; 372 bio->bio_done = diskiodone; 373 374 /* kdmsg_state_hold(msg->state); */ 375 atomic_add_int(&blk_active, 1); 376 atomic_add_int(&iost->count, 1); 377 if (msg->any.head.cmd & DMSGF_DELETE) 378 iost->eof = 1; 379 BUF_KERNPROC(bp); 380 dev_dstrategy(dp->d_rawdev, bio); 381 } 382 done: 383 if (reterr) { 384 if (msg->any.head.cmd & DMSGF_DELETE) { 385 if (iost && iost->count == 0) { 386 kfree(iost, M_DEVBUF); 387 msg->state->any.any = NULL; 388 } 389 kdmsg_msg_reply(msg, error); 390 } else { 391 kdmsg_msg_result(msg, error); 392 } 393 } 394 } 395 396 static 397 void 398 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg) 399 { 400 struct dios_io *iost; 401 struct buf *bp; 402 struct bio *bio; 403 int error = DMSG_ERR_NOSUPP; 404 int reterr = 1; 405 406 /* 407 * Only DMSG_BLK_WRITE commands imply read ops. 408 */ 409 iost = msg->state->any.any; 410 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) { 411 if (msg->any.blk_write.bytes < DEV_BSIZE || 412 msg->any.blk_write.bytes > MAXPHYS) { 413 error = DMSG_ERR_PARAM; 414 goto done; 415 } 416 if (iost == NULL) { 417 iost = kmalloc(sizeof(*iost), M_DEVBUF, 418 M_WAITOK | M_ZERO); 419 msg->state->any.any = iost; 420 } 421 422 /* 423 * Issue WRITE. Short data implies zeros. Try to optimize 424 * the buffer cache buffer for the case where we can just 425 * use the message's data pointer. 426 */ 427 reterr = 0; 428 if (msg->aux_size >= msg->any.blk_write.bytes) 429 bp = getpbuf(NULL); 430 else 431 bp = getpbuf_mem(NULL); 432 KKASSERT(msg->any.blk_write.bytes <= bp->b_bufsize); 433 bio = &bp->b_bio1; 434 bp->b_cmd = BUF_CMD_WRITE; 435 bp->b_bcount = msg->any.blk_write.bytes; 436 bp->b_resid = bp->b_bcount; 437 if (msg->aux_size >= msg->any.blk_write.bytes) { 438 bp->b_data = msg->aux_data; 439 kdmsg_detach_aux_data(msg, &iost->data); 440 } else { 441 bcopy(msg->aux_data, bp->b_data, msg->aux_size); 442 bzero(bp->b_data + msg->aux_size, 443 msg->any.blk_write.bytes - msg->aux_size); 444 bzero(&iost->data, sizeof(iost->data)); 445 } 446 bio->bio_offset = msg->any.blk_write.offset; 447 bio->bio_caller_info1.ptr = msg->state; 448 bio->bio_done = diskiodone; 449 450 /* kdmsg_state_hold(msg->state); */ 451 atomic_add_int(&blk_active, 1); 452 atomic_add_int(&iost->count, 1); 453 if (msg->any.head.cmd & DMSGF_DELETE) 454 iost->eof = 1; 455 BUF_KERNPROC(bp); 456 dev_dstrategy(dp->d_rawdev, bio); 457 } 458 done: 459 if (reterr) { 460 if (msg->any.head.cmd & DMSGF_DELETE) { 461 if (iost && iost->count == 0) { 462 kfree(iost, M_DEVBUF); 463 msg->state->any.any = NULL; 464 } 465 kdmsg_msg_reply(msg, error); 466 } else { 467 kdmsg_msg_result(msg, error); 468 } 469 } 470 } 471 472 static 473 void 474 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg) 475 { 476 struct dios_io *iost; 477 struct buf *bp; 478 struct bio *bio; 479 int error = DMSG_ERR_NOSUPP; 480 int reterr = 1; 481 482 /* 483 * Only DMSG_BLK_FLUSH commands imply read ops. 484 */ 485 iost = msg->state->any.any; 486 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) { 487 if (iost == NULL) { 488 iost = kmalloc(sizeof(*iost), M_DEVBUF, 489 M_WAITOK | M_ZERO); 490 msg->state->any.any = iost; 491 } 492 reterr = 0; 493 bp = getpbuf(NULL); 494 bio = &bp->b_bio1; 495 bp->b_cmd = BUF_CMD_FLUSH; 496 bp->b_bcount = msg->any.blk_flush.bytes; 497 bp->b_resid = 0; 498 bio->bio_offset = msg->any.blk_flush.offset; 499 bio->bio_caller_info1.ptr = msg->state; 500 bio->bio_done = diskiodone; 501 502 /* kdmsg_state_hold(msg->state); */ 503 atomic_add_int(&blk_active, 1); 504 atomic_add_int(&iost->count, 1); 505 if (msg->any.head.cmd & DMSGF_DELETE) 506 iost->eof = 1; 507 BUF_KERNPROC(bp); 508 dev_dstrategy(dp->d_rawdev, bio); 509 } 510 if (reterr) { 511 if (msg->any.head.cmd & DMSGF_DELETE) { 512 if (iost && iost->count == 0) { 513 kfree(iost, M_DEVBUF); 514 msg->state->any.any = NULL; 515 } 516 kdmsg_msg_reply(msg, error); 517 } else { 518 kdmsg_msg_result(msg, error); 519 } 520 } 521 } 522 523 static 524 void 525 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg) 526 { 527 struct dios_io *iost; 528 struct buf *bp; 529 struct bio *bio; 530 int error = DMSG_ERR_NOSUPP; 531 int reterr = 1; 532 533 /* 534 * Only DMSG_BLK_FREEBLKS commands imply read ops. 535 */ 536 iost = msg->state->any.any; 537 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) { 538 if (iost == NULL) { 539 iost = kmalloc(sizeof(*iost), M_DEVBUF, 540 M_WAITOK | M_ZERO); 541 msg->state->any.any = iost; 542 } 543 reterr = 0; 544 bp = getpbuf(NULL); 545 bio = &bp->b_bio1; 546 bp->b_cmd = BUF_CMD_FREEBLKS; 547 bp->b_bcount = msg->any.blk_freeblks.bytes; 548 bp->b_resid = 0; 549 bio->bio_offset = msg->any.blk_freeblks.offset; 550 bio->bio_caller_info1.ptr = msg->state; 551 bio->bio_done = diskiodone; 552 553 /* kdmsg_state_hold(msg->state); */ 554 atomic_add_int(&blk_active, 1); 555 atomic_add_int(&iost->count, 1); 556 if (msg->any.head.cmd & DMSGF_DELETE) 557 iost->eof = 1; 558 BUF_KERNPROC(bp); 559 dev_dstrategy(dp->d_rawdev, bio); 560 } 561 if (reterr) { 562 if (msg->any.head.cmd & DMSGF_DELETE) { 563 if (iost && iost->count == 0) { 564 kfree(iost, M_DEVBUF); 565 msg->state->any.any = NULL; 566 } 567 kdmsg_msg_reply(msg, error); 568 } else { 569 kdmsg_msg_result(msg, error); 570 } 571 } 572 } 573 574 static 575 void 576 diskiodone(struct bio *bio) 577 { 578 struct buf *bp = bio->bio_buf; 579 kdmsg_state_t *state = bio->bio_caller_info1.ptr; 580 kdmsg_msg_t *rmsg; 581 struct dios_io *iost = state->any.any; 582 int error; 583 int resid = 0; 584 int bytes; 585 uint32_t cmd; 586 void *data; 587 588 cmd = DMSG_LNK_ERROR; 589 data = NULL; 590 bytes = 0; 591 592 switch(bp->b_cmd) { 593 case BUF_CMD_READ: 594 cmd = DMSG_LNK_ERROR; 595 data = bp->b_data; 596 bytes = bp->b_bcount; 597 /* fall through */ 598 case BUF_CMD_WRITE: 599 if (bp->b_flags & B_ERROR) { 600 error = bp->b_error; 601 } else { 602 error = 0; 603 resid = bp->b_resid; 604 } 605 kdmsg_free_aux_data(&iost->data); 606 break; 607 case BUF_CMD_FLUSH: 608 case BUF_CMD_FREEBLKS: 609 if (bp->b_flags & B_ERROR) 610 error = bp->b_error; 611 else 612 error = 0; 613 break; 614 default: 615 panic("diskiodone: Unknown bio cmd = %d\n", 616 bio->bio_buf->b_cmd); 617 error = 0; /* avoid compiler warning */ 618 break; /* NOT REACHED */ 619 } 620 621 /* 622 * Convert error to DMSG_ERR_* code. 623 */ 624 if (error) 625 error = DMSG_ERR_IO; 626 627 /* 628 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will 629 * have already converted cmd to BLK_ERROR and set up data to return. 630 */ 631 if (resid && cmd == DMSG_LNK_ERROR) 632 cmd = DMSG_BLK_ERROR; 633 /* XXX txcmd is delayed so this won't work for streaming */ 634 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */ 635 cmd |= DMSGF_CREATE; 636 if (iost->eof) { 637 if (atomic_fetchadd_int(&iost->count, -1) == 1) 638 cmd |= DMSGF_DELETE; 639 } else { 640 atomic_add_int(&iost->count, -1); 641 } 642 atomic_add_int(&blk_active, -1); 643 cmd |= DMSGF_REPLY; 644 645 /* 646 * Allocate a basic or extended reply. Be careful not to populate 647 * extended header fields unless we allocated an extended reply. 648 */ 649 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0); 650 if (data) { 651 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT); 652 rmsg->aux_size = bytes; 653 rmsg->flags |= KDMSG_FLAG_AUXALLOC; 654 bcopy(data, rmsg->aux_data, bytes); 655 } 656 rmsg->any.blk_error.head.error = error; 657 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR) 658 rmsg->any.blk_error.resid = resid; 659 bio->bio_caller_info1.ptr = NULL; 660 /* kdmsg_state_drop(state); */ 661 kdmsg_msg_write(rmsg); 662 if (bp->b_flags & B_PAGING) { 663 relpbuf(bp, NULL); 664 } else { 665 bp->b_flags |= B_INVAL | B_AGE; 666 relpbuf(bp, NULL); 667 } 668 } 669