1 /* 2 * Copyright (c) 2012 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 #include <sys/buf.h> 40 #include <sys/conf.h> 41 #include <sys/disklabel.h> 42 #include <sys/disklabel32.h> 43 #include <sys/disklabel64.h> 44 #include <sys/diskslice.h> 45 #include <sys/diskmbr.h> 46 #include <sys/disk.h> 47 #include <sys/malloc.h> 48 #include <sys/device.h> 49 #include <sys/devfs.h> 50 #include <sys/thread.h> 51 #include <sys/queue.h> 52 #include <sys/lock.h> 53 #include <sys/stat.h> 54 #include <sys/uuid.h> 55 #include <sys/dmsg.h> 56 57 #include <sys/buf2.h> 58 #include <sys/msgport2.h> 59 #include <sys/thread2.h> 60 61 struct dios_open { 62 int openrd; 63 int openwr; 64 }; 65 66 struct dios_io { 67 int count; 68 int eof; 69 kdmsg_data_t data; 70 }; 71 72 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg"); 73 74 static int blk_active; 75 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0, 76 "Number of active iocom IOs"); 77 78 static int disk_iocom_reconnect(struct disk *dp, struct file *fp); 79 static int disk_rcvdmsg(kdmsg_msg_t *msg); 80 81 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg); 82 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg); 83 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg); 84 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg); 85 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg); 86 static void diskiodone(struct bio *bio); 87 88 void 89 disk_iocom_init(struct disk *dp) 90 { 91 kdmsg_iocom_init(&dp->d_iocom, dp, 92 KDMSG_IOCOMF_AUTOCONN | 93 KDMSG_IOCOMF_AUTORXSPAN | 94 KDMSG_IOCOMF_AUTOTXSPAN, 95 M_DMSG_DISK, disk_rcvdmsg); 96 } 97 98 void 99 disk_iocom_update(struct disk *dp) 100 { 101 } 102 103 void 104 disk_iocom_uninit(struct disk *dp) 105 { 106 kdmsg_iocom_uninit(&dp->d_iocom); 107 } 108 109 int 110 disk_iocom_ioctl(struct disk *dp, u_long cmd, void *data) 111 { 112 struct file *fp; 113 struct disk_ioc_recluster *recl; 114 int error; 115 116 switch(cmd) { 117 case DIOCRECLUSTER: 118 recl = data; 119 fp = holdfp(curproc->p_fd, recl->fd, -1); 120 if (fp) { 121 error = disk_iocom_reconnect(dp, fp); 122 } else { 123 error = EINVAL; 124 } 125 break; 126 default: 127 error = EOPNOTSUPP; 128 break; 129 } 130 return error; 131 } 132 133 static 134 int 135 disk_iocom_reconnect(struct disk *dp, struct file *fp) 136 { 137 char devname[64]; 138 139 ksnprintf(devname, sizeof(devname), "%s%d", 140 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev)); 141 142 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname); 143 144 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1; 145 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK; 146 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK; 147 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1; 148 #if 0 149 if (dp->d_info.d_serialno) { 150 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 151 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 152 "%s/%s", hostname, dp->d_info.d_serialno); 153 } else { 154 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 155 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 156 "%s/%s", hostname, devname); 157 } 158 #endif 159 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label, 160 sizeof(dp->d_iocom.auto_lnk_conn.peer_label), 161 "%s/%s", hostname, devname); 162 163 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1; 164 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK; 165 dp->d_iocom.auto_lnk_span.media.block.bytes = 166 dp->d_info.d_media_size; 167 dp->d_iocom.auto_lnk_span.media.block.blksize = 168 dp->d_info.d_media_blksize; 169 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label, 170 sizeof(dp->d_iocom.auto_lnk_span.peer_label), 171 "%s", dp->d_iocom.auto_lnk_conn.peer_label); 172 if (dp->d_info.d_serialno) { 173 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 174 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 175 "%s", dp->d_info.d_serialno); 176 } else { 177 /* 178 * If no serial number is available generate a dummy serial 179 * number from the host and device name and pray. This will 180 * allow e.g. /dev/vn* to look meaningful on a remote machine. 181 */ 182 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label, 183 sizeof(dp->d_iocom.auto_lnk_span.pfs_label), 184 "%s.%s", hostname, devname); 185 } 186 187 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL); 188 189 return (0); 190 } 191 192 static int 193 disk_rcvdmsg(kdmsg_msg_t *msg) 194 { 195 struct disk *dp = msg->state->iocom->handle; 196 197 /* 198 * Handle debug messages (these might not be in transactions) 199 */ 200 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) { 201 case DMSG_DBG_SHELL: 202 /* 203 * Execute shell command (not supported atm) 204 */ 205 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 206 return(0); 207 case DMSG_DBG_SHELL | DMSGF_REPLY: 208 if (msg->aux_data) { 209 msg->aux_data[msg->aux_size - 1] = 0; 210 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data); 211 } 212 return(0); 213 } 214 215 /* 216 * All remaining messages must be in a transaction. 217 * 218 * NOTE! We currently don't care if the transaction is just 219 * the span transaction (for disk probes) or if it is the 220 * BLK_OPEN transaction. 221 * 222 * NOTE! We are switching on the first message's command. The 223 * actual message command within the transaction may be 224 * different (if streaming within a transaction). 225 */ 226 if (msg->state == &msg->state->iocom->state0) { 227 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 228 return(0); 229 } 230 231 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) { 232 case DMSG_BLK_OPEN: 233 disk_blk_open(dp, msg); 234 break; 235 case DMSG_BLK_READ: 236 /* 237 * not reached normally but leave in for completeness 238 */ 239 disk_blk_read(dp, msg); 240 break; 241 case DMSG_BLK_WRITE: 242 disk_blk_write(dp, msg); 243 break; 244 case DMSG_BLK_FLUSH: 245 disk_blk_flush(dp, msg); 246 break; 247 case DMSG_BLK_FREEBLKS: 248 disk_blk_freeblks(dp, msg); 249 break; 250 default: 251 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) { 252 if (msg->any.head.cmd & DMSGF_DELETE) 253 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP); 254 else 255 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP); 256 } 257 break; 258 } 259 return (0); 260 } 261 262 static 263 void 264 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg) 265 { 266 struct dios_open *openst; 267 int error = DMSG_ERR_NOSUPP; 268 int fflags; 269 270 openst = msg->state->any.any; 271 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) { 272 if (openst == NULL) { 273 openst = kmalloc(sizeof(*openst), M_DEVBUF, 274 M_WAITOK | M_ZERO); 275 msg->state->any.any = openst; 276 } 277 fflags = 0; 278 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 279 fflags = FREAD; 280 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 281 fflags |= FWRITE; 282 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL); 283 if (error) { 284 error = DMSG_ERR_IO; 285 } else { 286 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 287 ++openst->openrd; 288 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 289 ++openst->openwr; 290 } 291 } 292 #if 0 293 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE && 294 openst) { 295 fflags = 0; 296 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) && 297 openst->openrd) { 298 fflags = FREAD; 299 } 300 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) && 301 openst->openwr) { 302 fflags |= FWRITE; 303 } 304 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL); 305 if (error) { 306 error = DMSG_ERR_IO; 307 } else { 308 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD) 309 --openst->openrd; 310 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR) 311 --openst->openwr; 312 } 313 } 314 #endif 315 if (msg->any.head.cmd & DMSGF_DELETE) { 316 if (openst) { 317 while (openst->openrd && openst->openwr) { 318 --openst->openrd; 319 --openst->openwr; 320 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL); 321 } 322 while (openst->openrd) { 323 --openst->openrd; 324 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL); 325 } 326 while (openst->openwr) { 327 --openst->openwr; 328 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL); 329 } 330 kfree(openst, M_DEVBUF); 331 msg->state->any.any = NULL; 332 } 333 kdmsg_msg_reply(msg, error); 334 } else { 335 kdmsg_msg_result(msg, error); 336 } 337 } 338 339 static 340 void 341 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg) 342 { 343 struct dios_io *iost; 344 struct buf *bp; 345 struct bio *bio; 346 int error = DMSG_ERR_NOSUPP; 347 int reterr = 1; 348 349 /* 350 * Only DMSG_BLK_READ commands imply read ops. 351 */ 352 iost = msg->state->any.any; 353 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) { 354 if (msg->any.blk_read.bytes < DEV_BSIZE || 355 msg->any.blk_read.bytes > MAXPHYS) { 356 error = DMSG_ERR_PARAM; 357 goto done; 358 } 359 if (iost == NULL) { 360 iost = kmalloc(sizeof(*iost), M_DEVBUF, 361 M_WAITOK | M_ZERO); 362 msg->state->any.any = iost; 363 } 364 reterr = 0; 365 bp = getpbuf_mem(NULL); 366 KKASSERT(msg->any.blk_read.bytes <= bp->b_bufsize); 367 bio = &bp->b_bio1; 368 bp->b_cmd = BUF_CMD_READ; 369 bp->b_bcount = msg->any.blk_read.bytes; 370 bp->b_resid = bp->b_bcount; 371 bio->bio_offset = msg->any.blk_read.offset; 372 bio->bio_caller_info1.ptr = msg->state; 373 bio->bio_done = diskiodone; 374 375 /* kdmsg_state_hold(msg->state); */ 376 atomic_add_int(&blk_active, 1); 377 atomic_add_int(&iost->count, 1); 378 if (msg->any.head.cmd & DMSGF_DELETE) 379 iost->eof = 1; 380 BUF_KERNPROC(bp); 381 dev_dstrategy(dp->d_rawdev, bio); 382 } 383 done: 384 if (reterr) { 385 if (msg->any.head.cmd & DMSGF_DELETE) { 386 if (iost && iost->count == 0) { 387 kfree(iost, M_DEVBUF); 388 msg->state->any.any = NULL; 389 } 390 kdmsg_msg_reply(msg, error); 391 } else { 392 kdmsg_msg_result(msg, error); 393 } 394 } 395 } 396 397 static 398 void 399 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg) 400 { 401 struct dios_io *iost; 402 struct buf *bp; 403 struct bio *bio; 404 int error = DMSG_ERR_NOSUPP; 405 int reterr = 1; 406 407 /* 408 * Only DMSG_BLK_WRITE commands imply read ops. 409 */ 410 iost = msg->state->any.any; 411 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) { 412 if (msg->any.blk_write.bytes < DEV_BSIZE || 413 msg->any.blk_write.bytes > MAXPHYS) { 414 error = DMSG_ERR_PARAM; 415 goto done; 416 } 417 if (iost == NULL) { 418 iost = kmalloc(sizeof(*iost), M_DEVBUF, 419 M_WAITOK | M_ZERO); 420 msg->state->any.any = iost; 421 } 422 423 /* 424 * Issue WRITE. Short data implies zeros. Try to optimize 425 * the buffer cache buffer for the case where we can just 426 * use the message's data pointer. 427 */ 428 reterr = 0; 429 if (msg->aux_size >= msg->any.blk_write.bytes) 430 bp = getpbuf(NULL); 431 else 432 bp = getpbuf_mem(NULL); 433 KKASSERT(msg->any.blk_write.bytes <= bp->b_bufsize); 434 bio = &bp->b_bio1; 435 bp->b_cmd = BUF_CMD_WRITE; 436 bp->b_bcount = msg->any.blk_write.bytes; 437 bp->b_resid = bp->b_bcount; 438 if (msg->aux_size >= msg->any.blk_write.bytes) { 439 bp->b_data = msg->aux_data; 440 kdmsg_detach_aux_data(msg, &iost->data); 441 } else { 442 bcopy(msg->aux_data, bp->b_data, msg->aux_size); 443 bzero(bp->b_data + msg->aux_size, 444 msg->any.blk_write.bytes - msg->aux_size); 445 bzero(&iost->data, sizeof(iost->data)); 446 } 447 bio->bio_offset = msg->any.blk_write.offset; 448 bio->bio_caller_info1.ptr = msg->state; 449 bio->bio_done = diskiodone; 450 451 /* kdmsg_state_hold(msg->state); */ 452 atomic_add_int(&blk_active, 1); 453 atomic_add_int(&iost->count, 1); 454 if (msg->any.head.cmd & DMSGF_DELETE) 455 iost->eof = 1; 456 BUF_KERNPROC(bp); 457 dev_dstrategy(dp->d_rawdev, bio); 458 } 459 done: 460 if (reterr) { 461 if (msg->any.head.cmd & DMSGF_DELETE) { 462 if (iost && iost->count == 0) { 463 kfree(iost, M_DEVBUF); 464 msg->state->any.any = NULL; 465 } 466 kdmsg_msg_reply(msg, error); 467 } else { 468 kdmsg_msg_result(msg, error); 469 } 470 } 471 } 472 473 static 474 void 475 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg) 476 { 477 struct dios_io *iost; 478 struct buf *bp; 479 struct bio *bio; 480 int error = DMSG_ERR_NOSUPP; 481 int reterr = 1; 482 483 /* 484 * Only DMSG_BLK_FLUSH commands imply read ops. 485 */ 486 iost = msg->state->any.any; 487 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) { 488 if (iost == NULL) { 489 iost = kmalloc(sizeof(*iost), M_DEVBUF, 490 M_WAITOK | M_ZERO); 491 msg->state->any.any = iost; 492 } 493 reterr = 0; 494 bp = getpbuf(NULL); 495 bio = &bp->b_bio1; 496 bp->b_cmd = BUF_CMD_FLUSH; 497 bp->b_bcount = msg->any.blk_flush.bytes; 498 bp->b_resid = 0; 499 bio->bio_offset = msg->any.blk_flush.offset; 500 bio->bio_caller_info1.ptr = msg->state; 501 bio->bio_done = diskiodone; 502 503 /* kdmsg_state_hold(msg->state); */ 504 atomic_add_int(&blk_active, 1); 505 atomic_add_int(&iost->count, 1); 506 if (msg->any.head.cmd & DMSGF_DELETE) 507 iost->eof = 1; 508 BUF_KERNPROC(bp); 509 dev_dstrategy(dp->d_rawdev, bio); 510 } 511 if (reterr) { 512 if (msg->any.head.cmd & DMSGF_DELETE) { 513 if (iost && iost->count == 0) { 514 kfree(iost, M_DEVBUF); 515 msg->state->any.any = NULL; 516 } 517 kdmsg_msg_reply(msg, error); 518 } else { 519 kdmsg_msg_result(msg, error); 520 } 521 } 522 } 523 524 static 525 void 526 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg) 527 { 528 struct dios_io *iost; 529 struct buf *bp; 530 struct bio *bio; 531 int error = DMSG_ERR_NOSUPP; 532 int reterr = 1; 533 534 /* 535 * Only DMSG_BLK_FREEBLKS commands imply read ops. 536 */ 537 iost = msg->state->any.any; 538 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) { 539 if (iost == NULL) { 540 iost = kmalloc(sizeof(*iost), M_DEVBUF, 541 M_WAITOK | M_ZERO); 542 msg->state->any.any = iost; 543 } 544 reterr = 0; 545 bp = getpbuf(NULL); 546 bio = &bp->b_bio1; 547 bp->b_cmd = BUF_CMD_FREEBLKS; 548 bp->b_bcount = msg->any.blk_freeblks.bytes; 549 bp->b_resid = 0; 550 bio->bio_offset = msg->any.blk_freeblks.offset; 551 bio->bio_caller_info1.ptr = msg->state; 552 bio->bio_done = diskiodone; 553 554 /* kdmsg_state_hold(msg->state); */ 555 atomic_add_int(&blk_active, 1); 556 atomic_add_int(&iost->count, 1); 557 if (msg->any.head.cmd & DMSGF_DELETE) 558 iost->eof = 1; 559 BUF_KERNPROC(bp); 560 dev_dstrategy(dp->d_rawdev, bio); 561 } 562 if (reterr) { 563 if (msg->any.head.cmd & DMSGF_DELETE) { 564 if (iost && iost->count == 0) { 565 kfree(iost, M_DEVBUF); 566 msg->state->any.any = NULL; 567 } 568 kdmsg_msg_reply(msg, error); 569 } else { 570 kdmsg_msg_result(msg, error); 571 } 572 } 573 } 574 575 static 576 void 577 diskiodone(struct bio *bio) 578 { 579 struct buf *bp = bio->bio_buf; 580 kdmsg_state_t *state = bio->bio_caller_info1.ptr; 581 kdmsg_msg_t *rmsg; 582 struct dios_io *iost = state->any.any; 583 int error; 584 int resid = 0; 585 int bytes; 586 uint32_t cmd; 587 void *data; 588 589 cmd = DMSG_LNK_ERROR; 590 data = NULL; 591 bytes = 0; 592 593 switch(bp->b_cmd) { 594 case BUF_CMD_READ: 595 cmd = DMSG_LNK_ERROR; 596 data = bp->b_data; 597 bytes = bp->b_bcount; 598 /* fall through */ 599 case BUF_CMD_WRITE: 600 if (bp->b_flags & B_ERROR) { 601 error = bp->b_error; 602 } else { 603 error = 0; 604 resid = bp->b_resid; 605 } 606 kdmsg_free_aux_data(&iost->data); 607 break; 608 case BUF_CMD_FLUSH: 609 case BUF_CMD_FREEBLKS: 610 if (bp->b_flags & B_ERROR) 611 error = bp->b_error; 612 else 613 error = 0; 614 break; 615 default: 616 panic("diskiodone: Unknown bio cmd = %d\n", 617 bio->bio_buf->b_cmd); 618 error = 0; /* avoid compiler warning */ 619 break; /* NOT REACHED */ 620 } 621 622 /* 623 * Convert error to DMSG_ERR_* code. 624 */ 625 if (error) 626 error = DMSG_ERR_IO; 627 628 /* 629 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will 630 * have already converted cmd to BLK_ERROR and set up data to return. 631 */ 632 if (resid && cmd == DMSG_LNK_ERROR) 633 cmd = DMSG_BLK_ERROR; 634 /* XXX txcmd is delayed so this won't work for streaming */ 635 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */ 636 cmd |= DMSGF_CREATE; 637 if (iost->eof) { 638 if (atomic_fetchadd_int(&iost->count, -1) == 1) 639 cmd |= DMSGF_DELETE; 640 } else { 641 atomic_add_int(&iost->count, -1); 642 } 643 atomic_add_int(&blk_active, -1); 644 cmd |= DMSGF_REPLY; 645 646 /* 647 * Allocate a basic or extended reply. Be careful not to populate 648 * extended header fields unless we allocated an extended reply. 649 */ 650 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0); 651 if (data) { 652 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT); 653 rmsg->aux_size = bytes; 654 rmsg->flags |= KDMSG_FLAG_AUXALLOC; 655 bcopy(data, rmsg->aux_data, bytes); 656 } 657 rmsg->any.blk_error.head.error = error; 658 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR) 659 rmsg->any.blk_error.resid = resid; 660 bio->bio_caller_info1.ptr = NULL; 661 /* kdmsg_state_drop(state); */ 662 kdmsg_msg_write(rmsg); 663 if (bp->b_flags & B_PAGING) { 664 relpbuf(bp, NULL); 665 } else { 666 bp->b_flags |= B_INVAL | B_AGE; 667 relpbuf(bp, NULL); 668 } 669 } 670