1 /*- 2 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD: head/sys/dev/virtio/scsi/virtio_scsi.c 311305 2017-01-04 20:26:42Z asomers $ 27 */ 28 29 /* Driver for VirtIO SCSI devices. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/kthread.h> 35 #include <sys/malloc.h> 36 #include <sys/module.h> 37 #include <sys/sglist.h> 38 #include <sys/sysctl.h> 39 #include <sys/lock.h> 40 #include <sys/callout.h> 41 #include <sys/queue.h> 42 #include <sys/sbuf.h> 43 44 #include <machine/stdarg.h> 45 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 49 #include <bus/cam/cam.h> 50 #include <bus/cam/cam_ccb.h> 51 #include <bus/cam/cam_sim.h> 52 #include <bus/cam/cam_periph.h> 53 #include <bus/cam/cam_xpt_periph.h> 54 #include <bus/cam/cam_xpt_sim.h> 55 #include <bus/cam/cam_debug.h> 56 #include <bus/cam/scsi/scsi_all.h> 57 #include <bus/cam/scsi/scsi_message.h> 58 59 #include <dev/virtual/virtio/virtio/virtio.h> 60 #include <dev/virtual/virtio/virtio/virtqueue.h> 61 #include <dev/virtual/virtio/scsi/virtio_scsi.h> 62 #include <dev/virtual/virtio/scsi/virtio_scsivar.h> 63 64 #include "virtio_if.h" 65 66 static int vtscsi_modevent(module_t, int, void *); 67 68 static int vtscsi_probe(device_t); 69 static int vtscsi_attach(device_t); 70 static int vtscsi_detach(device_t); 71 static int vtscsi_suspend(device_t); 72 static int vtscsi_resume(device_t); 73 74 static void vtscsi_negotiate_features(struct vtscsi_softc *); 75 static void vtscsi_read_config(struct vtscsi_softc *, 76 struct virtio_scsi_config *); 77 static int vtscsi_maximum_segments(struct vtscsi_softc *, int); 78 static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); 79 static void vtscsi_write_device_config(struct vtscsi_softc *); 80 static int vtscsi_reinit(struct vtscsi_softc *); 81 82 static int vtscsi_alloc_cam(struct vtscsi_softc *); 83 static int vtscsi_register_cam(struct vtscsi_softc *); 84 static void vtscsi_free_cam(struct vtscsi_softc *); 85 static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); 86 static int vtscsi_register_async(struct vtscsi_softc *); 87 static void vtscsi_deregister_async(struct vtscsi_softc *); 88 static void vtscsi_cam_action(struct cam_sim *, union ccb *); 89 static void vtscsi_cam_poll(struct cam_sim *); 90 91 static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, 92 union ccb *); 93 static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, 94 union ccb *); 95 static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); 96 static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); 97 static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); 98 static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, 99 struct cam_sim *, union ccb *); 100 101 static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, 102 struct sglist *, struct ccb_scsiio *); 103 static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, 104 struct vtscsi_request *, int *, int *); 105 static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, 106 struct vtscsi_request *); 107 static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); 108 static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, 109 struct vtscsi_request *); 110 static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, 111 struct vtscsi_request *); 112 static void vtscsi_timedout_scsi_cmd(void *); 113 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); 114 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, 115 struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); 116 static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, 117 struct vtscsi_request *); 118 119 static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, 120 struct vtscsi_request *); 121 static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, 122 struct vtscsi_request *, struct sglist *, int, int, int); 123 static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, 124 struct vtscsi_request *); 125 static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, 126 struct vtscsi_request *); 127 static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, 128 struct vtscsi_request *); 129 130 static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); 131 static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); 132 static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, 133 struct virtio_scsi_cmd_req *); 134 static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, 135 uintptr_t, struct virtio_scsi_ctrl_tmf_req *); 136 137 static void vtscsi_freeze_simq(struct vtscsi_softc *, int); 138 static int vtscsi_thaw_simq(struct vtscsi_softc *, int); 139 140 static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, 141 lun_id_t); 142 static void vtscsi_cam_rescan_callback(struct cam_periph *periph, 143 union ccb *ccb); 144 static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, 145 lun_id_t); 146 static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); 147 148 static void vtscsi_handle_event(struct vtscsi_softc *, 149 struct virtio_scsi_event *); 150 static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, 151 struct virtio_scsi_event *); 152 static int vtscsi_init_event_vq(struct vtscsi_softc *); 153 static void vtscsi_reinit_event_vq(struct vtscsi_softc *); 154 static void vtscsi_drain_event_vq(struct vtscsi_softc *); 155 156 static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); 157 static void vtscsi_complete_vqs(struct vtscsi_softc *); 158 static void vtscsi_drain_vqs(struct vtscsi_softc *); 159 static void vtscsi_cancel_request(struct vtscsi_softc *, 160 struct vtscsi_request *); 161 static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); 162 static void vtscsi_stop(struct vtscsi_softc *); 163 static int vtscsi_reset_bus(struct vtscsi_softc *); 164 165 static void vtscsi_init_request(struct vtscsi_softc *, 166 struct vtscsi_request *); 167 static int vtscsi_alloc_requests(struct vtscsi_softc *); 168 static void vtscsi_free_requests(struct vtscsi_softc *); 169 static void vtscsi_enqueue_request(struct vtscsi_softc *, 170 struct vtscsi_request *); 171 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); 172 173 static void vtscsi_complete_request(struct vtscsi_request *); 174 static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); 175 176 static int vtscsi_control_vq_intr(void *); 177 static int vtscsi_event_vq_intr(void *); 178 static int vtscsi_request_vq_intr(void *); 179 static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); 180 static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); 181 182 static void vtscsi_get_tunables(struct vtscsi_softc *); 183 static void vtscsi_add_sysctl(struct vtscsi_softc *); 184 185 static void vtscsi_printf_req(struct vtscsi_request *, const char *, 186 const char *, ...); 187 188 /* Global tunables. */ 189 /* 190 * The current QEMU VirtIO SCSI implementation does not cancel in-flight 191 * IO during virtio_stop(). So in-flight requests still complete after the 192 * device reset. We would have to wait for all the in-flight IO to complete, 193 * which defeats the typical purpose of a bus reset. We could simulate the 194 * bus reset with either I_T_NEXUS_RESET of all the targets, or with 195 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the 196 * control virtqueue). But this isn't very useful if things really go off 197 * the rails, so default to disabled for now. 198 */ 199 static int vtscsi_bus_reset_disable = 1; 200 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); 201 202 static struct virtio_feature_desc vtscsi_feature_desc[] = { 203 { VIRTIO_SCSI_F_INOUT, "InOut" }, 204 { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, 205 206 { 0, NULL } 207 }; 208 209 static device_method_t vtscsi_methods[] = { 210 /* Device methods. */ 211 DEVMETHOD(device_probe, vtscsi_probe), 212 DEVMETHOD(device_attach, vtscsi_attach), 213 DEVMETHOD(device_detach, vtscsi_detach), 214 DEVMETHOD(device_suspend, vtscsi_suspend), 215 DEVMETHOD(device_resume, vtscsi_resume), 216 217 DEVMETHOD_END 218 }; 219 220 static driver_t vtscsi_driver = { 221 "vtscsi", 222 vtscsi_methods, 223 sizeof(struct vtscsi_softc) 224 }; 225 static devclass_t vtscsi_devclass; 226 227 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, 228 vtscsi_modevent, NULL); 229 MODULE_VERSION(virtio_scsi, 1); 230 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); 231 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); 232 233 static int 234 vtscsi_modevent(module_t mod, int type, void *unused) 235 { 236 int error; 237 238 switch (type) { 239 case MOD_LOAD: 240 case MOD_UNLOAD: 241 case MOD_SHUTDOWN: 242 error = 0; 243 break; 244 default: 245 error = EOPNOTSUPP; 246 break; 247 } 248 249 return (error); 250 } 251 252 static int 253 vtscsi_probe(device_t dev) 254 { 255 256 if (virtio_get_device_type(dev) != VIRTIO_ID_SCSI) 257 return (ENXIO); 258 259 device_set_desc(dev, "VirtIO SCSI Adapter"); 260 261 return (BUS_PROBE_DEFAULT); 262 } 263 264 static int 265 vtscsi_attach(device_t dev) 266 { 267 struct vtscsi_softc *sc; 268 struct virtio_scsi_config scsicfg; 269 int error; 270 271 sc = device_get_softc(dev); 272 sc->vtscsi_dev = dev; 273 274 VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); 275 TAILQ_INIT(&sc->vtscsi_req_free); 276 277 vtscsi_get_tunables(sc); 278 vtscsi_add_sysctl(sc); 279 280 virtio_set_feature_desc(dev, vtscsi_feature_desc); 281 vtscsi_negotiate_features(sc); 282 283 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 284 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; 285 #ifndef __DragonFly__ /* XXX swildner */ 286 if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) 287 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; 288 #endif 289 if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) 290 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; 291 292 vtscsi_read_config(sc, &scsicfg); 293 294 sc->vtscsi_max_channel = scsicfg.max_channel; 295 sc->vtscsi_max_target = scsicfg.max_target; 296 sc->vtscsi_max_lun = scsicfg.max_lun; 297 sc->vtscsi_event_buf_size = scsicfg.event_info_size; 298 299 vtscsi_write_device_config(sc); 300 301 sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); 302 sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); 303 if (sc->vtscsi_sglist == NULL) { 304 error = ENOMEM; 305 device_printf(dev, "cannot allocate sglist\n"); 306 goto fail; 307 } 308 309 error = vtscsi_alloc_virtqueues(sc); 310 if (error) { 311 device_printf(dev, "cannot allocate virtqueues\n"); 312 goto fail; 313 } 314 315 error = vtscsi_init_event_vq(sc); 316 if (error) { 317 device_printf(dev, "cannot populate the eventvq\n"); 318 goto fail; 319 } 320 321 error = vtscsi_alloc_requests(sc); 322 if (error) { 323 device_printf(dev, "cannot allocate requests\n"); 324 goto fail; 325 } 326 327 error = vtscsi_alloc_cam(sc); 328 if (error) { 329 device_printf(dev, "cannot allocate CAM structures\n"); 330 goto fail; 331 } 332 333 error = virtio_setup_intr(dev, NULL); 334 if (error) { 335 device_printf(dev, "cannot setup virtqueue interrupts\n"); 336 goto fail; 337 } 338 339 vtscsi_enable_vqs_intr(sc); 340 341 /* 342 * Register with CAM after interrupts are enabled so we will get 343 * notified of the probe responses. 344 */ 345 error = vtscsi_register_cam(sc); 346 if (error) { 347 device_printf(dev, "cannot register with CAM\n"); 348 goto fail; 349 } 350 351 fail: 352 if (error) 353 vtscsi_detach(dev); 354 355 return (error); 356 } 357 358 static int 359 vtscsi_detach(device_t dev) 360 { 361 struct vtscsi_softc *sc; 362 363 sc = device_get_softc(dev); 364 365 VTSCSI_LOCK(sc); 366 sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; 367 if (device_is_attached(dev)) 368 vtscsi_stop(sc); 369 VTSCSI_UNLOCK(sc); 370 371 vtscsi_complete_vqs(sc); 372 vtscsi_drain_vqs(sc); 373 374 vtscsi_free_cam(sc); 375 vtscsi_free_requests(sc); 376 377 if (sc->vtscsi_sglist != NULL) { 378 sglist_free(sc->vtscsi_sglist); 379 sc->vtscsi_sglist = NULL; 380 } 381 382 VTSCSI_LOCK_DESTROY(sc); 383 384 return (0); 385 } 386 387 static int 388 vtscsi_suspend(device_t dev) 389 { 390 391 return (0); 392 } 393 394 static int 395 vtscsi_resume(device_t dev) 396 { 397 398 return (0); 399 } 400 401 static void 402 vtscsi_negotiate_features(struct vtscsi_softc *sc) 403 { 404 device_t dev; 405 uint64_t features; 406 407 dev = sc->vtscsi_dev; 408 features = virtio_negotiate_features(dev, VTSCSI_FEATURES); 409 sc->vtscsi_features = features; 410 } 411 412 #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ 413 virtio_read_device_config(_dev, \ 414 offsetof(struct virtio_scsi_config, _field), \ 415 &(_cfg)->_field, sizeof((_cfg)->_field)) \ 416 417 static void 418 vtscsi_read_config(struct vtscsi_softc *sc, 419 struct virtio_scsi_config *scsicfg) 420 { 421 device_t dev; 422 423 dev = sc->vtscsi_dev; 424 425 bzero(scsicfg, sizeof(struct virtio_scsi_config)); 426 427 VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); 428 VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); 429 VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); 430 VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); 431 VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); 432 VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); 433 VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); 434 VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); 435 VTSCSI_GET_CONFIG(dev, max_target, scsicfg); 436 VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); 437 } 438 439 #undef VTSCSI_GET_CONFIG 440 441 static int 442 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) 443 { 444 int nsegs; 445 446 nsegs = VTSCSI_MIN_SEGMENTS; 447 448 if (seg_max > 0) { 449 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1); 450 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) 451 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 452 } else 453 nsegs += 1; 454 455 return (nsegs); 456 } 457 458 static int 459 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) 460 { 461 device_t dev; 462 struct vq_alloc_info vq_info[3]; 463 int nvqs; 464 465 dev = sc->vtscsi_dev; 466 nvqs = 3; 467 468 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, 469 &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); 470 471 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, 472 &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); 473 474 VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, 475 vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, 476 "%s request", device_get_nameunit(dev)); 477 478 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 479 } 480 481 static void 482 vtscsi_write_device_config(struct vtscsi_softc *sc) 483 { 484 485 virtio_write_dev_config_4(sc->vtscsi_dev, 486 offsetof(struct virtio_scsi_config, sense_size), 487 VIRTIO_SCSI_SENSE_SIZE); 488 489 /* 490 * This is the size in the virtio_scsi_cmd_req structure. Note 491 * this value (32) is larger than the maximum CAM CDB size (16). 492 */ 493 virtio_write_dev_config_4(sc->vtscsi_dev, 494 offsetof(struct virtio_scsi_config, cdb_size), 495 VIRTIO_SCSI_CDB_SIZE); 496 } 497 498 static int 499 vtscsi_reinit(struct vtscsi_softc *sc) 500 { 501 device_t dev; 502 int error; 503 504 dev = sc->vtscsi_dev; 505 506 error = virtio_reinit(dev, sc->vtscsi_features); 507 if (error == 0) { 508 vtscsi_write_device_config(sc); 509 vtscsi_reinit_event_vq(sc); 510 virtio_reinit_complete(dev); 511 512 vtscsi_enable_vqs_intr(sc); 513 } 514 515 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); 516 517 return (error); 518 } 519 520 static int 521 vtscsi_alloc_cam(struct vtscsi_softc *sc) 522 { 523 device_t dev; 524 struct cam_devq *devq; 525 int openings; 526 527 dev = sc->vtscsi_dev; 528 openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; 529 530 devq = cam_simq_alloc(openings); 531 if (devq == NULL) { 532 device_printf(dev, "cannot allocate SIM queue\n"); 533 return (ENOMEM); 534 } 535 536 sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, 537 "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, 538 openings, devq); 539 cam_simq_release(devq); 540 if (sc->vtscsi_sim == NULL) { 541 device_printf(dev, "cannot allocate SIM\n"); 542 return (ENOMEM); 543 } 544 545 return (0); 546 } 547 548 static int 549 vtscsi_register_cam(struct vtscsi_softc *sc) 550 { 551 device_t dev; 552 int registered, error; 553 554 dev = sc->vtscsi_dev; 555 registered = 0; 556 557 VTSCSI_LOCK(sc); 558 559 if (xpt_bus_register(sc->vtscsi_sim, 0) != CAM_SUCCESS) { 560 error = ENOMEM; 561 device_printf(dev, "cannot register XPT bus\n"); 562 goto fail; 563 } 564 565 registered = 1; 566 567 if (xpt_create_path(&sc->vtscsi_path, NULL, 568 cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, 569 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 570 error = ENOMEM; 571 device_printf(dev, "cannot create bus path\n"); 572 goto fail; 573 } 574 575 if (vtscsi_register_async(sc) != CAM_REQ_CMP) { 576 error = EIO; 577 device_printf(dev, "cannot register async callback\n"); 578 goto fail; 579 } 580 581 VTSCSI_UNLOCK(sc); 582 583 return (0); 584 585 fail: 586 if (sc->vtscsi_path != NULL) { 587 xpt_free_path(sc->vtscsi_path); 588 sc->vtscsi_path = NULL; 589 } 590 591 if (registered != 0) 592 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 593 594 VTSCSI_UNLOCK(sc); 595 596 return (error); 597 } 598 599 static void 600 vtscsi_free_cam(struct vtscsi_softc *sc) 601 { 602 603 VTSCSI_LOCK(sc); 604 605 if (sc->vtscsi_path != NULL) { 606 vtscsi_deregister_async(sc); 607 608 xpt_free_path(sc->vtscsi_path); 609 sc->vtscsi_path = NULL; 610 611 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 612 } 613 614 if (sc->vtscsi_sim != NULL) { 615 cam_sim_free(sc->vtscsi_sim); 616 sc->vtscsi_sim = NULL; 617 } 618 619 VTSCSI_UNLOCK(sc); 620 } 621 622 static void 623 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) 624 { 625 struct cam_sim *sim; 626 struct vtscsi_softc *sc; 627 628 sim = cb_arg; 629 sc = cam_sim_softc(sim); 630 631 vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); 632 633 /* 634 * TODO Once QEMU supports event reporting, we should 635 * (un)subscribe to events here. 636 */ 637 switch (code) { 638 case AC_FOUND_DEVICE: 639 break; 640 case AC_LOST_DEVICE: 641 break; 642 } 643 } 644 645 static int 646 vtscsi_register_async(struct vtscsi_softc *sc) 647 { 648 struct ccb_setasync csa; 649 650 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 651 csa.ccb_h.func_code = XPT_SASYNC_CB; 652 csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; 653 csa.callback = vtscsi_cam_async; 654 csa.callback_arg = sc->vtscsi_sim; 655 656 xpt_action((union ccb *) &csa); 657 658 return (csa.ccb_h.status); 659 } 660 661 static void 662 vtscsi_deregister_async(struct vtscsi_softc *sc) 663 { 664 struct ccb_setasync csa; 665 666 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 667 csa.ccb_h.func_code = XPT_SASYNC_CB; 668 csa.event_enable = 0; 669 csa.callback = vtscsi_cam_async; 670 csa.callback_arg = sc->vtscsi_sim; 671 672 xpt_action((union ccb *) &csa); 673 } 674 675 static void 676 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) 677 { 678 struct vtscsi_softc *sc; 679 struct ccb_hdr *ccbh; 680 681 sc = cam_sim_softc(sim); 682 ccbh = &ccb->ccb_h; 683 684 VTSCSI_LOCK_OWNED(sc); 685 686 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { 687 /* 688 * The VTSCSI_MTX is briefly dropped between setting 689 * VTSCSI_FLAG_DETACH and deregistering with CAM, so 690 * drop any CCBs that come in during that window. 691 */ 692 ccbh->status = CAM_NO_HBA; 693 xpt_done(ccb); 694 return; 695 } 696 697 switch (ccbh->func_code) { 698 case XPT_SCSI_IO: 699 vtscsi_cam_scsi_io(sc, sim, ccb); 700 break; 701 702 case XPT_SET_TRAN_SETTINGS: 703 ccbh->status = CAM_FUNC_NOTAVAIL; 704 xpt_done(ccb); 705 break; 706 707 case XPT_GET_TRAN_SETTINGS: 708 vtscsi_cam_get_tran_settings(sc, ccb); 709 break; 710 711 case XPT_RESET_BUS: 712 vtscsi_cam_reset_bus(sc, ccb); 713 break; 714 715 case XPT_RESET_DEV: 716 vtscsi_cam_reset_dev(sc, ccb); 717 break; 718 719 case XPT_ABORT: 720 vtscsi_cam_abort(sc, ccb); 721 break; 722 723 case XPT_CALC_GEOMETRY: 724 cam_calc_geometry(&ccb->ccg, 1); 725 xpt_done(ccb); 726 break; 727 728 case XPT_PATH_INQ: 729 vtscsi_cam_path_inquiry(sc, sim, ccb); 730 break; 731 732 default: 733 vtscsi_dprintf(sc, VTSCSI_ERROR, 734 "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); 735 736 ccbh->status = CAM_REQ_INVALID; 737 xpt_done(ccb); 738 break; 739 } 740 } 741 742 static void 743 vtscsi_cam_poll(struct cam_sim *sim) 744 { 745 struct vtscsi_softc *sc; 746 747 sc = cam_sim_softc(sim); 748 749 vtscsi_complete_vqs_locked(sc); 750 } 751 752 static void 753 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, 754 union ccb *ccb) 755 { 756 struct ccb_hdr *ccbh; 757 struct ccb_scsiio *csio; 758 int error; 759 760 ccbh = &ccb->ccb_h; 761 csio = &ccb->csio; 762 763 if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { 764 error = EINVAL; 765 ccbh->status = CAM_REQ_INVALID; 766 goto done; 767 } 768 769 #ifndef __DragonFly__ /* XXX swildner */ 770 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && 771 (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { 772 error = EINVAL; 773 ccbh->status = CAM_REQ_INVALID; 774 goto done; 775 } 776 #endif 777 778 error = vtscsi_start_scsi_cmd(sc, ccb); 779 780 done: 781 if (error) { 782 vtscsi_dprintf(sc, VTSCSI_ERROR, 783 "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); 784 xpt_done(ccb); 785 } 786 } 787 788 static void 789 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) 790 { 791 struct ccb_trans_settings *cts; 792 struct ccb_trans_settings_scsi *scsi; 793 794 cts = &ccb->cts; 795 scsi = &cts->proto_specific.scsi; 796 797 cts->protocol = PROTO_SCSI; 798 cts->protocol_version = SCSI_REV_SPC3; 799 cts->transport = XPORT_SAS; 800 cts->transport_version = 0; 801 802 scsi->valid = CTS_SCSI_VALID_TQ; 803 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 804 805 ccb->ccb_h.status = CAM_REQ_CMP; 806 xpt_done(ccb); 807 } 808 809 static void 810 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) 811 { 812 int error; 813 814 error = vtscsi_reset_bus(sc); 815 if (error == 0) 816 ccb->ccb_h.status = CAM_REQ_CMP; 817 else 818 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 819 820 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", 821 error, ccb, ccb->ccb_h.status); 822 823 xpt_done(ccb); 824 } 825 826 static void 827 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) 828 { 829 struct ccb_hdr *ccbh; 830 struct vtscsi_request *req; 831 int error; 832 833 ccbh = &ccb->ccb_h; 834 835 req = vtscsi_dequeue_request(sc); 836 if (req == NULL) { 837 error = EAGAIN; 838 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 839 goto fail; 840 } 841 842 req->vsr_ccb = ccb; 843 844 error = vtscsi_execute_reset_dev_cmd(sc, req); 845 if (error == 0) 846 return; 847 848 vtscsi_enqueue_request(sc, req); 849 850 fail: 851 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 852 error, req, ccb); 853 854 if (error == EAGAIN) 855 ccbh->status = CAM_RESRC_UNAVAIL; 856 else 857 ccbh->status = CAM_REQ_CMP_ERR; 858 859 xpt_done(ccb); 860 } 861 862 static void 863 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) 864 { 865 struct vtscsi_request *req; 866 struct ccb_hdr *ccbh; 867 int error; 868 869 ccbh = &ccb->ccb_h; 870 871 req = vtscsi_dequeue_request(sc); 872 if (req == NULL) { 873 error = EAGAIN; 874 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 875 goto fail; 876 } 877 878 req->vsr_ccb = ccb; 879 880 error = vtscsi_execute_abort_task_cmd(sc, req); 881 if (error == 0) 882 return; 883 884 vtscsi_enqueue_request(sc, req); 885 886 fail: 887 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 888 error, req, ccb); 889 890 if (error == EAGAIN) 891 ccbh->status = CAM_RESRC_UNAVAIL; 892 else 893 ccbh->status = CAM_REQ_CMP_ERR; 894 895 xpt_done(ccb); 896 } 897 898 static void 899 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, 900 union ccb *ccb) 901 { 902 device_t dev; 903 struct ccb_pathinq *cpi; 904 905 dev = sc->vtscsi_dev; 906 cpi = &ccb->cpi; 907 908 vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); 909 910 cpi->version_num = 1; 911 cpi->hba_inquiry = PI_TAG_ABLE; 912 cpi->target_sprt = 0; 913 cpi->hba_misc = PIM_SEQSCAN; 914 if (vtscsi_bus_reset_disable != 0) 915 cpi->hba_misc |= PIM_NOBUSRESET; 916 cpi->hba_eng_cnt = 0; 917 918 cpi->max_target = sc->vtscsi_max_target; 919 cpi->max_lun = sc->vtscsi_max_lun; 920 cpi->initiator_id = VTSCSI_INITIATOR_ID; 921 922 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 923 strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); 924 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 925 926 cpi->unit_number = cam_sim_unit(sim); 927 cpi->bus_id = cam_sim_bus(sim); 928 929 cpi->base_transfer_speed = 300000; 930 931 cpi->protocol = PROTO_SCSI; 932 cpi->protocol_version = SCSI_REV_SPC3; 933 cpi->transport = XPORT_SAS; 934 cpi->transport_version = 0; 935 936 cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * 937 PAGE_SIZE; 938 939 #if 0 940 cpi->hba_vendor = virtio_get_vendor(dev); 941 cpi->hba_device = virtio_get_device(dev); 942 cpi->hba_subvendor = virtio_get_subvendor(dev); 943 cpi->hba_subdevice = virtio_get_subdevice(dev); 944 #endif 945 946 ccb->ccb_h.status = CAM_REQ_CMP; 947 xpt_done(ccb); 948 } 949 950 static int 951 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, 952 struct ccb_scsiio *csio) 953 { 954 struct ccb_hdr *ccbh; 955 struct bus_dma_segment *dseg; 956 int i, error; 957 958 ccbh = &csio->ccb_h; 959 error = 0; 960 961 if ((ccbh->flags & CAM_SCATTER_VALID) == 0) { 962 963 if ((ccbh->flags & CAM_DATA_PHYS) == 0) 964 error = sglist_append(sg, 965 csio->data_ptr, csio->dxfer_len); 966 else 967 error = sglist_append_phys(sg, 968 (vm_paddr_t)(vm_offset_t) csio->data_ptr, 969 csio->dxfer_len); 970 } else { 971 972 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 973 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 974 975 if ((ccbh->flags & CAM_SG_LIST_PHYS) == 0) 976 error = sglist_append(sg, 977 (void *)(vm_offset_t) dseg->ds_addr, 978 dseg->ds_len); 979 else 980 error = sglist_append_phys(sg, 981 (vm_paddr_t) dseg->ds_addr, dseg->ds_len); 982 } 983 } 984 985 return (error); 986 } 987 988 static int 989 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, 990 int *readable, int *writable) 991 { 992 struct sglist *sg; 993 struct ccb_hdr *ccbh; 994 struct ccb_scsiio *csio; 995 struct virtio_scsi_cmd_req *cmd_req; 996 struct virtio_scsi_cmd_resp *cmd_resp; 997 int error; 998 999 sg = sc->vtscsi_sglist; 1000 csio = &req->vsr_ccb->csio; 1001 ccbh = &csio->ccb_h; 1002 cmd_req = &req->vsr_cmd_req; 1003 cmd_resp = &req->vsr_cmd_resp; 1004 1005 sglist_reset(sg); 1006 1007 sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); 1008 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { 1009 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1010 /* At least one segment must be left for the response. */ 1011 if (error || sg->sg_nseg == sg->sg_maxseg) 1012 goto fail; 1013 } 1014 1015 *readable = sg->sg_nseg; 1016 1017 sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); 1018 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { 1019 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1020 if (error) 1021 goto fail; 1022 } 1023 1024 *writable = sg->sg_nseg - *readable; 1025 1026 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " 1027 "writable=%d\n", req, ccbh, *readable, *writable); 1028 1029 return (0); 1030 1031 fail: 1032 /* 1033 * This should never happen unless maxio was incorrectly set. 1034 */ 1035 vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); 1036 1037 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " 1038 "nseg=%d maxseg=%d\n", 1039 error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); 1040 1041 return (EFBIG); 1042 } 1043 1044 static int 1045 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1046 { 1047 struct sglist *sg; 1048 struct virtqueue *vq; 1049 struct ccb_scsiio *csio; 1050 struct ccb_hdr *ccbh; 1051 struct virtio_scsi_cmd_req *cmd_req; 1052 struct virtio_scsi_cmd_resp *cmd_resp; 1053 int readable, writable, error; 1054 1055 sg = sc->vtscsi_sglist; 1056 vq = sc->vtscsi_request_vq; 1057 csio = &req->vsr_ccb->csio; 1058 ccbh = &csio->ccb_h; 1059 cmd_req = &req->vsr_cmd_req; 1060 cmd_resp = &req->vsr_cmd_resp; 1061 1062 vtscsi_init_scsi_cmd_req(csio, cmd_req); 1063 1064 error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); 1065 if (error) 1066 return (error); 1067 1068 req->vsr_complete = vtscsi_complete_scsi_cmd; 1069 cmd_resp->response = -1; 1070 1071 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1072 if (error) { 1073 vtscsi_dprintf(sc, VTSCSI_ERROR, 1074 "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); 1075 1076 ccbh->status = CAM_REQUEUE_REQ; 1077 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); 1078 return (error); 1079 } 1080 1081 ccbh->status |= CAM_SIM_QUEUED; 1082 ccbh->ccbh_vtscsi_req = req; 1083 1084 virtqueue_notify(vq, NULL); 1085 1086 if (ccbh->timeout != CAM_TIME_INFINITY) { 1087 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; 1088 callout_reset(&req->vsr_callout, ccbh->timeout * hz / 1000, 1089 vtscsi_timedout_scsi_cmd, req); 1090 } 1091 1092 vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", 1093 req, ccbh); 1094 1095 return (0); 1096 } 1097 1098 static int 1099 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) 1100 { 1101 struct vtscsi_request *req; 1102 int error; 1103 1104 req = vtscsi_dequeue_request(sc); 1105 if (req == NULL) { 1106 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1107 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 1108 return (ENOBUFS); 1109 } 1110 1111 req->vsr_ccb = ccb; 1112 1113 error = vtscsi_execute_scsi_cmd(sc, req); 1114 if (error) 1115 vtscsi_enqueue_request(sc, req); 1116 1117 return (error); 1118 } 1119 1120 static void 1121 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1122 struct vtscsi_request *req) 1123 { 1124 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1125 struct vtscsi_request *to_req; 1126 uint8_t response; 1127 1128 tmf_resp = &req->vsr_tmf_resp; 1129 response = tmf_resp->response; 1130 to_req = req->vsr_timedout_req; 1131 1132 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", 1133 req, to_req, response); 1134 1135 vtscsi_enqueue_request(sc, req); 1136 1137 /* 1138 * The timedout request could have completed between when the 1139 * abort task was sent and when the host processed it. 1140 */ 1141 if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) 1142 return; 1143 1144 /* The timedout request was successfully aborted. */ 1145 if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) 1146 return; 1147 1148 /* Don't bother if the device is going away. */ 1149 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1150 return; 1151 1152 /* The timedout request will be aborted by the reset. */ 1153 if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) 1154 return; 1155 1156 vtscsi_reset_bus(sc); 1157 } 1158 1159 static int 1160 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1161 struct vtscsi_request *to_req) 1162 { 1163 struct sglist *sg; 1164 struct ccb_hdr *to_ccbh; 1165 struct vtscsi_request *req; 1166 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1167 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1168 int error; 1169 1170 sg = sc->vtscsi_sglist; 1171 to_ccbh = &to_req->vsr_ccb->ccb_h; 1172 1173 req = vtscsi_dequeue_request(sc); 1174 if (req == NULL) { 1175 error = ENOBUFS; 1176 goto fail; 1177 } 1178 1179 tmf_req = &req->vsr_tmf_req; 1180 tmf_resp = &req->vsr_tmf_resp; 1181 1182 vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1183 (uintptr_t) to_ccbh, tmf_req); 1184 1185 sglist_reset(sg); 1186 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1187 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1188 1189 req->vsr_timedout_req = to_req; 1190 req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; 1191 tmf_resp->response = -1; 1192 1193 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1194 VTSCSI_EXECUTE_ASYNC); 1195 if (error == 0) 1196 return (0); 1197 1198 vtscsi_enqueue_request(sc, req); 1199 1200 fail: 1201 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " 1202 "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); 1203 1204 return (error); 1205 } 1206 1207 static void 1208 vtscsi_timedout_scsi_cmd(void *xreq) 1209 { 1210 struct vtscsi_softc *sc; 1211 struct vtscsi_request *to_req; 1212 1213 to_req = xreq; 1214 sc = to_req->vsr_softc; 1215 1216 vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", 1217 to_req, to_req->vsr_ccb, to_req->vsr_state); 1218 1219 /* Don't bother if the device is going away. */ 1220 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1221 return; 1222 1223 /* 1224 * Bail if the request is not in use. We likely raced when 1225 * stopping the callout handler or it has already been aborted. 1226 */ 1227 if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || 1228 (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) 1229 return; 1230 1231 /* 1232 * Complete the request queue in case the timedout request is 1233 * actually just pending. 1234 */ 1235 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1236 if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) 1237 return; 1238 1239 sc->vtscsi_stats.scsi_cmd_timeouts++; 1240 to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; 1241 1242 if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) 1243 return; 1244 1245 vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); 1246 vtscsi_reset_bus(sc); 1247 } 1248 1249 static cam_status 1250 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) 1251 { 1252 cam_status status; 1253 1254 switch (cmd_resp->response) { 1255 case VIRTIO_SCSI_S_OK: 1256 status = CAM_REQ_CMP; 1257 break; 1258 case VIRTIO_SCSI_S_OVERRUN: 1259 status = CAM_DATA_RUN_ERR; 1260 break; 1261 case VIRTIO_SCSI_S_ABORTED: 1262 status = CAM_REQ_ABORTED; 1263 break; 1264 case VIRTIO_SCSI_S_BAD_TARGET: 1265 status = CAM_SEL_TIMEOUT; 1266 break; 1267 case VIRTIO_SCSI_S_RESET: 1268 status = CAM_SCSI_BUS_RESET; 1269 break; 1270 case VIRTIO_SCSI_S_BUSY: 1271 status = CAM_SCSI_BUSY; 1272 break; 1273 case VIRTIO_SCSI_S_TRANSPORT_FAILURE: 1274 case VIRTIO_SCSI_S_TARGET_FAILURE: 1275 case VIRTIO_SCSI_S_NEXUS_FAILURE: 1276 status = CAM_SCSI_IT_NEXUS_LOST; 1277 break; 1278 default: /* VIRTIO_SCSI_S_FAILURE */ 1279 status = CAM_REQ_CMP_ERR; 1280 break; 1281 } 1282 1283 return (status); 1284 } 1285 1286 static cam_status 1287 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, 1288 struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) 1289 { 1290 cam_status status; 1291 1292 csio->scsi_status = cmd_resp->status; 1293 csio->resid = cmd_resp->resid; 1294 1295 if (csio->scsi_status == SCSI_STATUS_OK) 1296 status = CAM_REQ_CMP; 1297 else 1298 status = CAM_SCSI_STATUS_ERROR; 1299 1300 if (cmd_resp->sense_len > 0) { 1301 status |= CAM_AUTOSNS_VALID; 1302 1303 if (cmd_resp->sense_len < csio->sense_len) 1304 csio->sense_resid = csio->sense_len - 1305 cmd_resp->sense_len; 1306 else 1307 csio->sense_resid = 0; 1308 1309 bzero(&csio->sense_data, sizeof(csio->sense_data)); 1310 memcpy(cmd_resp->sense, &csio->sense_data, 1311 csio->sense_len - csio->sense_resid); 1312 } 1313 1314 vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, 1315 "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", 1316 csio, csio->scsi_status, csio->resid, csio->sense_resid); 1317 1318 return (status); 1319 } 1320 1321 static void 1322 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1323 { 1324 struct ccb_hdr *ccbh; 1325 struct ccb_scsiio *csio; 1326 struct virtio_scsi_cmd_resp *cmd_resp; 1327 cam_status status; 1328 1329 csio = &req->vsr_ccb->csio; 1330 ccbh = &csio->ccb_h; 1331 cmd_resp = &req->vsr_cmd_resp; 1332 1333 KASSERT(ccbh->ccbh_vtscsi_req == req, 1334 ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); 1335 1336 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1337 callout_stop(&req->vsr_callout); 1338 1339 status = vtscsi_scsi_cmd_cam_status(cmd_resp); 1340 if (status == CAM_REQ_ABORTED) { 1341 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) 1342 status = CAM_CMD_TIMEOUT; 1343 } else if (status == CAM_REQ_CMP) 1344 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); 1345 1346 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1347 status |= CAM_DEV_QFRZN; 1348 xpt_freeze_devq(ccbh->path, 1); 1349 } 1350 1351 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1352 status |= CAM_RELEASE_SIMQ; 1353 1354 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", 1355 req, ccbh, status); 1356 1357 ccbh->status = status; 1358 xpt_done(req->vsr_ccb); 1359 vtscsi_enqueue_request(sc, req); 1360 } 1361 1362 static void 1363 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) 1364 { 1365 1366 /* XXX We probably shouldn't poll forever. */ 1367 req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; 1368 do 1369 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1370 while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); 1371 1372 req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; 1373 } 1374 1375 static int 1376 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, 1377 struct sglist *sg, int readable, int writable, int flag) 1378 { 1379 struct virtqueue *vq; 1380 int error; 1381 1382 vq = sc->vtscsi_control_vq; 1383 1384 KKASSERT(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); 1385 1386 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1387 if (error) { 1388 /* 1389 * Return EAGAIN when the virtqueue does not have enough 1390 * descriptors available. 1391 */ 1392 if (error == ENOSPC || error == EMSGSIZE) 1393 error = EAGAIN; 1394 1395 return (error); 1396 } 1397 1398 virtqueue_notify(vq, NULL); 1399 if (flag == VTSCSI_EXECUTE_POLL) 1400 vtscsi_poll_ctrl_req(sc, req); 1401 1402 return (0); 1403 } 1404 1405 static void 1406 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, 1407 struct vtscsi_request *req) 1408 { 1409 union ccb *ccb; 1410 struct ccb_hdr *ccbh; 1411 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1412 1413 ccb = req->vsr_ccb; 1414 ccbh = &ccb->ccb_h; 1415 tmf_resp = &req->vsr_tmf_resp; 1416 1417 switch (tmf_resp->response) { 1418 case VIRTIO_SCSI_S_FUNCTION_COMPLETE: 1419 ccbh->status = CAM_REQ_CMP; 1420 break; 1421 case VIRTIO_SCSI_S_FUNCTION_REJECTED: 1422 ccbh->status = CAM_UA_ABORT; 1423 break; 1424 default: 1425 ccbh->status = CAM_REQ_CMP_ERR; 1426 break; 1427 } 1428 1429 xpt_done(ccb); 1430 vtscsi_enqueue_request(sc, req); 1431 } 1432 1433 static int 1434 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, 1435 struct vtscsi_request *req) 1436 { 1437 struct sglist *sg; 1438 struct ccb_abort *cab; 1439 struct ccb_hdr *ccbh; 1440 struct ccb_hdr *abort_ccbh; 1441 struct vtscsi_request *abort_req; 1442 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1443 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1444 int error; 1445 1446 sg = sc->vtscsi_sglist; 1447 cab = &req->vsr_ccb->cab; 1448 ccbh = &cab->ccb_h; 1449 tmf_req = &req->vsr_tmf_req; 1450 tmf_resp = &req->vsr_tmf_resp; 1451 1452 /* CCB header and request that's to be aborted. */ 1453 abort_ccbh = &cab->abort_ccb->ccb_h; 1454 abort_req = abort_ccbh->ccbh_vtscsi_req; 1455 1456 if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { 1457 error = EINVAL; 1458 goto fail; 1459 } 1460 1461 /* Only attempt to abort requests that could be in-flight. */ 1462 if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { 1463 error = EALREADY; 1464 goto fail; 1465 } 1466 1467 abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; 1468 if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1469 callout_stop(&abort_req->vsr_callout); 1470 1471 vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1472 (uintptr_t) abort_ccbh, tmf_req); 1473 1474 sglist_reset(sg); 1475 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1476 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1477 1478 req->vsr_complete = vtscsi_complete_abort_task_cmd; 1479 tmf_resp->response = -1; 1480 1481 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1482 VTSCSI_EXECUTE_ASYNC); 1483 1484 fail: 1485 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " 1486 "abort_req=%p\n", error, req, abort_ccbh, abort_req); 1487 1488 return (error); 1489 } 1490 1491 static void 1492 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, 1493 struct vtscsi_request *req) 1494 { 1495 union ccb *ccb; 1496 struct ccb_hdr *ccbh; 1497 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1498 1499 ccb = req->vsr_ccb; 1500 ccbh = &ccb->ccb_h; 1501 tmf_resp = &req->vsr_tmf_resp; 1502 1503 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", 1504 req, ccb, tmf_resp->response); 1505 1506 if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { 1507 ccbh->status = CAM_REQ_CMP; 1508 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, 1509 ccbh->target_lun); 1510 } else 1511 ccbh->status = CAM_REQ_CMP_ERR; 1512 1513 xpt_done(ccb); 1514 vtscsi_enqueue_request(sc, req); 1515 } 1516 1517 static int 1518 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, 1519 struct vtscsi_request *req) 1520 { 1521 struct sglist *sg; 1522 struct ccb_resetdev *crd; 1523 struct ccb_hdr *ccbh; 1524 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1525 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1526 uint32_t subtype; 1527 int error; 1528 1529 sg = sc->vtscsi_sglist; 1530 crd = &req->vsr_ccb->crd; 1531 ccbh = &crd->ccb_h; 1532 tmf_req = &req->vsr_tmf_req; 1533 tmf_resp = &req->vsr_tmf_resp; 1534 1535 if (ccbh->target_lun == CAM_LUN_WILDCARD) 1536 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; 1537 else 1538 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; 1539 1540 vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); 1541 1542 sglist_reset(sg); 1543 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1544 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1545 1546 req->vsr_complete = vtscsi_complete_reset_dev_cmd; 1547 tmf_resp->response = -1; 1548 1549 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1550 VTSCSI_EXECUTE_ASYNC); 1551 1552 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", 1553 error, req, ccbh); 1554 1555 return (error); 1556 } 1557 1558 static void 1559 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) 1560 { 1561 1562 *target_id = lun[1]; 1563 *lun_id = (lun[2] << 8) | lun[3]; 1564 } 1565 1566 static void 1567 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) 1568 { 1569 1570 lun[0] = 1; 1571 lun[1] = ccbh->target_id; 1572 lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); 1573 lun[3] = ccbh->target_lun & 0xFF; 1574 } 1575 1576 static void 1577 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, 1578 struct virtio_scsi_cmd_req *cmd_req) 1579 { 1580 uint8_t attr; 1581 1582 switch (csio->tag_action) { 1583 case MSG_HEAD_OF_Q_TAG: 1584 attr = VIRTIO_SCSI_S_HEAD; 1585 break; 1586 case MSG_ORDERED_Q_TAG: 1587 attr = VIRTIO_SCSI_S_ORDERED; 1588 break; 1589 case MSG_ACA_TASK: 1590 attr = VIRTIO_SCSI_S_ACA; 1591 break; 1592 default: /* MSG_SIMPLE_Q_TAG */ 1593 attr = VIRTIO_SCSI_S_SIMPLE; 1594 break; 1595 } 1596 1597 vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); 1598 cmd_req->tag = (uintptr_t) csio; 1599 cmd_req->task_attr = attr; 1600 1601 memcpy(cmd_req->cdb, 1602 csio->ccb_h.flags & CAM_CDB_POINTER ? 1603 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, 1604 csio->cdb_len); 1605 } 1606 1607 static void 1608 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, 1609 uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) 1610 { 1611 1612 vtscsi_set_request_lun(ccbh, tmf_req->lun); 1613 1614 tmf_req->type = VIRTIO_SCSI_T_TMF; 1615 tmf_req->subtype = subtype; 1616 tmf_req->tag = tag; 1617 } 1618 1619 static void 1620 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) 1621 { 1622 int frozen; 1623 1624 frozen = sc->vtscsi_frozen; 1625 1626 if (reason & VTSCSI_REQUEST && 1627 (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) 1628 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; 1629 1630 if (reason & VTSCSI_REQUEST_VQ && 1631 (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) 1632 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; 1633 1634 /* Freeze the SIMQ if transitioned to frozen. */ 1635 if (frozen == 0 && sc->vtscsi_frozen != 0) { 1636 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); 1637 xpt_freeze_simq(sc->vtscsi_sim, 1); 1638 } 1639 } 1640 1641 static int 1642 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) 1643 { 1644 int thawed; 1645 1646 if (sc->vtscsi_frozen == 0 || reason == 0) 1647 return (0); 1648 1649 if (reason & VTSCSI_REQUEST && 1650 sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) 1651 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; 1652 1653 if (reason & VTSCSI_REQUEST_VQ && 1654 sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) 1655 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; 1656 1657 thawed = sc->vtscsi_frozen == 0; 1658 if (thawed != 0) 1659 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); 1660 1661 return (thawed); 1662 } 1663 1664 static void 1665 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, 1666 target_id_t target_id, lun_id_t lun_id) 1667 { 1668 struct cam_path *path; 1669 1670 /* Use the wildcard path from our softc for bus announcements. */ 1671 if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { 1672 xpt_async(ac_code, sc->vtscsi_path, NULL); 1673 return; 1674 } 1675 1676 if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), 1677 target_id, lun_id) != CAM_REQ_CMP) { 1678 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); 1679 return; 1680 } 1681 1682 xpt_async(ac_code, path, NULL); 1683 xpt_free_path(path); 1684 } 1685 1686 static void 1687 vtscsi_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb) 1688 { 1689 xpt_free_path(ccb->ccb_h.path); 1690 xpt_free_ccb(ccb); 1691 } 1692 1693 static void 1694 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, 1695 lun_id_t lun_id) 1696 { 1697 union ccb *ccb; 1698 cam_status status; 1699 1700 ccb = xpt_alloc_ccb(); 1701 if (ccb == NULL) { 1702 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1703 return; 1704 } 1705 1706 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1707 cam_sim_path(sc->vtscsi_sim), target_id, lun_id); 1708 if (status != CAM_REQ_CMP) { 1709 xpt_free_ccb(ccb); 1710 return; 1711 } 1712 1713 xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/); 1714 ccb->ccb_h.func_code = XPT_SCAN_LUN; 1715 ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback; 1716 ccb->crcn.flags = CAM_FLAG_NONE; 1717 xpt_action(ccb); 1718 } 1719 1720 static void 1721 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) 1722 { 1723 union ccb *ccb; 1724 cam_status status; 1725 1726 ccb = xpt_alloc_ccb(); 1727 if (ccb == NULL) { 1728 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1729 return; 1730 } 1731 1732 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1733 cam_sim_path(sc->vtscsi_sim), 1734 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); 1735 if (status != CAM_REQ_CMP) { 1736 xpt_free_ccb(ccb); 1737 return; 1738 } 1739 1740 xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/); 1741 ccb->ccb_h.func_code = XPT_SCAN_BUS; 1742 ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback; 1743 ccb->crcn.flags = CAM_FLAG_NONE; 1744 xpt_action(ccb); 1745 } 1746 1747 static void 1748 vtscsi_transport_reset_event(struct vtscsi_softc *sc, 1749 struct virtio_scsi_event *event) 1750 { 1751 target_id_t target_id; 1752 lun_id_t lun_id; 1753 1754 vtscsi_get_request_lun(event->lun, &target_id, &lun_id); 1755 1756 switch (event->reason) { 1757 case VIRTIO_SCSI_EVT_RESET_RESCAN: 1758 case VIRTIO_SCSI_EVT_RESET_REMOVED: 1759 vtscsi_execute_rescan(sc, target_id, lun_id); 1760 break; 1761 default: 1762 device_printf(sc->vtscsi_dev, 1763 "unhandled transport event reason: %d\n", event->reason); 1764 break; 1765 } 1766 } 1767 1768 static void 1769 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) 1770 { 1771 int error; 1772 1773 if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { 1774 switch (event->event) { 1775 case VIRTIO_SCSI_T_TRANSPORT_RESET: 1776 vtscsi_transport_reset_event(sc, event); 1777 break; 1778 default: 1779 device_printf(sc->vtscsi_dev, 1780 "unhandled event: %d\n", event->event); 1781 break; 1782 } 1783 } else 1784 vtscsi_execute_rescan_bus(sc); 1785 1786 /* 1787 * This should always be successful since the buffer 1788 * was just dequeued. 1789 */ 1790 error = vtscsi_enqueue_event_buf(sc, event); 1791 KASSERT(error == 0, 1792 ("cannot requeue event buffer: %d", error)); 1793 } 1794 1795 static int 1796 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, 1797 struct virtio_scsi_event *event) 1798 { 1799 struct sglist *sg; 1800 struct virtqueue *vq; 1801 int size, error; 1802 1803 sg = sc->vtscsi_sglist; 1804 vq = sc->vtscsi_event_vq; 1805 size = sc->vtscsi_event_buf_size; 1806 1807 bzero(event, size); 1808 1809 sglist_reset(sg); 1810 error = sglist_append(sg, event, size); 1811 if (error) 1812 return (error); 1813 1814 error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); 1815 if (error) 1816 return (error); 1817 1818 virtqueue_notify(vq, NULL); 1819 1820 return (0); 1821 } 1822 1823 static int 1824 vtscsi_init_event_vq(struct vtscsi_softc *sc) 1825 { 1826 struct virtio_scsi_event *event; 1827 int i, size, error; 1828 1829 /* 1830 * The first release of QEMU with VirtIO SCSI support would crash 1831 * when attempting to notify the event virtqueue. This was fixed 1832 * when hotplug support was added. 1833 */ 1834 if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) 1835 size = sc->vtscsi_event_buf_size; 1836 else 1837 size = 0; 1838 1839 if (size < sizeof(struct virtio_scsi_event)) 1840 return (0); 1841 1842 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1843 event = &sc->vtscsi_event_bufs[i]; 1844 1845 error = vtscsi_enqueue_event_buf(sc, event); 1846 if (error) 1847 break; 1848 } 1849 1850 /* 1851 * Even just one buffer is enough. Missed events are 1852 * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. 1853 */ 1854 if (i > 0) 1855 error = 0; 1856 1857 return (error); 1858 } 1859 1860 static void 1861 vtscsi_reinit_event_vq(struct vtscsi_softc *sc) 1862 { 1863 struct virtio_scsi_event *event; 1864 int i, error; 1865 1866 if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || 1867 sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) 1868 return; 1869 1870 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1871 event = &sc->vtscsi_event_bufs[i]; 1872 1873 error = vtscsi_enqueue_event_buf(sc, event); 1874 if (error) 1875 break; 1876 } 1877 1878 KASSERT(i > 0, ("cannot reinit event vq: %d", error)); 1879 } 1880 1881 static void 1882 vtscsi_drain_event_vq(struct vtscsi_softc *sc) 1883 { 1884 struct virtqueue *vq; 1885 int last; 1886 1887 vq = sc->vtscsi_event_vq; 1888 last = 0; 1889 1890 while (virtqueue_drain(vq, &last) != NULL) 1891 ; 1892 1893 KASSERT(virtqueue_empty(vq), ("eventvq not empty")); 1894 } 1895 1896 static void 1897 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) 1898 { 1899 1900 VTSCSI_LOCK_OWNED(sc); 1901 1902 if (sc->vtscsi_request_vq != NULL) 1903 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1904 if (sc->vtscsi_control_vq != NULL) 1905 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1906 } 1907 1908 static void 1909 vtscsi_complete_vqs(struct vtscsi_softc *sc) 1910 { 1911 1912 VTSCSI_LOCK(sc); 1913 vtscsi_complete_vqs_locked(sc); 1914 VTSCSI_UNLOCK(sc); 1915 } 1916 1917 static void 1918 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 1919 { 1920 union ccb *ccb; 1921 int detach; 1922 1923 ccb = req->vsr_ccb; 1924 1925 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); 1926 1927 /* 1928 * The callout must be drained when detaching since the request is 1929 * about to be freed. The VTSCSI_MTX must not be held for this in 1930 * case the callout is pending because there is a deadlock potential. 1931 * Otherwise, the virtqueue is being drained because of a bus reset 1932 * so we only need to attempt to stop the callouts. 1933 */ 1934 detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; 1935 if (detach != 0) 1936 VTSCSI_LOCK_NOTOWNED(sc); 1937 else 1938 VTSCSI_LOCK_OWNED(sc); 1939 1940 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { 1941 if (detach != 0) 1942 callout_drain(&req->vsr_callout); 1943 else 1944 callout_stop(&req->vsr_callout); 1945 } 1946 1947 if (ccb != NULL) { 1948 if (detach != 0) { 1949 VTSCSI_LOCK(sc); 1950 ccb->ccb_h.status = CAM_NO_HBA; 1951 } else 1952 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1953 xpt_done(ccb); 1954 if (detach != 0) 1955 VTSCSI_UNLOCK(sc); 1956 } 1957 1958 vtscsi_enqueue_request(sc, req); 1959 } 1960 1961 static void 1962 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 1963 { 1964 struct vtscsi_request *req; 1965 int last; 1966 1967 last = 0; 1968 1969 vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); 1970 1971 while ((req = virtqueue_drain(vq, &last)) != NULL) 1972 vtscsi_cancel_request(sc, req); 1973 1974 KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1975 } 1976 1977 static void 1978 vtscsi_drain_vqs(struct vtscsi_softc *sc) 1979 { 1980 1981 if (sc->vtscsi_control_vq != NULL) 1982 vtscsi_drain_vq(sc, sc->vtscsi_control_vq); 1983 if (sc->vtscsi_request_vq != NULL) 1984 vtscsi_drain_vq(sc, sc->vtscsi_request_vq); 1985 if (sc->vtscsi_event_vq != NULL) 1986 vtscsi_drain_event_vq(sc); 1987 } 1988 1989 static void 1990 vtscsi_stop(struct vtscsi_softc *sc) 1991 { 1992 1993 vtscsi_disable_vqs_intr(sc); 1994 virtio_stop(sc->vtscsi_dev); 1995 } 1996 1997 static int 1998 vtscsi_reset_bus(struct vtscsi_softc *sc) 1999 { 2000 int error; 2001 2002 VTSCSI_LOCK_OWNED(sc); 2003 2004 if (vtscsi_bus_reset_disable != 0) { 2005 device_printf(sc->vtscsi_dev, "bus reset disabled\n"); 2006 return (0); 2007 } 2008 2009 sc->vtscsi_flags |= VTSCSI_FLAG_RESET; 2010 2011 /* 2012 * vtscsi_stop() will cause the in-flight requests to be canceled. 2013 * Those requests are then completed here so CAM will retry them 2014 * after the reset is complete. 2015 */ 2016 vtscsi_stop(sc); 2017 vtscsi_complete_vqs_locked(sc); 2018 2019 /* Rid the virtqueues of any remaining requests. */ 2020 vtscsi_drain_vqs(sc); 2021 2022 /* 2023 * Any resource shortage that froze the SIMQ cannot persist across 2024 * a bus reset so ensure it gets thawed here. 2025 */ 2026 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 2027 xpt_release_simq(sc->vtscsi_sim, 0); 2028 2029 error = vtscsi_reinit(sc); 2030 if (error) { 2031 device_printf(sc->vtscsi_dev, 2032 "reinitialization failed, stopping device...\n"); 2033 vtscsi_stop(sc); 2034 } else 2035 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, 2036 CAM_LUN_WILDCARD); 2037 2038 sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; 2039 2040 return (error); 2041 } 2042 2043 static void 2044 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2045 { 2046 2047 #ifdef INVARIANTS 2048 int req_nsegs, resp_nsegs; 2049 2050 req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2051 resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2052 2053 KASSERT(req_nsegs == 1, ("request crossed page boundary")); 2054 KASSERT(resp_nsegs == 1, ("response crossed page boundary")); 2055 #endif 2056 2057 req->vsr_softc = sc; 2058 callout_init_lk(&req->vsr_callout, VTSCSI_MTX(sc)); 2059 } 2060 2061 static int 2062 vtscsi_alloc_requests(struct vtscsi_softc *sc) 2063 { 2064 struct vtscsi_request *req; 2065 int i, nreqs; 2066 2067 /* 2068 * Commands destined for either the request or control queues come 2069 * from the same SIM queue. Use the size of the request virtqueue 2070 * as it (should) be much more frequently used. Some additional 2071 * requests are allocated for internal (TMF) use. 2072 */ 2073 nreqs = virtqueue_size(sc->vtscsi_request_vq); 2074 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) 2075 nreqs /= VTSCSI_MIN_SEGMENTS; 2076 nreqs += VTSCSI_RESERVED_REQUESTS; 2077 2078 for (i = 0; i < nreqs; i++) { 2079 req = contigmalloc(sizeof(struct vtscsi_request), M_DEVBUF, 2080 M_WAITOK, 0, BUS_SPACE_MAXADDR, 16, 0); 2081 if (req == NULL) 2082 return (ENOMEM); 2083 2084 vtscsi_init_request(sc, req); 2085 2086 sc->vtscsi_nrequests++; 2087 vtscsi_enqueue_request(sc, req); 2088 } 2089 2090 return (0); 2091 } 2092 2093 static void 2094 vtscsi_free_requests(struct vtscsi_softc *sc) 2095 { 2096 struct vtscsi_request *req; 2097 2098 while ((req = vtscsi_dequeue_request(sc)) != NULL) { 2099 KASSERT(callout_active(&req->vsr_callout) == 0, 2100 ("request callout still active")); 2101 2102 sc->vtscsi_nrequests--; 2103 contigfree(req, sizeof(struct vtscsi_request), M_DEVBUF); 2104 } 2105 2106 KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", 2107 sc->vtscsi_nrequests)); 2108 } 2109 2110 static void 2111 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2112 { 2113 2114 KASSERT(req->vsr_softc == sc, 2115 ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); 2116 2117 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2118 2119 /* A request is available so the SIMQ could be released. */ 2120 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) 2121 xpt_release_simq(sc->vtscsi_sim, 1); 2122 2123 req->vsr_ccb = NULL; 2124 req->vsr_complete = NULL; 2125 req->vsr_ptr0 = NULL; 2126 req->vsr_state = VTSCSI_REQ_STATE_FREE; 2127 req->vsr_flags = 0; 2128 2129 bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2130 bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2131 2132 /* 2133 * We insert at the tail of the queue in order to make it 2134 * very unlikely a request will be reused if we race with 2135 * stopping its callout handler. 2136 */ 2137 TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); 2138 } 2139 2140 static struct vtscsi_request * 2141 vtscsi_dequeue_request(struct vtscsi_softc *sc) 2142 { 2143 struct vtscsi_request *req; 2144 2145 req = TAILQ_FIRST(&sc->vtscsi_req_free); 2146 if (req != NULL) { 2147 req->vsr_state = VTSCSI_REQ_STATE_INUSE; 2148 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); 2149 } else 2150 sc->vtscsi_stats.dequeue_no_requests++; 2151 2152 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2153 2154 return (req); 2155 } 2156 2157 static void 2158 vtscsi_complete_request(struct vtscsi_request *req) 2159 { 2160 2161 if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) 2162 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; 2163 2164 if (req->vsr_complete != NULL) 2165 req->vsr_complete(req->vsr_softc, req); 2166 } 2167 2168 static void 2169 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2170 { 2171 struct vtscsi_request *req; 2172 2173 VTSCSI_LOCK_OWNED(sc); 2174 2175 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) 2176 vtscsi_complete_request(req); 2177 } 2178 2179 static int 2180 vtscsi_control_vq_intr(void *xsc) 2181 { 2182 struct vtscsi_softc *sc; 2183 struct virtqueue *vq; 2184 2185 sc = xsc; 2186 vq = sc->vtscsi_control_vq; 2187 2188 again: 2189 VTSCSI_LOCK(sc); 2190 2191 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 2192 2193 if (virtqueue_enable_intr(vq) != 0) { 2194 virtqueue_disable_intr(vq); 2195 VTSCSI_UNLOCK(sc); 2196 goto again; 2197 } 2198 2199 VTSCSI_UNLOCK(sc); 2200 2201 return (1); 2202 } 2203 2204 static int 2205 vtscsi_event_vq_intr(void *xsc) 2206 { 2207 struct vtscsi_softc *sc; 2208 struct virtqueue *vq; 2209 struct virtio_scsi_event *event; 2210 2211 sc = xsc; 2212 vq = sc->vtscsi_event_vq; 2213 2214 again: 2215 VTSCSI_LOCK(sc); 2216 2217 while ((event = virtqueue_dequeue(vq, NULL)) != NULL) 2218 vtscsi_handle_event(sc, event); 2219 2220 if (virtqueue_enable_intr(vq) != 0) { 2221 virtqueue_disable_intr(vq); 2222 VTSCSI_UNLOCK(sc); 2223 goto again; 2224 } 2225 2226 VTSCSI_UNLOCK(sc); 2227 2228 return (1); 2229 } 2230 2231 static int 2232 vtscsi_request_vq_intr(void *xsc) 2233 { 2234 struct vtscsi_softc *sc; 2235 struct virtqueue *vq; 2236 2237 sc = xsc; 2238 vq = sc->vtscsi_request_vq; 2239 2240 again: 2241 VTSCSI_LOCK(sc); 2242 2243 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 2244 2245 if (virtqueue_enable_intr(vq) != 0) { 2246 virtqueue_disable_intr(vq); 2247 VTSCSI_UNLOCK(sc); 2248 goto again; 2249 } 2250 2251 VTSCSI_UNLOCK(sc); 2252 2253 return (1); 2254 } 2255 2256 static void 2257 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) 2258 { 2259 2260 virtqueue_disable_intr(sc->vtscsi_control_vq); 2261 virtqueue_disable_intr(sc->vtscsi_event_vq); 2262 virtqueue_disable_intr(sc->vtscsi_request_vq); 2263 } 2264 2265 static void 2266 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) 2267 { 2268 2269 virtqueue_enable_intr(sc->vtscsi_control_vq); 2270 virtqueue_enable_intr(sc->vtscsi_event_vq); 2271 virtqueue_enable_intr(sc->vtscsi_request_vq); 2272 } 2273 2274 static void 2275 vtscsi_get_tunables(struct vtscsi_softc *sc) 2276 { 2277 char tmpstr[64]; 2278 2279 TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); 2280 2281 ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", 2282 device_get_unit(sc->vtscsi_dev)); 2283 TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); 2284 } 2285 2286 static void 2287 vtscsi_add_sysctl(struct vtscsi_softc *sc) 2288 { 2289 device_t dev; 2290 struct vtscsi_statistics *stats; 2291 struct sysctl_ctx_list *ctx; 2292 struct sysctl_oid *tree; 2293 struct sysctl_oid_list *child; 2294 2295 dev = sc->vtscsi_dev; 2296 stats = &sc->vtscsi_stats; 2297 ctx = device_get_sysctl_ctx(dev); 2298 tree = device_get_sysctl_tree(dev); 2299 child = SYSCTL_CHILDREN(tree); 2300 2301 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 2302 CTLFLAG_RW, &sc->vtscsi_debug, 0, 2303 "Debug level"); 2304 2305 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", 2306 CTLFLAG_RD, &stats->scsi_cmd_timeouts, 2307 "SCSI command timeouts"); 2308 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", 2309 CTLFLAG_RD, &stats->dequeue_no_requests, 2310 "No available requests to dequeue"); 2311 } 2312 2313 static void 2314 vtscsi_printf_req(struct vtscsi_request *req, const char *func, 2315 const char *fmt, ...) 2316 { 2317 struct vtscsi_softc *sc; 2318 union ccb *ccb; 2319 struct sbuf sb; 2320 __va_list ap; 2321 char str[192]; 2322 char path_str[64]; 2323 2324 if (req == NULL) 2325 return; 2326 2327 sc = req->vsr_softc; 2328 ccb = req->vsr_ccb; 2329 2330 __va_start(ap, fmt); 2331 sbuf_new(&sb, str, sizeof(str), 0); 2332 2333 if (ccb == NULL) { 2334 sbuf_printf(&sb, "(noperiph:%s%d:%u): ", 2335 cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), 2336 cam_sim_bus(sc->vtscsi_sim)); 2337 } else { 2338 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); 2339 sbuf_cat(&sb, path_str); 2340 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2341 scsi_command_string(&ccb->csio, &sb); 2342 sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); 2343 } 2344 } 2345 2346 sbuf_vprintf(&sb, fmt, ap); 2347 __va_end(ap); 2348 2349 sbuf_finish(&sb); 2350 kprintf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, 2351 sbuf_data(&sb)); 2352 } 2353