1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO SCSI devices. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/kthread.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/sglist.h> 41 #include <sys/sysctl.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/callout.h> 45 #include <sys/queue.h> 46 #include <sys/sbuf.h> 47 48 #include <machine/stdarg.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 #include <sys/bus.h> 53 #include <sys/rman.h> 54 55 #include <cam/cam.h> 56 #include <cam/cam_ccb.h> 57 #include <cam/cam_sim.h> 58 #include <cam/cam_periph.h> 59 #include <cam/cam_xpt_sim.h> 60 #include <cam/cam_debug.h> 61 #include <cam/scsi/scsi_all.h> 62 #include <cam/scsi/scsi_message.h> 63 64 #include <dev/virtio/virtio.h> 65 #include <dev/virtio/virtqueue.h> 66 #include <dev/virtio/scsi/virtio_scsi.h> 67 #include <dev/virtio/scsi/virtio_scsivar.h> 68 69 #include "virtio_if.h" 70 71 static int vtscsi_modevent(module_t, int, void *); 72 73 static int vtscsi_probe(device_t); 74 static int vtscsi_attach(device_t); 75 static int vtscsi_detach(device_t); 76 static int vtscsi_suspend(device_t); 77 static int vtscsi_resume(device_t); 78 79 static void vtscsi_negotiate_features(struct vtscsi_softc *); 80 static void vtscsi_read_config(struct vtscsi_softc *, 81 struct virtio_scsi_config *); 82 static int vtscsi_maximum_segments(struct vtscsi_softc *, int); 83 static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); 84 static void vtscsi_check_sizes(struct vtscsi_softc *); 85 static void vtscsi_write_device_config(struct vtscsi_softc *); 86 static int vtscsi_reinit(struct vtscsi_softc *); 87 88 static int vtscsi_alloc_cam(struct vtscsi_softc *); 89 static int vtscsi_register_cam(struct vtscsi_softc *); 90 static void vtscsi_free_cam(struct vtscsi_softc *); 91 static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); 92 static int vtscsi_register_async(struct vtscsi_softc *); 93 static void vtscsi_deregister_async(struct vtscsi_softc *); 94 static void vtscsi_cam_action(struct cam_sim *, union ccb *); 95 static void vtscsi_cam_poll(struct cam_sim *); 96 97 static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, 98 union ccb *); 99 static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, 100 union ccb *); 101 static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); 102 static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); 103 static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); 104 static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, 105 struct cam_sim *, union ccb *); 106 107 static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, 108 struct sglist *, struct ccb_scsiio *); 109 static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, 110 struct vtscsi_request *, int *, int *); 111 static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, 112 struct vtscsi_request *); 113 static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); 114 static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, 115 struct vtscsi_request *); 116 static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, 117 struct vtscsi_request *); 118 static void vtscsi_timedout_scsi_cmd(void *); 119 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); 120 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, 121 struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); 122 static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, 123 struct vtscsi_request *); 124 125 static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, 126 struct vtscsi_request *); 127 static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, 128 struct vtscsi_request *, struct sglist *, int, int, int); 129 static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, 130 struct vtscsi_request *); 131 static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, 132 struct vtscsi_request *); 133 static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, 134 struct vtscsi_request *); 135 136 static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); 137 static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); 138 static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, 139 struct virtio_scsi_cmd_req *); 140 static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, 141 uintptr_t, struct virtio_scsi_ctrl_tmf_req *); 142 143 static void vtscsi_freeze_simq(struct vtscsi_softc *, int); 144 static int vtscsi_thaw_simq(struct vtscsi_softc *, int); 145 146 static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, 147 lun_id_t); 148 static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, 149 lun_id_t); 150 static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); 151 152 static void vtscsi_handle_event(struct vtscsi_softc *, 153 struct virtio_scsi_event *); 154 static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, 155 struct virtio_scsi_event *); 156 static int vtscsi_init_event_vq(struct vtscsi_softc *); 157 static void vtscsi_reinit_event_vq(struct vtscsi_softc *); 158 static void vtscsi_drain_event_vq(struct vtscsi_softc *); 159 160 static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); 161 static void vtscsi_complete_vqs(struct vtscsi_softc *); 162 static void vtscsi_drain_vqs(struct vtscsi_softc *); 163 static void vtscsi_cancel_request(struct vtscsi_softc *, 164 struct vtscsi_request *); 165 static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); 166 static void vtscsi_stop(struct vtscsi_softc *); 167 static int vtscsi_reset_bus(struct vtscsi_softc *); 168 169 static void vtscsi_init_request(struct vtscsi_softc *, 170 struct vtscsi_request *); 171 static int vtscsi_alloc_requests(struct vtscsi_softc *); 172 static void vtscsi_free_requests(struct vtscsi_softc *); 173 static void vtscsi_enqueue_request(struct vtscsi_softc *, 174 struct vtscsi_request *); 175 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); 176 177 static void vtscsi_complete_request(struct vtscsi_request *); 178 static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); 179 180 static void vtscsi_control_vq_intr(void *); 181 static void vtscsi_event_vq_intr(void *); 182 static void vtscsi_request_vq_intr(void *); 183 static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); 184 static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); 185 186 static void vtscsi_get_tunables(struct vtscsi_softc *); 187 static void vtscsi_add_sysctl(struct vtscsi_softc *); 188 189 static void vtscsi_printf_req(struct vtscsi_request *, const char *, 190 const char *, ...); 191 192 /* Global tunables. */ 193 /* 194 * The current QEMU VirtIO SCSI implementation does not cancel in-flight 195 * IO during virtio_stop(). So in-flight requests still complete after the 196 * device reset. We would have to wait for all the in-flight IO to complete, 197 * which defeats the typical purpose of a bus reset. We could simulate the 198 * bus reset with either I_T_NEXUS_RESET of all the targets, or with 199 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the 200 * control virtqueue). But this isn't very useful if things really go off 201 * the rails, so default to disabled for now. 202 */ 203 static int vtscsi_bus_reset_disable = 1; 204 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); 205 206 static struct virtio_feature_desc vtscsi_feature_desc[] = { 207 { VIRTIO_SCSI_F_INOUT, "InOut" }, 208 { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, 209 { 0, NULL } 210 }; 211 212 static device_method_t vtscsi_methods[] = { 213 /* Device methods. */ 214 DEVMETHOD(device_probe, vtscsi_probe), 215 DEVMETHOD(device_attach, vtscsi_attach), 216 DEVMETHOD(device_detach, vtscsi_detach), 217 DEVMETHOD(device_suspend, vtscsi_suspend), 218 DEVMETHOD(device_resume, vtscsi_resume), 219 220 DEVMETHOD_END 221 }; 222 223 static driver_t vtscsi_driver = { 224 "vtscsi", 225 vtscsi_methods, 226 sizeof(struct vtscsi_softc) 227 }; 228 static devclass_t vtscsi_devclass; 229 230 DRIVER_MODULE(virtio_scsi, virtio_mmio, vtscsi_driver, vtscsi_devclass, 231 vtscsi_modevent, 0); 232 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, 233 vtscsi_modevent, 0); 234 MODULE_VERSION(virtio_scsi, 1); 235 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); 236 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); 237 238 VIRTIO_SIMPLE_PNPTABLE(virtio_scsi, VIRTIO_ID_SCSI, "VirtIO SCSI Adapter"); 239 VIRTIO_SIMPLE_PNPINFO(virtio_mmio, virtio_scsi); 240 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_scsi); 241 242 static int 243 vtscsi_modevent(module_t mod, int type, void *unused) 244 { 245 int error; 246 247 switch (type) { 248 case MOD_LOAD: 249 case MOD_QUIESCE: 250 case MOD_UNLOAD: 251 case MOD_SHUTDOWN: 252 error = 0; 253 break; 254 default: 255 error = EOPNOTSUPP; 256 break; 257 } 258 259 return (error); 260 } 261 262 static int 263 vtscsi_probe(device_t dev) 264 { 265 return (VIRTIO_SIMPLE_PROBE(dev, virtio_scsi)); 266 } 267 268 static int 269 vtscsi_attach(device_t dev) 270 { 271 struct vtscsi_softc *sc; 272 struct virtio_scsi_config scsicfg; 273 int error; 274 275 sc = device_get_softc(dev); 276 sc->vtscsi_dev = dev; 277 278 VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); 279 TAILQ_INIT(&sc->vtscsi_req_free); 280 281 vtscsi_get_tunables(sc); 282 vtscsi_add_sysctl(sc); 283 284 virtio_set_feature_desc(dev, vtscsi_feature_desc); 285 vtscsi_negotiate_features(sc); 286 287 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 288 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; 289 if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) 290 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; 291 if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) 292 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; 293 294 vtscsi_read_config(sc, &scsicfg); 295 296 sc->vtscsi_max_channel = scsicfg.max_channel; 297 sc->vtscsi_max_target = scsicfg.max_target; 298 sc->vtscsi_max_lun = scsicfg.max_lun; 299 sc->vtscsi_event_buf_size = scsicfg.event_info_size; 300 301 vtscsi_write_device_config(sc); 302 303 sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); 304 sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); 305 if (sc->vtscsi_sglist == NULL) { 306 error = ENOMEM; 307 device_printf(dev, "cannot allocate sglist\n"); 308 goto fail; 309 } 310 311 error = vtscsi_alloc_virtqueues(sc); 312 if (error) { 313 device_printf(dev, "cannot allocate virtqueues\n"); 314 goto fail; 315 } 316 317 vtscsi_check_sizes(sc); 318 319 error = vtscsi_init_event_vq(sc); 320 if (error) { 321 device_printf(dev, "cannot populate the eventvq\n"); 322 goto fail; 323 } 324 325 error = vtscsi_alloc_requests(sc); 326 if (error) { 327 device_printf(dev, "cannot allocate requests\n"); 328 goto fail; 329 } 330 331 error = vtscsi_alloc_cam(sc); 332 if (error) { 333 device_printf(dev, "cannot allocate CAM structures\n"); 334 goto fail; 335 } 336 337 error = virtio_setup_intr(dev, INTR_TYPE_CAM); 338 if (error) { 339 device_printf(dev, "cannot setup virtqueue interrupts\n"); 340 goto fail; 341 } 342 343 vtscsi_enable_vqs_intr(sc); 344 345 /* 346 * Register with CAM after interrupts are enabled so we will get 347 * notified of the probe responses. 348 */ 349 error = vtscsi_register_cam(sc); 350 if (error) { 351 device_printf(dev, "cannot register with CAM\n"); 352 goto fail; 353 } 354 355 fail: 356 if (error) 357 vtscsi_detach(dev); 358 359 return (error); 360 } 361 362 static int 363 vtscsi_detach(device_t dev) 364 { 365 struct vtscsi_softc *sc; 366 367 sc = device_get_softc(dev); 368 369 VTSCSI_LOCK(sc); 370 sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; 371 if (device_is_attached(dev)) 372 vtscsi_stop(sc); 373 VTSCSI_UNLOCK(sc); 374 375 vtscsi_complete_vqs(sc); 376 vtscsi_drain_vqs(sc); 377 378 vtscsi_free_cam(sc); 379 vtscsi_free_requests(sc); 380 381 if (sc->vtscsi_sglist != NULL) { 382 sglist_free(sc->vtscsi_sglist); 383 sc->vtscsi_sglist = NULL; 384 } 385 386 VTSCSI_LOCK_DESTROY(sc); 387 388 return (0); 389 } 390 391 static int 392 vtscsi_suspend(device_t dev) 393 { 394 395 return (0); 396 } 397 398 static int 399 vtscsi_resume(device_t dev) 400 { 401 402 return (0); 403 } 404 405 static void 406 vtscsi_negotiate_features(struct vtscsi_softc *sc) 407 { 408 device_t dev; 409 uint64_t features; 410 411 dev = sc->vtscsi_dev; 412 features = virtio_negotiate_features(dev, VTSCSI_FEATURES); 413 sc->vtscsi_features = features; 414 } 415 416 #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ 417 virtio_read_device_config(_dev, \ 418 offsetof(struct virtio_scsi_config, _field), \ 419 &(_cfg)->_field, sizeof((_cfg)->_field)) \ 420 421 static void 422 vtscsi_read_config(struct vtscsi_softc *sc, 423 struct virtio_scsi_config *scsicfg) 424 { 425 device_t dev; 426 427 dev = sc->vtscsi_dev; 428 429 bzero(scsicfg, sizeof(struct virtio_scsi_config)); 430 431 VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); 432 VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); 433 VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); 434 VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); 435 VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); 436 VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); 437 VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); 438 VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); 439 VTSCSI_GET_CONFIG(dev, max_target, scsicfg); 440 VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); 441 } 442 443 #undef VTSCSI_GET_CONFIG 444 445 static int 446 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) 447 { 448 int nsegs; 449 450 nsegs = VTSCSI_MIN_SEGMENTS; 451 452 if (seg_max > 0) { 453 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1); 454 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) 455 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 456 } else 457 nsegs += 1; 458 459 return (nsegs); 460 } 461 462 static int 463 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) 464 { 465 device_t dev; 466 struct vq_alloc_info vq_info[3]; 467 int nvqs; 468 469 dev = sc->vtscsi_dev; 470 nvqs = 3; 471 472 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, 473 &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); 474 475 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, 476 &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); 477 478 VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, 479 vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, 480 "%s request", device_get_nameunit(dev)); 481 482 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 483 } 484 485 static void 486 vtscsi_check_sizes(struct vtscsi_softc *sc) 487 { 488 int rqsize; 489 490 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) { 491 /* 492 * Ensure the assertions in virtqueue_enqueue(), 493 * even if the hypervisor reports a bad seg_max. 494 */ 495 rqsize = virtqueue_size(sc->vtscsi_request_vq); 496 if (sc->vtscsi_max_nsegs > rqsize) { 497 device_printf(sc->vtscsi_dev, 498 "clamping seg_max (%d %d)\n", sc->vtscsi_max_nsegs, 499 rqsize); 500 sc->vtscsi_max_nsegs = rqsize; 501 } 502 } 503 } 504 505 static void 506 vtscsi_write_device_config(struct vtscsi_softc *sc) 507 { 508 509 virtio_write_dev_config_4(sc->vtscsi_dev, 510 offsetof(struct virtio_scsi_config, sense_size), 511 VIRTIO_SCSI_SENSE_SIZE); 512 513 /* 514 * This is the size in the virtio_scsi_cmd_req structure. Note 515 * this value (32) is larger than the maximum CAM CDB size (16). 516 */ 517 virtio_write_dev_config_4(sc->vtscsi_dev, 518 offsetof(struct virtio_scsi_config, cdb_size), 519 VIRTIO_SCSI_CDB_SIZE); 520 } 521 522 static int 523 vtscsi_reinit(struct vtscsi_softc *sc) 524 { 525 device_t dev; 526 int error; 527 528 dev = sc->vtscsi_dev; 529 530 error = virtio_reinit(dev, sc->vtscsi_features); 531 if (error == 0) { 532 vtscsi_write_device_config(sc); 533 vtscsi_reinit_event_vq(sc); 534 virtio_reinit_complete(dev); 535 536 vtscsi_enable_vqs_intr(sc); 537 } 538 539 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); 540 541 return (error); 542 } 543 544 static int 545 vtscsi_alloc_cam(struct vtscsi_softc *sc) 546 { 547 device_t dev; 548 struct cam_devq *devq; 549 int openings; 550 551 dev = sc->vtscsi_dev; 552 openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; 553 554 devq = cam_simq_alloc(openings); 555 if (devq == NULL) { 556 device_printf(dev, "cannot allocate SIM queue\n"); 557 return (ENOMEM); 558 } 559 560 sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, 561 "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, 562 openings, devq); 563 if (sc->vtscsi_sim == NULL) { 564 cam_simq_free(devq); 565 device_printf(dev, "cannot allocate SIM\n"); 566 return (ENOMEM); 567 } 568 569 return (0); 570 } 571 572 static int 573 vtscsi_register_cam(struct vtscsi_softc *sc) 574 { 575 device_t dev; 576 int registered, error; 577 578 dev = sc->vtscsi_dev; 579 registered = 0; 580 581 VTSCSI_LOCK(sc); 582 583 if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) { 584 error = ENOMEM; 585 device_printf(dev, "cannot register XPT bus\n"); 586 goto fail; 587 } 588 589 registered = 1; 590 591 if (xpt_create_path(&sc->vtscsi_path, NULL, 592 cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, 593 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 594 error = ENOMEM; 595 device_printf(dev, "cannot create bus path\n"); 596 goto fail; 597 } 598 599 if (vtscsi_register_async(sc) != CAM_REQ_CMP) { 600 error = EIO; 601 device_printf(dev, "cannot register async callback\n"); 602 goto fail; 603 } 604 605 VTSCSI_UNLOCK(sc); 606 607 return (0); 608 609 fail: 610 if (sc->vtscsi_path != NULL) { 611 xpt_free_path(sc->vtscsi_path); 612 sc->vtscsi_path = NULL; 613 } 614 615 if (registered != 0) 616 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 617 618 VTSCSI_UNLOCK(sc); 619 620 return (error); 621 } 622 623 static void 624 vtscsi_free_cam(struct vtscsi_softc *sc) 625 { 626 627 VTSCSI_LOCK(sc); 628 629 if (sc->vtscsi_path != NULL) { 630 vtscsi_deregister_async(sc); 631 632 xpt_free_path(sc->vtscsi_path); 633 sc->vtscsi_path = NULL; 634 635 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 636 } 637 638 if (sc->vtscsi_sim != NULL) { 639 cam_sim_free(sc->vtscsi_sim, 1); 640 sc->vtscsi_sim = NULL; 641 } 642 643 VTSCSI_UNLOCK(sc); 644 } 645 646 static void 647 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) 648 { 649 struct cam_sim *sim; 650 struct vtscsi_softc *sc; 651 652 sim = cb_arg; 653 sc = cam_sim_softc(sim); 654 655 vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); 656 657 /* 658 * TODO Once QEMU supports event reporting, we should 659 * (un)subscribe to events here. 660 */ 661 switch (code) { 662 case AC_FOUND_DEVICE: 663 break; 664 case AC_LOST_DEVICE: 665 break; 666 } 667 } 668 669 static int 670 vtscsi_register_async(struct vtscsi_softc *sc) 671 { 672 struct ccb_setasync csa; 673 674 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 675 csa.ccb_h.func_code = XPT_SASYNC_CB; 676 csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; 677 csa.callback = vtscsi_cam_async; 678 csa.callback_arg = sc->vtscsi_sim; 679 680 xpt_action((union ccb *) &csa); 681 682 return (csa.ccb_h.status); 683 } 684 685 static void 686 vtscsi_deregister_async(struct vtscsi_softc *sc) 687 { 688 struct ccb_setasync csa; 689 690 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 691 csa.ccb_h.func_code = XPT_SASYNC_CB; 692 csa.event_enable = 0; 693 csa.callback = vtscsi_cam_async; 694 csa.callback_arg = sc->vtscsi_sim; 695 696 xpt_action((union ccb *) &csa); 697 } 698 699 static void 700 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) 701 { 702 struct vtscsi_softc *sc; 703 struct ccb_hdr *ccbh; 704 705 sc = cam_sim_softc(sim); 706 ccbh = &ccb->ccb_h; 707 708 VTSCSI_LOCK_OWNED(sc); 709 710 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { 711 /* 712 * The VTSCSI_MTX is briefly dropped between setting 713 * VTSCSI_FLAG_DETACH and deregistering with CAM, so 714 * drop any CCBs that come in during that window. 715 */ 716 ccbh->status = CAM_NO_HBA; 717 xpt_done(ccb); 718 return; 719 } 720 721 switch (ccbh->func_code) { 722 case XPT_SCSI_IO: 723 vtscsi_cam_scsi_io(sc, sim, ccb); 724 break; 725 726 case XPT_SET_TRAN_SETTINGS: 727 ccbh->status = CAM_FUNC_NOTAVAIL; 728 xpt_done(ccb); 729 break; 730 731 case XPT_GET_TRAN_SETTINGS: 732 vtscsi_cam_get_tran_settings(sc, ccb); 733 break; 734 735 case XPT_RESET_BUS: 736 vtscsi_cam_reset_bus(sc, ccb); 737 break; 738 739 case XPT_RESET_DEV: 740 vtscsi_cam_reset_dev(sc, ccb); 741 break; 742 743 case XPT_ABORT: 744 vtscsi_cam_abort(sc, ccb); 745 break; 746 747 case XPT_CALC_GEOMETRY: 748 cam_calc_geometry(&ccb->ccg, 1); 749 xpt_done(ccb); 750 break; 751 752 case XPT_PATH_INQ: 753 vtscsi_cam_path_inquiry(sc, sim, ccb); 754 break; 755 756 default: 757 vtscsi_dprintf(sc, VTSCSI_ERROR, 758 "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); 759 760 ccbh->status = CAM_REQ_INVALID; 761 xpt_done(ccb); 762 break; 763 } 764 } 765 766 static void 767 vtscsi_cam_poll(struct cam_sim *sim) 768 { 769 struct vtscsi_softc *sc; 770 771 sc = cam_sim_softc(sim); 772 773 vtscsi_complete_vqs_locked(sc); 774 } 775 776 static void 777 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, 778 union ccb *ccb) 779 { 780 struct ccb_hdr *ccbh; 781 struct ccb_scsiio *csio; 782 int error; 783 784 ccbh = &ccb->ccb_h; 785 csio = &ccb->csio; 786 787 if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { 788 error = EINVAL; 789 ccbh->status = CAM_REQ_INVALID; 790 goto done; 791 } 792 793 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && 794 (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { 795 error = EINVAL; 796 ccbh->status = CAM_REQ_INVALID; 797 goto done; 798 } 799 800 error = vtscsi_start_scsi_cmd(sc, ccb); 801 802 done: 803 if (error) { 804 vtscsi_dprintf(sc, VTSCSI_ERROR, 805 "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); 806 xpt_done(ccb); 807 } 808 } 809 810 static void 811 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) 812 { 813 struct ccb_trans_settings *cts; 814 struct ccb_trans_settings_scsi *scsi; 815 816 cts = &ccb->cts; 817 scsi = &cts->proto_specific.scsi; 818 819 cts->protocol = PROTO_SCSI; 820 cts->protocol_version = SCSI_REV_SPC3; 821 cts->transport = XPORT_SAS; 822 cts->transport_version = 0; 823 824 scsi->valid = CTS_SCSI_VALID_TQ; 825 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 826 827 ccb->ccb_h.status = CAM_REQ_CMP; 828 xpt_done(ccb); 829 } 830 831 static void 832 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) 833 { 834 int error; 835 836 error = vtscsi_reset_bus(sc); 837 if (error == 0) 838 ccb->ccb_h.status = CAM_REQ_CMP; 839 else 840 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 841 842 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", 843 error, ccb, ccb->ccb_h.status); 844 845 xpt_done(ccb); 846 } 847 848 static void 849 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) 850 { 851 struct ccb_hdr *ccbh; 852 struct vtscsi_request *req; 853 int error; 854 855 ccbh = &ccb->ccb_h; 856 857 req = vtscsi_dequeue_request(sc); 858 if (req == NULL) { 859 error = EAGAIN; 860 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 861 goto fail; 862 } 863 864 req->vsr_ccb = ccb; 865 866 error = vtscsi_execute_reset_dev_cmd(sc, req); 867 if (error == 0) 868 return; 869 870 vtscsi_enqueue_request(sc, req); 871 872 fail: 873 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 874 error, req, ccb); 875 876 if (error == EAGAIN) 877 ccbh->status = CAM_RESRC_UNAVAIL; 878 else 879 ccbh->status = CAM_REQ_CMP_ERR; 880 881 xpt_done(ccb); 882 } 883 884 static void 885 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) 886 { 887 struct vtscsi_request *req; 888 struct ccb_hdr *ccbh; 889 int error; 890 891 ccbh = &ccb->ccb_h; 892 893 req = vtscsi_dequeue_request(sc); 894 if (req == NULL) { 895 error = EAGAIN; 896 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 897 goto fail; 898 } 899 900 req->vsr_ccb = ccb; 901 902 error = vtscsi_execute_abort_task_cmd(sc, req); 903 if (error == 0) 904 return; 905 906 vtscsi_enqueue_request(sc, req); 907 908 fail: 909 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 910 error, req, ccb); 911 912 if (error == EAGAIN) 913 ccbh->status = CAM_RESRC_UNAVAIL; 914 else 915 ccbh->status = CAM_REQ_CMP_ERR; 916 917 xpt_done(ccb); 918 } 919 920 static void 921 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, 922 union ccb *ccb) 923 { 924 device_t dev; 925 struct ccb_pathinq *cpi; 926 927 dev = sc->vtscsi_dev; 928 cpi = &ccb->cpi; 929 930 vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); 931 932 cpi->version_num = 1; 933 cpi->hba_inquiry = PI_TAG_ABLE; 934 cpi->target_sprt = 0; 935 cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 936 if (vtscsi_bus_reset_disable != 0) 937 cpi->hba_misc |= PIM_NOBUSRESET; 938 cpi->hba_eng_cnt = 0; 939 940 cpi->max_target = sc->vtscsi_max_target; 941 cpi->max_lun = sc->vtscsi_max_lun; 942 cpi->initiator_id = cpi->max_target + 1; 943 944 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 945 strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); 946 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 947 948 cpi->unit_number = cam_sim_unit(sim); 949 cpi->bus_id = cam_sim_bus(sim); 950 951 cpi->base_transfer_speed = 300000; 952 953 cpi->protocol = PROTO_SCSI; 954 cpi->protocol_version = SCSI_REV_SPC3; 955 cpi->transport = XPORT_SAS; 956 cpi->transport_version = 0; 957 958 cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * 959 PAGE_SIZE; 960 961 cpi->hba_vendor = virtio_get_vendor(dev); 962 cpi->hba_device = virtio_get_device(dev); 963 cpi->hba_subvendor = virtio_get_subvendor(dev); 964 cpi->hba_subdevice = virtio_get_subdevice(dev); 965 966 ccb->ccb_h.status = CAM_REQ_CMP; 967 xpt_done(ccb); 968 } 969 970 static int 971 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, 972 struct ccb_scsiio *csio) 973 { 974 struct ccb_hdr *ccbh; 975 struct bus_dma_segment *dseg; 976 int i, error; 977 978 ccbh = &csio->ccb_h; 979 error = 0; 980 981 switch ((ccbh->flags & CAM_DATA_MASK)) { 982 case CAM_DATA_VADDR: 983 error = sglist_append(sg, csio->data_ptr, csio->dxfer_len); 984 break; 985 case CAM_DATA_PADDR: 986 error = sglist_append_phys(sg, 987 (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len); 988 break; 989 case CAM_DATA_SG: 990 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 991 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 992 error = sglist_append(sg, 993 (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len); 994 } 995 break; 996 case CAM_DATA_SG_PADDR: 997 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 998 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 999 error = sglist_append_phys(sg, 1000 (vm_paddr_t) dseg->ds_addr, dseg->ds_len); 1001 } 1002 break; 1003 case CAM_DATA_BIO: 1004 error = sglist_append_bio(sg, (struct bio *) csio->data_ptr); 1005 break; 1006 default: 1007 error = EINVAL; 1008 break; 1009 } 1010 1011 return (error); 1012 } 1013 1014 static int 1015 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, 1016 int *readable, int *writable) 1017 { 1018 struct sglist *sg; 1019 struct ccb_hdr *ccbh; 1020 struct ccb_scsiio *csio; 1021 struct virtio_scsi_cmd_req *cmd_req; 1022 struct virtio_scsi_cmd_resp *cmd_resp; 1023 int error; 1024 1025 sg = sc->vtscsi_sglist; 1026 csio = &req->vsr_ccb->csio; 1027 ccbh = &csio->ccb_h; 1028 cmd_req = &req->vsr_cmd_req; 1029 cmd_resp = &req->vsr_cmd_resp; 1030 1031 sglist_reset(sg); 1032 1033 sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); 1034 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { 1035 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1036 /* At least one segment must be left for the response. */ 1037 if (error || sg->sg_nseg == sg->sg_maxseg) 1038 goto fail; 1039 } 1040 1041 *readable = sg->sg_nseg; 1042 1043 sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); 1044 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { 1045 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1046 if (error) 1047 goto fail; 1048 } 1049 1050 *writable = sg->sg_nseg - *readable; 1051 1052 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " 1053 "writable=%d\n", req, ccbh, *readable, *writable); 1054 1055 return (0); 1056 1057 fail: 1058 /* 1059 * This should never happen unless maxio was incorrectly set. 1060 */ 1061 vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); 1062 1063 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " 1064 "nseg=%d maxseg=%d\n", 1065 error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); 1066 1067 return (EFBIG); 1068 } 1069 1070 static int 1071 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1072 { 1073 struct sglist *sg; 1074 struct virtqueue *vq; 1075 struct ccb_scsiio *csio; 1076 struct ccb_hdr *ccbh; 1077 struct virtio_scsi_cmd_req *cmd_req; 1078 struct virtio_scsi_cmd_resp *cmd_resp; 1079 int readable, writable, error; 1080 1081 sg = sc->vtscsi_sglist; 1082 vq = sc->vtscsi_request_vq; 1083 csio = &req->vsr_ccb->csio; 1084 ccbh = &csio->ccb_h; 1085 cmd_req = &req->vsr_cmd_req; 1086 cmd_resp = &req->vsr_cmd_resp; 1087 1088 vtscsi_init_scsi_cmd_req(csio, cmd_req); 1089 1090 error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); 1091 if (error) 1092 return (error); 1093 1094 req->vsr_complete = vtscsi_complete_scsi_cmd; 1095 cmd_resp->response = -1; 1096 1097 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1098 if (error) { 1099 vtscsi_dprintf(sc, VTSCSI_ERROR, 1100 "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); 1101 1102 ccbh->status = CAM_REQUEUE_REQ; 1103 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); 1104 return (error); 1105 } 1106 1107 ccbh->status |= CAM_SIM_QUEUED; 1108 ccbh->ccbh_vtscsi_req = req; 1109 1110 virtqueue_notify(vq); 1111 1112 if (ccbh->timeout != CAM_TIME_INFINITY) { 1113 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; 1114 callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout, 1115 0, vtscsi_timedout_scsi_cmd, req, 0); 1116 } 1117 1118 vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", 1119 req, ccbh); 1120 1121 return (0); 1122 } 1123 1124 static int 1125 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) 1126 { 1127 struct vtscsi_request *req; 1128 int error; 1129 1130 req = vtscsi_dequeue_request(sc); 1131 if (req == NULL) { 1132 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1133 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 1134 return (ENOBUFS); 1135 } 1136 1137 req->vsr_ccb = ccb; 1138 1139 error = vtscsi_execute_scsi_cmd(sc, req); 1140 if (error) 1141 vtscsi_enqueue_request(sc, req); 1142 1143 return (error); 1144 } 1145 1146 static void 1147 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1148 struct vtscsi_request *req) 1149 { 1150 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1151 struct vtscsi_request *to_req; 1152 uint8_t response; 1153 1154 tmf_resp = &req->vsr_tmf_resp; 1155 response = tmf_resp->response; 1156 to_req = req->vsr_timedout_req; 1157 1158 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", 1159 req, to_req, response); 1160 1161 vtscsi_enqueue_request(sc, req); 1162 1163 /* 1164 * The timedout request could have completed between when the 1165 * abort task was sent and when the host processed it. 1166 */ 1167 if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) 1168 return; 1169 1170 /* The timedout request was successfully aborted. */ 1171 if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) 1172 return; 1173 1174 /* Don't bother if the device is going away. */ 1175 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1176 return; 1177 1178 /* The timedout request will be aborted by the reset. */ 1179 if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) 1180 return; 1181 1182 vtscsi_reset_bus(sc); 1183 } 1184 1185 static int 1186 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1187 struct vtscsi_request *to_req) 1188 { 1189 struct sglist *sg; 1190 struct ccb_hdr *to_ccbh; 1191 struct vtscsi_request *req; 1192 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1193 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1194 int error; 1195 1196 sg = sc->vtscsi_sglist; 1197 to_ccbh = &to_req->vsr_ccb->ccb_h; 1198 1199 req = vtscsi_dequeue_request(sc); 1200 if (req == NULL) { 1201 error = ENOBUFS; 1202 goto fail; 1203 } 1204 1205 tmf_req = &req->vsr_tmf_req; 1206 tmf_resp = &req->vsr_tmf_resp; 1207 1208 vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1209 (uintptr_t) to_ccbh, tmf_req); 1210 1211 sglist_reset(sg); 1212 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1213 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1214 1215 req->vsr_timedout_req = to_req; 1216 req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; 1217 tmf_resp->response = -1; 1218 1219 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1220 VTSCSI_EXECUTE_ASYNC); 1221 if (error == 0) 1222 return (0); 1223 1224 vtscsi_enqueue_request(sc, req); 1225 1226 fail: 1227 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " 1228 "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); 1229 1230 return (error); 1231 } 1232 1233 static void 1234 vtscsi_timedout_scsi_cmd(void *xreq) 1235 { 1236 struct vtscsi_softc *sc; 1237 struct vtscsi_request *to_req; 1238 1239 to_req = xreq; 1240 sc = to_req->vsr_softc; 1241 1242 vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", 1243 to_req, to_req->vsr_ccb, to_req->vsr_state); 1244 1245 /* Don't bother if the device is going away. */ 1246 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1247 return; 1248 1249 /* 1250 * Bail if the request is not in use. We likely raced when 1251 * stopping the callout handler or it has already been aborted. 1252 */ 1253 if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || 1254 (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) 1255 return; 1256 1257 /* 1258 * Complete the request queue in case the timedout request is 1259 * actually just pending. 1260 */ 1261 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1262 if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) 1263 return; 1264 1265 sc->vtscsi_stats.scsi_cmd_timeouts++; 1266 to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; 1267 1268 if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) 1269 return; 1270 1271 vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); 1272 vtscsi_reset_bus(sc); 1273 } 1274 1275 static cam_status 1276 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) 1277 { 1278 cam_status status; 1279 1280 switch (cmd_resp->response) { 1281 case VIRTIO_SCSI_S_OK: 1282 status = CAM_REQ_CMP; 1283 break; 1284 case VIRTIO_SCSI_S_OVERRUN: 1285 status = CAM_DATA_RUN_ERR; 1286 break; 1287 case VIRTIO_SCSI_S_ABORTED: 1288 status = CAM_REQ_ABORTED; 1289 break; 1290 case VIRTIO_SCSI_S_BAD_TARGET: 1291 status = CAM_SEL_TIMEOUT; 1292 break; 1293 case VIRTIO_SCSI_S_RESET: 1294 status = CAM_SCSI_BUS_RESET; 1295 break; 1296 case VIRTIO_SCSI_S_BUSY: 1297 status = CAM_SCSI_BUSY; 1298 break; 1299 case VIRTIO_SCSI_S_TRANSPORT_FAILURE: 1300 case VIRTIO_SCSI_S_TARGET_FAILURE: 1301 case VIRTIO_SCSI_S_NEXUS_FAILURE: 1302 status = CAM_SCSI_IT_NEXUS_LOST; 1303 break; 1304 default: /* VIRTIO_SCSI_S_FAILURE */ 1305 status = CAM_REQ_CMP_ERR; 1306 break; 1307 } 1308 1309 return (status); 1310 } 1311 1312 static cam_status 1313 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, 1314 struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) 1315 { 1316 cam_status status; 1317 1318 csio->scsi_status = cmd_resp->status; 1319 csio->resid = cmd_resp->resid; 1320 1321 if (csio->scsi_status == SCSI_STATUS_OK) 1322 status = CAM_REQ_CMP; 1323 else 1324 status = CAM_SCSI_STATUS_ERROR; 1325 1326 if (cmd_resp->sense_len > 0) { 1327 status |= CAM_AUTOSNS_VALID; 1328 1329 if (cmd_resp->sense_len < csio->sense_len) 1330 csio->sense_resid = csio->sense_len - 1331 cmd_resp->sense_len; 1332 else 1333 csio->sense_resid = 0; 1334 1335 memcpy(&csio->sense_data, cmd_resp->sense, 1336 csio->sense_len - csio->sense_resid); 1337 } 1338 1339 vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, 1340 "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", 1341 csio, csio->scsi_status, csio->resid, csio->sense_resid); 1342 1343 return (status); 1344 } 1345 1346 static void 1347 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1348 { 1349 struct ccb_hdr *ccbh; 1350 struct ccb_scsiio *csio; 1351 struct virtio_scsi_cmd_resp *cmd_resp; 1352 cam_status status; 1353 1354 csio = &req->vsr_ccb->csio; 1355 ccbh = &csio->ccb_h; 1356 cmd_resp = &req->vsr_cmd_resp; 1357 1358 KASSERT(ccbh->ccbh_vtscsi_req == req, 1359 ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); 1360 1361 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1362 callout_stop(&req->vsr_callout); 1363 1364 status = vtscsi_scsi_cmd_cam_status(cmd_resp); 1365 if (status == CAM_REQ_ABORTED) { 1366 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) 1367 status = CAM_CMD_TIMEOUT; 1368 } else if (status == CAM_REQ_CMP) 1369 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); 1370 1371 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1372 status |= CAM_DEV_QFRZN; 1373 xpt_freeze_devq(ccbh->path, 1); 1374 } 1375 1376 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1377 status |= CAM_RELEASE_SIMQ; 1378 1379 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", 1380 req, ccbh, status); 1381 1382 ccbh->status = status; 1383 xpt_done(req->vsr_ccb); 1384 vtscsi_enqueue_request(sc, req); 1385 } 1386 1387 static void 1388 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) 1389 { 1390 1391 /* XXX We probably shouldn't poll forever. */ 1392 req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; 1393 do 1394 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1395 while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); 1396 1397 req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; 1398 } 1399 1400 static int 1401 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, 1402 struct sglist *sg, int readable, int writable, int flag) 1403 { 1404 struct virtqueue *vq; 1405 int error; 1406 1407 vq = sc->vtscsi_control_vq; 1408 1409 MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); 1410 1411 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1412 if (error) { 1413 /* 1414 * Return EAGAIN when the virtqueue does not have enough 1415 * descriptors available. 1416 */ 1417 if (error == ENOSPC || error == EMSGSIZE) 1418 error = EAGAIN; 1419 1420 return (error); 1421 } 1422 1423 virtqueue_notify(vq); 1424 if (flag == VTSCSI_EXECUTE_POLL) 1425 vtscsi_poll_ctrl_req(sc, req); 1426 1427 return (0); 1428 } 1429 1430 static void 1431 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, 1432 struct vtscsi_request *req) 1433 { 1434 union ccb *ccb; 1435 struct ccb_hdr *ccbh; 1436 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1437 1438 ccb = req->vsr_ccb; 1439 ccbh = &ccb->ccb_h; 1440 tmf_resp = &req->vsr_tmf_resp; 1441 1442 switch (tmf_resp->response) { 1443 case VIRTIO_SCSI_S_FUNCTION_COMPLETE: 1444 ccbh->status = CAM_REQ_CMP; 1445 break; 1446 case VIRTIO_SCSI_S_FUNCTION_REJECTED: 1447 ccbh->status = CAM_UA_ABORT; 1448 break; 1449 default: 1450 ccbh->status = CAM_REQ_CMP_ERR; 1451 break; 1452 } 1453 1454 xpt_done(ccb); 1455 vtscsi_enqueue_request(sc, req); 1456 } 1457 1458 static int 1459 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, 1460 struct vtscsi_request *req) 1461 { 1462 struct sglist *sg; 1463 struct ccb_abort *cab; 1464 struct ccb_hdr *ccbh; 1465 struct ccb_hdr *abort_ccbh; 1466 struct vtscsi_request *abort_req; 1467 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1468 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1469 int error; 1470 1471 sg = sc->vtscsi_sglist; 1472 cab = &req->vsr_ccb->cab; 1473 ccbh = &cab->ccb_h; 1474 tmf_req = &req->vsr_tmf_req; 1475 tmf_resp = &req->vsr_tmf_resp; 1476 1477 /* CCB header and request that's to be aborted. */ 1478 abort_ccbh = &cab->abort_ccb->ccb_h; 1479 abort_req = abort_ccbh->ccbh_vtscsi_req; 1480 1481 if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { 1482 error = EINVAL; 1483 goto fail; 1484 } 1485 1486 /* Only attempt to abort requests that could be in-flight. */ 1487 if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { 1488 error = EALREADY; 1489 goto fail; 1490 } 1491 1492 abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; 1493 if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1494 callout_stop(&abort_req->vsr_callout); 1495 1496 vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1497 (uintptr_t) abort_ccbh, tmf_req); 1498 1499 sglist_reset(sg); 1500 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1501 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1502 1503 req->vsr_complete = vtscsi_complete_abort_task_cmd; 1504 tmf_resp->response = -1; 1505 1506 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1507 VTSCSI_EXECUTE_ASYNC); 1508 1509 fail: 1510 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " 1511 "abort_req=%p\n", error, req, abort_ccbh, abort_req); 1512 1513 return (error); 1514 } 1515 1516 static void 1517 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, 1518 struct vtscsi_request *req) 1519 { 1520 union ccb *ccb; 1521 struct ccb_hdr *ccbh; 1522 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1523 1524 ccb = req->vsr_ccb; 1525 ccbh = &ccb->ccb_h; 1526 tmf_resp = &req->vsr_tmf_resp; 1527 1528 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", 1529 req, ccb, tmf_resp->response); 1530 1531 if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { 1532 ccbh->status = CAM_REQ_CMP; 1533 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, 1534 ccbh->target_lun); 1535 } else 1536 ccbh->status = CAM_REQ_CMP_ERR; 1537 1538 xpt_done(ccb); 1539 vtscsi_enqueue_request(sc, req); 1540 } 1541 1542 static int 1543 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, 1544 struct vtscsi_request *req) 1545 { 1546 struct sglist *sg; 1547 struct ccb_resetdev *crd; 1548 struct ccb_hdr *ccbh; 1549 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1550 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1551 uint32_t subtype; 1552 int error; 1553 1554 sg = sc->vtscsi_sglist; 1555 crd = &req->vsr_ccb->crd; 1556 ccbh = &crd->ccb_h; 1557 tmf_req = &req->vsr_tmf_req; 1558 tmf_resp = &req->vsr_tmf_resp; 1559 1560 if (ccbh->target_lun == CAM_LUN_WILDCARD) 1561 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; 1562 else 1563 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; 1564 1565 vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); 1566 1567 sglist_reset(sg); 1568 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1569 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1570 1571 req->vsr_complete = vtscsi_complete_reset_dev_cmd; 1572 tmf_resp->response = -1; 1573 1574 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1575 VTSCSI_EXECUTE_ASYNC); 1576 1577 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", 1578 error, req, ccbh); 1579 1580 return (error); 1581 } 1582 1583 static void 1584 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) 1585 { 1586 1587 *target_id = lun[1]; 1588 *lun_id = (lun[2] << 8) | lun[3]; 1589 } 1590 1591 static void 1592 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) 1593 { 1594 1595 lun[0] = 1; 1596 lun[1] = ccbh->target_id; 1597 lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); 1598 lun[3] = ccbh->target_lun & 0xFF; 1599 } 1600 1601 static void 1602 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, 1603 struct virtio_scsi_cmd_req *cmd_req) 1604 { 1605 uint8_t attr; 1606 1607 switch (csio->tag_action) { 1608 case MSG_HEAD_OF_Q_TAG: 1609 attr = VIRTIO_SCSI_S_HEAD; 1610 break; 1611 case MSG_ORDERED_Q_TAG: 1612 attr = VIRTIO_SCSI_S_ORDERED; 1613 break; 1614 case MSG_ACA_TASK: 1615 attr = VIRTIO_SCSI_S_ACA; 1616 break; 1617 default: /* MSG_SIMPLE_Q_TAG */ 1618 attr = VIRTIO_SCSI_S_SIMPLE; 1619 break; 1620 } 1621 1622 vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); 1623 cmd_req->tag = (uintptr_t) csio; 1624 cmd_req->task_attr = attr; 1625 1626 memcpy(cmd_req->cdb, 1627 csio->ccb_h.flags & CAM_CDB_POINTER ? 1628 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, 1629 csio->cdb_len); 1630 } 1631 1632 static void 1633 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, 1634 uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) 1635 { 1636 1637 vtscsi_set_request_lun(ccbh, tmf_req->lun); 1638 1639 tmf_req->type = VIRTIO_SCSI_T_TMF; 1640 tmf_req->subtype = subtype; 1641 tmf_req->tag = tag; 1642 } 1643 1644 static void 1645 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) 1646 { 1647 int frozen; 1648 1649 frozen = sc->vtscsi_frozen; 1650 1651 if (reason & VTSCSI_REQUEST && 1652 (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) 1653 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; 1654 1655 if (reason & VTSCSI_REQUEST_VQ && 1656 (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) 1657 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; 1658 1659 /* Freeze the SIMQ if transitioned to frozen. */ 1660 if (frozen == 0 && sc->vtscsi_frozen != 0) { 1661 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); 1662 xpt_freeze_simq(sc->vtscsi_sim, 1); 1663 } 1664 } 1665 1666 static int 1667 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) 1668 { 1669 int thawed; 1670 1671 if (sc->vtscsi_frozen == 0 || reason == 0) 1672 return (0); 1673 1674 if (reason & VTSCSI_REQUEST && 1675 sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) 1676 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; 1677 1678 if (reason & VTSCSI_REQUEST_VQ && 1679 sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) 1680 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; 1681 1682 thawed = sc->vtscsi_frozen == 0; 1683 if (thawed != 0) 1684 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); 1685 1686 return (thawed); 1687 } 1688 1689 static void 1690 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, 1691 target_id_t target_id, lun_id_t lun_id) 1692 { 1693 struct cam_path *path; 1694 1695 /* Use the wildcard path from our softc for bus announcements. */ 1696 if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { 1697 xpt_async(ac_code, sc->vtscsi_path, NULL); 1698 return; 1699 } 1700 1701 if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), 1702 target_id, lun_id) != CAM_REQ_CMP) { 1703 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); 1704 return; 1705 } 1706 1707 xpt_async(ac_code, path, NULL); 1708 xpt_free_path(path); 1709 } 1710 1711 static void 1712 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, 1713 lun_id_t lun_id) 1714 { 1715 union ccb *ccb; 1716 cam_status status; 1717 1718 ccb = xpt_alloc_ccb_nowait(); 1719 if (ccb == NULL) { 1720 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1721 return; 1722 } 1723 1724 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1725 cam_sim_path(sc->vtscsi_sim), target_id, lun_id); 1726 if (status != CAM_REQ_CMP) { 1727 xpt_free_ccb(ccb); 1728 return; 1729 } 1730 1731 xpt_rescan(ccb); 1732 } 1733 1734 static void 1735 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) 1736 { 1737 1738 vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); 1739 } 1740 1741 static void 1742 vtscsi_transport_reset_event(struct vtscsi_softc *sc, 1743 struct virtio_scsi_event *event) 1744 { 1745 target_id_t target_id; 1746 lun_id_t lun_id; 1747 1748 vtscsi_get_request_lun(event->lun, &target_id, &lun_id); 1749 1750 switch (event->reason) { 1751 case VIRTIO_SCSI_EVT_RESET_RESCAN: 1752 case VIRTIO_SCSI_EVT_RESET_REMOVED: 1753 vtscsi_execute_rescan(sc, target_id, lun_id); 1754 break; 1755 default: 1756 device_printf(sc->vtscsi_dev, 1757 "unhandled transport event reason: %d\n", event->reason); 1758 break; 1759 } 1760 } 1761 1762 static void 1763 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) 1764 { 1765 int error; 1766 1767 if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { 1768 switch (event->event) { 1769 case VIRTIO_SCSI_T_TRANSPORT_RESET: 1770 vtscsi_transport_reset_event(sc, event); 1771 break; 1772 default: 1773 device_printf(sc->vtscsi_dev, 1774 "unhandled event: %d\n", event->event); 1775 break; 1776 } 1777 } else 1778 vtscsi_execute_rescan_bus(sc); 1779 1780 /* 1781 * This should always be successful since the buffer 1782 * was just dequeued. 1783 */ 1784 error = vtscsi_enqueue_event_buf(sc, event); 1785 KASSERT(error == 0, 1786 ("cannot requeue event buffer: %d", error)); 1787 } 1788 1789 static int 1790 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, 1791 struct virtio_scsi_event *event) 1792 { 1793 struct sglist *sg; 1794 struct virtqueue *vq; 1795 int size, error; 1796 1797 sg = sc->vtscsi_sglist; 1798 vq = sc->vtscsi_event_vq; 1799 size = sc->vtscsi_event_buf_size; 1800 1801 bzero(event, size); 1802 1803 sglist_reset(sg); 1804 error = sglist_append(sg, event, size); 1805 if (error) 1806 return (error); 1807 1808 error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); 1809 if (error) 1810 return (error); 1811 1812 virtqueue_notify(vq); 1813 1814 return (0); 1815 } 1816 1817 static int 1818 vtscsi_init_event_vq(struct vtscsi_softc *sc) 1819 { 1820 struct virtio_scsi_event *event; 1821 int i, size, error; 1822 1823 /* 1824 * The first release of QEMU with VirtIO SCSI support would crash 1825 * when attempting to notify the event virtqueue. This was fixed 1826 * when hotplug support was added. 1827 */ 1828 if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) 1829 size = sc->vtscsi_event_buf_size; 1830 else 1831 size = 0; 1832 1833 if (size < sizeof(struct virtio_scsi_event)) 1834 return (0); 1835 1836 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1837 event = &sc->vtscsi_event_bufs[i]; 1838 1839 error = vtscsi_enqueue_event_buf(sc, event); 1840 if (error) 1841 break; 1842 } 1843 1844 /* 1845 * Even just one buffer is enough. Missed events are 1846 * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. 1847 */ 1848 if (i > 0) 1849 error = 0; 1850 1851 return (error); 1852 } 1853 1854 static void 1855 vtscsi_reinit_event_vq(struct vtscsi_softc *sc) 1856 { 1857 struct virtio_scsi_event *event; 1858 int i, error; 1859 1860 if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || 1861 sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) 1862 return; 1863 1864 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1865 event = &sc->vtscsi_event_bufs[i]; 1866 1867 error = vtscsi_enqueue_event_buf(sc, event); 1868 if (error) 1869 break; 1870 } 1871 1872 KASSERT(i > 0, ("cannot reinit event vq: %d", error)); 1873 } 1874 1875 static void 1876 vtscsi_drain_event_vq(struct vtscsi_softc *sc) 1877 { 1878 struct virtqueue *vq; 1879 int last; 1880 1881 vq = sc->vtscsi_event_vq; 1882 last = 0; 1883 1884 while (virtqueue_drain(vq, &last) != NULL) 1885 ; 1886 1887 KASSERT(virtqueue_empty(vq), ("eventvq not empty")); 1888 } 1889 1890 static void 1891 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) 1892 { 1893 1894 VTSCSI_LOCK_OWNED(sc); 1895 1896 if (sc->vtscsi_request_vq != NULL) 1897 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1898 if (sc->vtscsi_control_vq != NULL) 1899 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1900 } 1901 1902 static void 1903 vtscsi_complete_vqs(struct vtscsi_softc *sc) 1904 { 1905 1906 VTSCSI_LOCK(sc); 1907 vtscsi_complete_vqs_locked(sc); 1908 VTSCSI_UNLOCK(sc); 1909 } 1910 1911 static void 1912 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 1913 { 1914 union ccb *ccb; 1915 int detach; 1916 1917 ccb = req->vsr_ccb; 1918 1919 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); 1920 1921 /* 1922 * The callout must be drained when detaching since the request is 1923 * about to be freed. The VTSCSI_MTX must not be held for this in 1924 * case the callout is pending because there is a deadlock potential. 1925 * Otherwise, the virtqueue is being drained because of a bus reset 1926 * so we only need to attempt to stop the callouts. 1927 */ 1928 detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; 1929 if (detach != 0) 1930 VTSCSI_LOCK_NOTOWNED(sc); 1931 else 1932 VTSCSI_LOCK_OWNED(sc); 1933 1934 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { 1935 if (detach != 0) 1936 callout_drain(&req->vsr_callout); 1937 else 1938 callout_stop(&req->vsr_callout); 1939 } 1940 1941 if (ccb != NULL) { 1942 if (detach != 0) { 1943 VTSCSI_LOCK(sc); 1944 ccb->ccb_h.status = CAM_NO_HBA; 1945 } else 1946 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1947 xpt_done(ccb); 1948 if (detach != 0) 1949 VTSCSI_UNLOCK(sc); 1950 } 1951 1952 vtscsi_enqueue_request(sc, req); 1953 } 1954 1955 static void 1956 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 1957 { 1958 struct vtscsi_request *req; 1959 int last; 1960 1961 last = 0; 1962 1963 vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); 1964 1965 while ((req = virtqueue_drain(vq, &last)) != NULL) 1966 vtscsi_cancel_request(sc, req); 1967 1968 KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1969 } 1970 1971 static void 1972 vtscsi_drain_vqs(struct vtscsi_softc *sc) 1973 { 1974 1975 if (sc->vtscsi_control_vq != NULL) 1976 vtscsi_drain_vq(sc, sc->vtscsi_control_vq); 1977 if (sc->vtscsi_request_vq != NULL) 1978 vtscsi_drain_vq(sc, sc->vtscsi_request_vq); 1979 if (sc->vtscsi_event_vq != NULL) 1980 vtscsi_drain_event_vq(sc); 1981 } 1982 1983 static void 1984 vtscsi_stop(struct vtscsi_softc *sc) 1985 { 1986 1987 vtscsi_disable_vqs_intr(sc); 1988 virtio_stop(sc->vtscsi_dev); 1989 } 1990 1991 static int 1992 vtscsi_reset_bus(struct vtscsi_softc *sc) 1993 { 1994 int error; 1995 1996 VTSCSI_LOCK_OWNED(sc); 1997 1998 if (vtscsi_bus_reset_disable != 0) { 1999 device_printf(sc->vtscsi_dev, "bus reset disabled\n"); 2000 return (0); 2001 } 2002 2003 sc->vtscsi_flags |= VTSCSI_FLAG_RESET; 2004 2005 /* 2006 * vtscsi_stop() will cause the in-flight requests to be canceled. 2007 * Those requests are then completed here so CAM will retry them 2008 * after the reset is complete. 2009 */ 2010 vtscsi_stop(sc); 2011 vtscsi_complete_vqs_locked(sc); 2012 2013 /* Rid the virtqueues of any remaining requests. */ 2014 vtscsi_drain_vqs(sc); 2015 2016 /* 2017 * Any resource shortage that froze the SIMQ cannot persist across 2018 * a bus reset so ensure it gets thawed here. 2019 */ 2020 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 2021 xpt_release_simq(sc->vtscsi_sim, 0); 2022 2023 error = vtscsi_reinit(sc); 2024 if (error) { 2025 device_printf(sc->vtscsi_dev, 2026 "reinitialization failed, stopping device...\n"); 2027 vtscsi_stop(sc); 2028 } else 2029 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, 2030 CAM_LUN_WILDCARD); 2031 2032 sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; 2033 2034 return (error); 2035 } 2036 2037 static void 2038 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2039 { 2040 2041 #ifdef INVARIANTS 2042 int req_nsegs, resp_nsegs; 2043 2044 req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2045 resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2046 2047 KASSERT(req_nsegs == 1, ("request crossed page boundary")); 2048 KASSERT(resp_nsegs == 1, ("response crossed page boundary")); 2049 #endif 2050 2051 req->vsr_softc = sc; 2052 callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0); 2053 } 2054 2055 static int 2056 vtscsi_alloc_requests(struct vtscsi_softc *sc) 2057 { 2058 struct vtscsi_request *req; 2059 int i, nreqs; 2060 2061 /* 2062 * Commands destined for either the request or control queues come 2063 * from the same SIM queue. Use the size of the request virtqueue 2064 * as it (should) be much more frequently used. Some additional 2065 * requests are allocated for internal (TMF) use. 2066 */ 2067 nreqs = virtqueue_size(sc->vtscsi_request_vq); 2068 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) 2069 nreqs /= VTSCSI_MIN_SEGMENTS; 2070 nreqs += VTSCSI_RESERVED_REQUESTS; 2071 2072 for (i = 0; i < nreqs; i++) { 2073 req = malloc(sizeof(struct vtscsi_request), M_DEVBUF, 2074 M_NOWAIT); 2075 if (req == NULL) 2076 return (ENOMEM); 2077 2078 vtscsi_init_request(sc, req); 2079 2080 sc->vtscsi_nrequests++; 2081 vtscsi_enqueue_request(sc, req); 2082 } 2083 2084 return (0); 2085 } 2086 2087 static void 2088 vtscsi_free_requests(struct vtscsi_softc *sc) 2089 { 2090 struct vtscsi_request *req; 2091 2092 while ((req = vtscsi_dequeue_request(sc)) != NULL) { 2093 KASSERT(callout_active(&req->vsr_callout) == 0, 2094 ("request callout still active")); 2095 2096 sc->vtscsi_nrequests--; 2097 free(req, M_DEVBUF); 2098 } 2099 2100 KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", 2101 sc->vtscsi_nrequests)); 2102 } 2103 2104 static void 2105 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2106 { 2107 2108 KASSERT(req->vsr_softc == sc, 2109 ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); 2110 2111 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2112 2113 /* A request is available so the SIMQ could be released. */ 2114 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) 2115 xpt_release_simq(sc->vtscsi_sim, 1); 2116 2117 req->vsr_ccb = NULL; 2118 req->vsr_complete = NULL; 2119 req->vsr_ptr0 = NULL; 2120 req->vsr_state = VTSCSI_REQ_STATE_FREE; 2121 req->vsr_flags = 0; 2122 2123 bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2124 bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2125 2126 /* 2127 * We insert at the tail of the queue in order to make it 2128 * very unlikely a request will be reused if we race with 2129 * stopping its callout handler. 2130 */ 2131 TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); 2132 } 2133 2134 static struct vtscsi_request * 2135 vtscsi_dequeue_request(struct vtscsi_softc *sc) 2136 { 2137 struct vtscsi_request *req; 2138 2139 req = TAILQ_FIRST(&sc->vtscsi_req_free); 2140 if (req != NULL) { 2141 req->vsr_state = VTSCSI_REQ_STATE_INUSE; 2142 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); 2143 } else 2144 sc->vtscsi_stats.dequeue_no_requests++; 2145 2146 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2147 2148 return (req); 2149 } 2150 2151 static void 2152 vtscsi_complete_request(struct vtscsi_request *req) 2153 { 2154 2155 if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) 2156 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; 2157 2158 if (req->vsr_complete != NULL) 2159 req->vsr_complete(req->vsr_softc, req); 2160 } 2161 2162 static void 2163 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2164 { 2165 struct vtscsi_request *req; 2166 2167 VTSCSI_LOCK_OWNED(sc); 2168 2169 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) 2170 vtscsi_complete_request(req); 2171 } 2172 2173 static void 2174 vtscsi_control_vq_intr(void *xsc) 2175 { 2176 struct vtscsi_softc *sc; 2177 struct virtqueue *vq; 2178 2179 sc = xsc; 2180 vq = sc->vtscsi_control_vq; 2181 2182 again: 2183 VTSCSI_LOCK(sc); 2184 2185 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 2186 2187 if (virtqueue_enable_intr(vq) != 0) { 2188 virtqueue_disable_intr(vq); 2189 VTSCSI_UNLOCK(sc); 2190 goto again; 2191 } 2192 2193 VTSCSI_UNLOCK(sc); 2194 } 2195 2196 static void 2197 vtscsi_event_vq_intr(void *xsc) 2198 { 2199 struct vtscsi_softc *sc; 2200 struct virtqueue *vq; 2201 struct virtio_scsi_event *event; 2202 2203 sc = xsc; 2204 vq = sc->vtscsi_event_vq; 2205 2206 again: 2207 VTSCSI_LOCK(sc); 2208 2209 while ((event = virtqueue_dequeue(vq, NULL)) != NULL) 2210 vtscsi_handle_event(sc, event); 2211 2212 if (virtqueue_enable_intr(vq) != 0) { 2213 virtqueue_disable_intr(vq); 2214 VTSCSI_UNLOCK(sc); 2215 goto again; 2216 } 2217 2218 VTSCSI_UNLOCK(sc); 2219 } 2220 2221 static void 2222 vtscsi_request_vq_intr(void *xsc) 2223 { 2224 struct vtscsi_softc *sc; 2225 struct virtqueue *vq; 2226 2227 sc = xsc; 2228 vq = sc->vtscsi_request_vq; 2229 2230 again: 2231 VTSCSI_LOCK(sc); 2232 2233 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 2234 2235 if (virtqueue_enable_intr(vq) != 0) { 2236 virtqueue_disable_intr(vq); 2237 VTSCSI_UNLOCK(sc); 2238 goto again; 2239 } 2240 2241 VTSCSI_UNLOCK(sc); 2242 } 2243 2244 static void 2245 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) 2246 { 2247 2248 virtqueue_disable_intr(sc->vtscsi_control_vq); 2249 virtqueue_disable_intr(sc->vtscsi_event_vq); 2250 virtqueue_disable_intr(sc->vtscsi_request_vq); 2251 } 2252 2253 static void 2254 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) 2255 { 2256 2257 virtqueue_enable_intr(sc->vtscsi_control_vq); 2258 virtqueue_enable_intr(sc->vtscsi_event_vq); 2259 virtqueue_enable_intr(sc->vtscsi_request_vq); 2260 } 2261 2262 static void 2263 vtscsi_get_tunables(struct vtscsi_softc *sc) 2264 { 2265 char tmpstr[64]; 2266 2267 TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); 2268 2269 snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", 2270 device_get_unit(sc->vtscsi_dev)); 2271 TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); 2272 } 2273 2274 static void 2275 vtscsi_add_sysctl(struct vtscsi_softc *sc) 2276 { 2277 device_t dev; 2278 struct vtscsi_statistics *stats; 2279 struct sysctl_ctx_list *ctx; 2280 struct sysctl_oid *tree; 2281 struct sysctl_oid_list *child; 2282 2283 dev = sc->vtscsi_dev; 2284 stats = &sc->vtscsi_stats; 2285 ctx = device_get_sysctl_ctx(dev); 2286 tree = device_get_sysctl_tree(dev); 2287 child = SYSCTL_CHILDREN(tree); 2288 2289 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 2290 CTLFLAG_RW, &sc->vtscsi_debug, 0, 2291 "Debug level"); 2292 2293 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", 2294 CTLFLAG_RD, &stats->scsi_cmd_timeouts, 2295 "SCSI command timeouts"); 2296 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", 2297 CTLFLAG_RD, &stats->dequeue_no_requests, 2298 "No available requests to dequeue"); 2299 } 2300 2301 static void 2302 vtscsi_printf_req(struct vtscsi_request *req, const char *func, 2303 const char *fmt, ...) 2304 { 2305 struct vtscsi_softc *sc; 2306 union ccb *ccb; 2307 struct sbuf sb; 2308 va_list ap; 2309 char str[192]; 2310 char path_str[64]; 2311 2312 if (req == NULL) 2313 return; 2314 2315 sc = req->vsr_softc; 2316 ccb = req->vsr_ccb; 2317 2318 va_start(ap, fmt); 2319 sbuf_new(&sb, str, sizeof(str), 0); 2320 2321 if (ccb == NULL) { 2322 sbuf_printf(&sb, "(noperiph:%s%d:%u): ", 2323 cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), 2324 cam_sim_bus(sc->vtscsi_sim)); 2325 } else { 2326 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); 2327 sbuf_cat(&sb, path_str); 2328 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2329 scsi_command_string(&ccb->csio, &sb); 2330 sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); 2331 } 2332 } 2333 2334 sbuf_vprintf(&sb, fmt, ap); 2335 va_end(ap); 2336 2337 sbuf_finish(&sb); 2338 printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, 2339 sbuf_data(&sb)); 2340 } 2341