1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for VirtIO SCSI devices. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/kthread.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/sglist.h> 41 #include <sys/sysctl.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/callout.h> 45 #include <sys/queue.h> 46 #include <sys/sbuf.h> 47 48 #include <machine/stdarg.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 #include <sys/bus.h> 53 #include <sys/rman.h> 54 55 #include <cam/cam.h> 56 #include <cam/cam_ccb.h> 57 #include <cam/cam_sim.h> 58 #include <cam/cam_periph.h> 59 #include <cam/cam_xpt_sim.h> 60 #include <cam/cam_debug.h> 61 #include <cam/scsi/scsi_all.h> 62 #include <cam/scsi/scsi_message.h> 63 64 #include <dev/virtio/virtio.h> 65 #include <dev/virtio/virtqueue.h> 66 #include <dev/virtio/scsi/virtio_scsi.h> 67 #include <dev/virtio/scsi/virtio_scsivar.h> 68 69 #include "virtio_if.h" 70 71 static int vtscsi_modevent(module_t, int, void *); 72 73 static int vtscsi_probe(device_t); 74 static int vtscsi_attach(device_t); 75 static int vtscsi_detach(device_t); 76 static int vtscsi_suspend(device_t); 77 static int vtscsi_resume(device_t); 78 79 static void vtscsi_negotiate_features(struct vtscsi_softc *); 80 static void vtscsi_read_config(struct vtscsi_softc *, 81 struct virtio_scsi_config *); 82 static int vtscsi_maximum_segments(struct vtscsi_softc *, int); 83 static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); 84 static void vtscsi_check_sizes(struct vtscsi_softc *); 85 static void vtscsi_write_device_config(struct vtscsi_softc *); 86 static int vtscsi_reinit(struct vtscsi_softc *); 87 88 static int vtscsi_alloc_cam(struct vtscsi_softc *); 89 static int vtscsi_register_cam(struct vtscsi_softc *); 90 static void vtscsi_free_cam(struct vtscsi_softc *); 91 static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); 92 static int vtscsi_register_async(struct vtscsi_softc *); 93 static void vtscsi_deregister_async(struct vtscsi_softc *); 94 static void vtscsi_cam_action(struct cam_sim *, union ccb *); 95 static void vtscsi_cam_poll(struct cam_sim *); 96 97 static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, 98 union ccb *); 99 static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, 100 union ccb *); 101 static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); 102 static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); 103 static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); 104 static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, 105 struct cam_sim *, union ccb *); 106 107 static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, 108 struct sglist *, struct ccb_scsiio *); 109 static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, 110 struct vtscsi_request *, int *, int *); 111 static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, 112 struct vtscsi_request *); 113 static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); 114 static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, 115 struct vtscsi_request *); 116 static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, 117 struct vtscsi_request *); 118 static void vtscsi_timedout_scsi_cmd(void *); 119 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); 120 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, 121 struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); 122 static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, 123 struct vtscsi_request *); 124 125 static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, 126 struct vtscsi_request *); 127 static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, 128 struct vtscsi_request *, struct sglist *, int, int, int); 129 static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, 130 struct vtscsi_request *); 131 static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, 132 struct vtscsi_request *); 133 static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, 134 struct vtscsi_request *); 135 136 static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); 137 static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); 138 static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, 139 struct virtio_scsi_cmd_req *); 140 static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, 141 uintptr_t, struct virtio_scsi_ctrl_tmf_req *); 142 143 static void vtscsi_freeze_simq(struct vtscsi_softc *, int); 144 static int vtscsi_thaw_simq(struct vtscsi_softc *, int); 145 146 static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, 147 lun_id_t); 148 static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, 149 lun_id_t); 150 static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); 151 152 static void vtscsi_handle_event(struct vtscsi_softc *, 153 struct virtio_scsi_event *); 154 static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, 155 struct virtio_scsi_event *); 156 static int vtscsi_init_event_vq(struct vtscsi_softc *); 157 static void vtscsi_reinit_event_vq(struct vtscsi_softc *); 158 static void vtscsi_drain_event_vq(struct vtscsi_softc *); 159 160 static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); 161 static void vtscsi_complete_vqs(struct vtscsi_softc *); 162 static void vtscsi_drain_vqs(struct vtscsi_softc *); 163 static void vtscsi_cancel_request(struct vtscsi_softc *, 164 struct vtscsi_request *); 165 static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); 166 static void vtscsi_stop(struct vtscsi_softc *); 167 static int vtscsi_reset_bus(struct vtscsi_softc *); 168 169 static void vtscsi_init_request(struct vtscsi_softc *, 170 struct vtscsi_request *); 171 static int vtscsi_alloc_requests(struct vtscsi_softc *); 172 static void vtscsi_free_requests(struct vtscsi_softc *); 173 static void vtscsi_enqueue_request(struct vtscsi_softc *, 174 struct vtscsi_request *); 175 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); 176 177 static void vtscsi_complete_request(struct vtscsi_request *); 178 static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); 179 180 static void vtscsi_control_vq_intr(void *); 181 static void vtscsi_event_vq_intr(void *); 182 static void vtscsi_request_vq_intr(void *); 183 static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); 184 static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); 185 186 static void vtscsi_get_tunables(struct vtscsi_softc *); 187 static void vtscsi_add_sysctl(struct vtscsi_softc *); 188 189 static void vtscsi_printf_req(struct vtscsi_request *, const char *, 190 const char *, ...); 191 192 /* Global tunables. */ 193 /* 194 * The current QEMU VirtIO SCSI implementation does not cancel in-flight 195 * IO during virtio_stop(). So in-flight requests still complete after the 196 * device reset. We would have to wait for all the in-flight IO to complete, 197 * which defeats the typical purpose of a bus reset. We could simulate the 198 * bus reset with either I_T_NEXUS_RESET of all the targets, or with 199 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the 200 * control virtqueue). But this isn't very useful if things really go off 201 * the rails, so default to disabled for now. 202 */ 203 static int vtscsi_bus_reset_disable = 1; 204 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); 205 206 static struct virtio_feature_desc vtscsi_feature_desc[] = { 207 { VIRTIO_SCSI_F_INOUT, "InOut" }, 208 { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, 209 210 { 0, NULL } 211 }; 212 213 static device_method_t vtscsi_methods[] = { 214 /* Device methods. */ 215 DEVMETHOD(device_probe, vtscsi_probe), 216 DEVMETHOD(device_attach, vtscsi_attach), 217 DEVMETHOD(device_detach, vtscsi_detach), 218 DEVMETHOD(device_suspend, vtscsi_suspend), 219 DEVMETHOD(device_resume, vtscsi_resume), 220 221 DEVMETHOD_END 222 }; 223 224 static driver_t vtscsi_driver = { 225 "vtscsi", 226 vtscsi_methods, 227 sizeof(struct vtscsi_softc) 228 }; 229 static devclass_t vtscsi_devclass; 230 231 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, 232 vtscsi_modevent, 0); 233 MODULE_VERSION(virtio_scsi, 1); 234 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); 235 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); 236 237 VIRTIO_SIMPLE_PNPTABLE(virtio_scsi, VIRTIO_ID_SCSI, "VirtIO SCSI Adapter"); 238 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_scsi); 239 240 static int 241 vtscsi_modevent(module_t mod, int type, void *unused) 242 { 243 int error; 244 245 switch (type) { 246 case MOD_LOAD: 247 case MOD_QUIESCE: 248 case MOD_UNLOAD: 249 case MOD_SHUTDOWN: 250 error = 0; 251 break; 252 default: 253 error = EOPNOTSUPP; 254 break; 255 } 256 257 return (error); 258 } 259 260 static int 261 vtscsi_probe(device_t dev) 262 { 263 return (VIRTIO_SIMPLE_PROBE(dev, virtio_scsi)); 264 } 265 266 static int 267 vtscsi_attach(device_t dev) 268 { 269 struct vtscsi_softc *sc; 270 struct virtio_scsi_config scsicfg; 271 int error; 272 273 sc = device_get_softc(dev); 274 sc->vtscsi_dev = dev; 275 276 VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); 277 TAILQ_INIT(&sc->vtscsi_req_free); 278 279 vtscsi_get_tunables(sc); 280 vtscsi_add_sysctl(sc); 281 282 virtio_set_feature_desc(dev, vtscsi_feature_desc); 283 vtscsi_negotiate_features(sc); 284 285 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 286 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; 287 if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) 288 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; 289 if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) 290 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; 291 292 vtscsi_read_config(sc, &scsicfg); 293 294 sc->vtscsi_max_channel = scsicfg.max_channel; 295 sc->vtscsi_max_target = scsicfg.max_target; 296 sc->vtscsi_max_lun = scsicfg.max_lun; 297 sc->vtscsi_event_buf_size = scsicfg.event_info_size; 298 299 vtscsi_write_device_config(sc); 300 301 sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); 302 sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); 303 if (sc->vtscsi_sglist == NULL) { 304 error = ENOMEM; 305 device_printf(dev, "cannot allocate sglist\n"); 306 goto fail; 307 } 308 309 error = vtscsi_alloc_virtqueues(sc); 310 if (error) { 311 device_printf(dev, "cannot allocate virtqueues\n"); 312 goto fail; 313 } 314 315 vtscsi_check_sizes(sc); 316 317 error = vtscsi_init_event_vq(sc); 318 if (error) { 319 device_printf(dev, "cannot populate the eventvq\n"); 320 goto fail; 321 } 322 323 error = vtscsi_alloc_requests(sc); 324 if (error) { 325 device_printf(dev, "cannot allocate requests\n"); 326 goto fail; 327 } 328 329 error = vtscsi_alloc_cam(sc); 330 if (error) { 331 device_printf(dev, "cannot allocate CAM structures\n"); 332 goto fail; 333 } 334 335 error = virtio_setup_intr(dev, INTR_TYPE_CAM); 336 if (error) { 337 device_printf(dev, "cannot setup virtqueue interrupts\n"); 338 goto fail; 339 } 340 341 vtscsi_enable_vqs_intr(sc); 342 343 /* 344 * Register with CAM after interrupts are enabled so we will get 345 * notified of the probe responses. 346 */ 347 error = vtscsi_register_cam(sc); 348 if (error) { 349 device_printf(dev, "cannot register with CAM\n"); 350 goto fail; 351 } 352 353 fail: 354 if (error) 355 vtscsi_detach(dev); 356 357 return (error); 358 } 359 360 static int 361 vtscsi_detach(device_t dev) 362 { 363 struct vtscsi_softc *sc; 364 365 sc = device_get_softc(dev); 366 367 VTSCSI_LOCK(sc); 368 sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; 369 if (device_is_attached(dev)) 370 vtscsi_stop(sc); 371 VTSCSI_UNLOCK(sc); 372 373 vtscsi_complete_vqs(sc); 374 vtscsi_drain_vqs(sc); 375 376 vtscsi_free_cam(sc); 377 vtscsi_free_requests(sc); 378 379 if (sc->vtscsi_sglist != NULL) { 380 sglist_free(sc->vtscsi_sglist); 381 sc->vtscsi_sglist = NULL; 382 } 383 384 VTSCSI_LOCK_DESTROY(sc); 385 386 return (0); 387 } 388 389 static int 390 vtscsi_suspend(device_t dev) 391 { 392 393 return (0); 394 } 395 396 static int 397 vtscsi_resume(device_t dev) 398 { 399 400 return (0); 401 } 402 403 static void 404 vtscsi_negotiate_features(struct vtscsi_softc *sc) 405 { 406 device_t dev; 407 uint64_t features; 408 409 dev = sc->vtscsi_dev; 410 features = virtio_negotiate_features(dev, VTSCSI_FEATURES); 411 sc->vtscsi_features = features; 412 } 413 414 #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ 415 virtio_read_device_config(_dev, \ 416 offsetof(struct virtio_scsi_config, _field), \ 417 &(_cfg)->_field, sizeof((_cfg)->_field)) \ 418 419 static void 420 vtscsi_read_config(struct vtscsi_softc *sc, 421 struct virtio_scsi_config *scsicfg) 422 { 423 device_t dev; 424 425 dev = sc->vtscsi_dev; 426 427 bzero(scsicfg, sizeof(struct virtio_scsi_config)); 428 429 VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); 430 VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); 431 VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); 432 VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); 433 VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); 434 VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); 435 VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); 436 VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); 437 VTSCSI_GET_CONFIG(dev, max_target, scsicfg); 438 VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); 439 } 440 441 #undef VTSCSI_GET_CONFIG 442 443 static int 444 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) 445 { 446 int nsegs; 447 448 nsegs = VTSCSI_MIN_SEGMENTS; 449 450 if (seg_max > 0) { 451 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1); 452 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) 453 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 454 } else 455 nsegs += 1; 456 457 return (nsegs); 458 } 459 460 static int 461 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) 462 { 463 device_t dev; 464 struct vq_alloc_info vq_info[3]; 465 int nvqs; 466 467 dev = sc->vtscsi_dev; 468 nvqs = 3; 469 470 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc, 471 &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev)); 472 473 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc, 474 &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev)); 475 476 VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, 477 vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq, 478 "%s request", device_get_nameunit(dev)); 479 480 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 481 } 482 483 static void 484 vtscsi_check_sizes(struct vtscsi_softc *sc) 485 { 486 int rqsize; 487 488 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) { 489 /* 490 * Ensure the assertions in virtqueue_enqueue(), 491 * even if the hypervisor reports a bad seg_max. 492 */ 493 rqsize = virtqueue_size(sc->vtscsi_request_vq); 494 if (sc->vtscsi_max_nsegs > rqsize) { 495 device_printf(sc->vtscsi_dev, 496 "clamping seg_max (%d %d)\n", sc->vtscsi_max_nsegs, 497 rqsize); 498 sc->vtscsi_max_nsegs = rqsize; 499 } 500 } 501 } 502 503 static void 504 vtscsi_write_device_config(struct vtscsi_softc *sc) 505 { 506 507 virtio_write_dev_config_4(sc->vtscsi_dev, 508 offsetof(struct virtio_scsi_config, sense_size), 509 VIRTIO_SCSI_SENSE_SIZE); 510 511 /* 512 * This is the size in the virtio_scsi_cmd_req structure. Note 513 * this value (32) is larger than the maximum CAM CDB size (16). 514 */ 515 virtio_write_dev_config_4(sc->vtscsi_dev, 516 offsetof(struct virtio_scsi_config, cdb_size), 517 VIRTIO_SCSI_CDB_SIZE); 518 } 519 520 static int 521 vtscsi_reinit(struct vtscsi_softc *sc) 522 { 523 device_t dev; 524 int error; 525 526 dev = sc->vtscsi_dev; 527 528 error = virtio_reinit(dev, sc->vtscsi_features); 529 if (error == 0) { 530 vtscsi_write_device_config(sc); 531 vtscsi_reinit_event_vq(sc); 532 virtio_reinit_complete(dev); 533 534 vtscsi_enable_vqs_intr(sc); 535 } 536 537 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); 538 539 return (error); 540 } 541 542 static int 543 vtscsi_alloc_cam(struct vtscsi_softc *sc) 544 { 545 device_t dev; 546 struct cam_devq *devq; 547 int openings; 548 549 dev = sc->vtscsi_dev; 550 openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; 551 552 devq = cam_simq_alloc(openings); 553 if (devq == NULL) { 554 device_printf(dev, "cannot allocate SIM queue\n"); 555 return (ENOMEM); 556 } 557 558 sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, 559 "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, 560 openings, devq); 561 if (sc->vtscsi_sim == NULL) { 562 cam_simq_free(devq); 563 device_printf(dev, "cannot allocate SIM\n"); 564 return (ENOMEM); 565 } 566 567 return (0); 568 } 569 570 static int 571 vtscsi_register_cam(struct vtscsi_softc *sc) 572 { 573 device_t dev; 574 int registered, error; 575 576 dev = sc->vtscsi_dev; 577 registered = 0; 578 579 VTSCSI_LOCK(sc); 580 581 if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) { 582 error = ENOMEM; 583 device_printf(dev, "cannot register XPT bus\n"); 584 goto fail; 585 } 586 587 registered = 1; 588 589 if (xpt_create_path(&sc->vtscsi_path, NULL, 590 cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, 591 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 592 error = ENOMEM; 593 device_printf(dev, "cannot create bus path\n"); 594 goto fail; 595 } 596 597 if (vtscsi_register_async(sc) != CAM_REQ_CMP) { 598 error = EIO; 599 device_printf(dev, "cannot register async callback\n"); 600 goto fail; 601 } 602 603 VTSCSI_UNLOCK(sc); 604 605 return (0); 606 607 fail: 608 if (sc->vtscsi_path != NULL) { 609 xpt_free_path(sc->vtscsi_path); 610 sc->vtscsi_path = NULL; 611 } 612 613 if (registered != 0) 614 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 615 616 VTSCSI_UNLOCK(sc); 617 618 return (error); 619 } 620 621 static void 622 vtscsi_free_cam(struct vtscsi_softc *sc) 623 { 624 625 VTSCSI_LOCK(sc); 626 627 if (sc->vtscsi_path != NULL) { 628 vtscsi_deregister_async(sc); 629 630 xpt_free_path(sc->vtscsi_path); 631 sc->vtscsi_path = NULL; 632 633 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 634 } 635 636 if (sc->vtscsi_sim != NULL) { 637 cam_sim_free(sc->vtscsi_sim, 1); 638 sc->vtscsi_sim = NULL; 639 } 640 641 VTSCSI_UNLOCK(sc); 642 } 643 644 static void 645 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) 646 { 647 struct cam_sim *sim; 648 struct vtscsi_softc *sc; 649 650 sim = cb_arg; 651 sc = cam_sim_softc(sim); 652 653 vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); 654 655 /* 656 * TODO Once QEMU supports event reporting, we should 657 * (un)subscribe to events here. 658 */ 659 switch (code) { 660 case AC_FOUND_DEVICE: 661 break; 662 case AC_LOST_DEVICE: 663 break; 664 } 665 } 666 667 static int 668 vtscsi_register_async(struct vtscsi_softc *sc) 669 { 670 struct ccb_setasync csa; 671 672 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 673 csa.ccb_h.func_code = XPT_SASYNC_CB; 674 csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; 675 csa.callback = vtscsi_cam_async; 676 csa.callback_arg = sc->vtscsi_sim; 677 678 xpt_action((union ccb *) &csa); 679 680 return (csa.ccb_h.status); 681 } 682 683 static void 684 vtscsi_deregister_async(struct vtscsi_softc *sc) 685 { 686 struct ccb_setasync csa; 687 688 xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5); 689 csa.ccb_h.func_code = XPT_SASYNC_CB; 690 csa.event_enable = 0; 691 csa.callback = vtscsi_cam_async; 692 csa.callback_arg = sc->vtscsi_sim; 693 694 xpt_action((union ccb *) &csa); 695 } 696 697 static void 698 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) 699 { 700 struct vtscsi_softc *sc; 701 struct ccb_hdr *ccbh; 702 703 sc = cam_sim_softc(sim); 704 ccbh = &ccb->ccb_h; 705 706 VTSCSI_LOCK_OWNED(sc); 707 708 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { 709 /* 710 * The VTSCSI_MTX is briefly dropped between setting 711 * VTSCSI_FLAG_DETACH and deregistering with CAM, so 712 * drop any CCBs that come in during that window. 713 */ 714 ccbh->status = CAM_NO_HBA; 715 xpt_done(ccb); 716 return; 717 } 718 719 switch (ccbh->func_code) { 720 case XPT_SCSI_IO: 721 vtscsi_cam_scsi_io(sc, sim, ccb); 722 break; 723 724 case XPT_SET_TRAN_SETTINGS: 725 ccbh->status = CAM_FUNC_NOTAVAIL; 726 xpt_done(ccb); 727 break; 728 729 case XPT_GET_TRAN_SETTINGS: 730 vtscsi_cam_get_tran_settings(sc, ccb); 731 break; 732 733 case XPT_RESET_BUS: 734 vtscsi_cam_reset_bus(sc, ccb); 735 break; 736 737 case XPT_RESET_DEV: 738 vtscsi_cam_reset_dev(sc, ccb); 739 break; 740 741 case XPT_ABORT: 742 vtscsi_cam_abort(sc, ccb); 743 break; 744 745 case XPT_CALC_GEOMETRY: 746 cam_calc_geometry(&ccb->ccg, 1); 747 xpt_done(ccb); 748 break; 749 750 case XPT_PATH_INQ: 751 vtscsi_cam_path_inquiry(sc, sim, ccb); 752 break; 753 754 default: 755 vtscsi_dprintf(sc, VTSCSI_ERROR, 756 "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); 757 758 ccbh->status = CAM_REQ_INVALID; 759 xpt_done(ccb); 760 break; 761 } 762 } 763 764 static void 765 vtscsi_cam_poll(struct cam_sim *sim) 766 { 767 struct vtscsi_softc *sc; 768 769 sc = cam_sim_softc(sim); 770 771 vtscsi_complete_vqs_locked(sc); 772 } 773 774 static void 775 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, 776 union ccb *ccb) 777 { 778 struct ccb_hdr *ccbh; 779 struct ccb_scsiio *csio; 780 int error; 781 782 ccbh = &ccb->ccb_h; 783 csio = &ccb->csio; 784 785 if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { 786 error = EINVAL; 787 ccbh->status = CAM_REQ_INVALID; 788 goto done; 789 } 790 791 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && 792 (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { 793 error = EINVAL; 794 ccbh->status = CAM_REQ_INVALID; 795 goto done; 796 } 797 798 error = vtscsi_start_scsi_cmd(sc, ccb); 799 800 done: 801 if (error) { 802 vtscsi_dprintf(sc, VTSCSI_ERROR, 803 "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); 804 xpt_done(ccb); 805 } 806 } 807 808 static void 809 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) 810 { 811 struct ccb_trans_settings *cts; 812 struct ccb_trans_settings_scsi *scsi; 813 814 cts = &ccb->cts; 815 scsi = &cts->proto_specific.scsi; 816 817 cts->protocol = PROTO_SCSI; 818 cts->protocol_version = SCSI_REV_SPC3; 819 cts->transport = XPORT_SAS; 820 cts->transport_version = 0; 821 822 scsi->valid = CTS_SCSI_VALID_TQ; 823 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 824 825 ccb->ccb_h.status = CAM_REQ_CMP; 826 xpt_done(ccb); 827 } 828 829 static void 830 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) 831 { 832 int error; 833 834 error = vtscsi_reset_bus(sc); 835 if (error == 0) 836 ccb->ccb_h.status = CAM_REQ_CMP; 837 else 838 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 839 840 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", 841 error, ccb, ccb->ccb_h.status); 842 843 xpt_done(ccb); 844 } 845 846 static void 847 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) 848 { 849 struct ccb_hdr *ccbh; 850 struct vtscsi_request *req; 851 int error; 852 853 ccbh = &ccb->ccb_h; 854 855 req = vtscsi_dequeue_request(sc); 856 if (req == NULL) { 857 error = EAGAIN; 858 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 859 goto fail; 860 } 861 862 req->vsr_ccb = ccb; 863 864 error = vtscsi_execute_reset_dev_cmd(sc, req); 865 if (error == 0) 866 return; 867 868 vtscsi_enqueue_request(sc, req); 869 870 fail: 871 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 872 error, req, ccb); 873 874 if (error == EAGAIN) 875 ccbh->status = CAM_RESRC_UNAVAIL; 876 else 877 ccbh->status = CAM_REQ_CMP_ERR; 878 879 xpt_done(ccb); 880 } 881 882 static void 883 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) 884 { 885 struct vtscsi_request *req; 886 struct ccb_hdr *ccbh; 887 int error; 888 889 ccbh = &ccb->ccb_h; 890 891 req = vtscsi_dequeue_request(sc); 892 if (req == NULL) { 893 error = EAGAIN; 894 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 895 goto fail; 896 } 897 898 req->vsr_ccb = ccb; 899 900 error = vtscsi_execute_abort_task_cmd(sc, req); 901 if (error == 0) 902 return; 903 904 vtscsi_enqueue_request(sc, req); 905 906 fail: 907 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 908 error, req, ccb); 909 910 if (error == EAGAIN) 911 ccbh->status = CAM_RESRC_UNAVAIL; 912 else 913 ccbh->status = CAM_REQ_CMP_ERR; 914 915 xpt_done(ccb); 916 } 917 918 static void 919 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, 920 union ccb *ccb) 921 { 922 device_t dev; 923 struct ccb_pathinq *cpi; 924 925 dev = sc->vtscsi_dev; 926 cpi = &ccb->cpi; 927 928 vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); 929 930 cpi->version_num = 1; 931 cpi->hba_inquiry = PI_TAG_ABLE; 932 cpi->target_sprt = 0; 933 cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 934 if (vtscsi_bus_reset_disable != 0) 935 cpi->hba_misc |= PIM_NOBUSRESET; 936 cpi->hba_eng_cnt = 0; 937 938 cpi->max_target = sc->vtscsi_max_target; 939 cpi->max_lun = sc->vtscsi_max_lun; 940 cpi->initiator_id = cpi->max_target + 1; 941 942 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 943 strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); 944 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 945 946 cpi->unit_number = cam_sim_unit(sim); 947 cpi->bus_id = cam_sim_bus(sim); 948 949 cpi->base_transfer_speed = 300000; 950 951 cpi->protocol = PROTO_SCSI; 952 cpi->protocol_version = SCSI_REV_SPC3; 953 cpi->transport = XPORT_SAS; 954 cpi->transport_version = 0; 955 956 cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * 957 PAGE_SIZE; 958 959 cpi->hba_vendor = virtio_get_vendor(dev); 960 cpi->hba_device = virtio_get_device(dev); 961 cpi->hba_subvendor = virtio_get_subvendor(dev); 962 cpi->hba_subdevice = virtio_get_subdevice(dev); 963 964 ccb->ccb_h.status = CAM_REQ_CMP; 965 xpt_done(ccb); 966 } 967 968 static int 969 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, 970 struct ccb_scsiio *csio) 971 { 972 struct ccb_hdr *ccbh; 973 struct bus_dma_segment *dseg; 974 int i, error; 975 976 ccbh = &csio->ccb_h; 977 error = 0; 978 979 switch ((ccbh->flags & CAM_DATA_MASK)) { 980 case CAM_DATA_VADDR: 981 error = sglist_append(sg, csio->data_ptr, csio->dxfer_len); 982 break; 983 case CAM_DATA_PADDR: 984 error = sglist_append_phys(sg, 985 (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len); 986 break; 987 case CAM_DATA_SG: 988 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 989 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 990 error = sglist_append(sg, 991 (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len); 992 } 993 break; 994 case CAM_DATA_SG_PADDR: 995 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 996 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 997 error = sglist_append_phys(sg, 998 (vm_paddr_t) dseg->ds_addr, dseg->ds_len); 999 } 1000 break; 1001 case CAM_DATA_BIO: 1002 error = sglist_append_bio(sg, (struct bio *) csio->data_ptr); 1003 break; 1004 default: 1005 error = EINVAL; 1006 break; 1007 } 1008 1009 return (error); 1010 } 1011 1012 static int 1013 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, 1014 int *readable, int *writable) 1015 { 1016 struct sglist *sg; 1017 struct ccb_hdr *ccbh; 1018 struct ccb_scsiio *csio; 1019 struct virtio_scsi_cmd_req *cmd_req; 1020 struct virtio_scsi_cmd_resp *cmd_resp; 1021 int error; 1022 1023 sg = sc->vtscsi_sglist; 1024 csio = &req->vsr_ccb->csio; 1025 ccbh = &csio->ccb_h; 1026 cmd_req = &req->vsr_cmd_req; 1027 cmd_resp = &req->vsr_cmd_resp; 1028 1029 sglist_reset(sg); 1030 1031 sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); 1032 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { 1033 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1034 /* At least one segment must be left for the response. */ 1035 if (error || sg->sg_nseg == sg->sg_maxseg) 1036 goto fail; 1037 } 1038 1039 *readable = sg->sg_nseg; 1040 1041 sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); 1042 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { 1043 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1044 if (error) 1045 goto fail; 1046 } 1047 1048 *writable = sg->sg_nseg - *readable; 1049 1050 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " 1051 "writable=%d\n", req, ccbh, *readable, *writable); 1052 1053 return (0); 1054 1055 fail: 1056 /* 1057 * This should never happen unless maxio was incorrectly set. 1058 */ 1059 vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); 1060 1061 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " 1062 "nseg=%d maxseg=%d\n", 1063 error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); 1064 1065 return (EFBIG); 1066 } 1067 1068 static int 1069 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1070 { 1071 struct sglist *sg; 1072 struct virtqueue *vq; 1073 struct ccb_scsiio *csio; 1074 struct ccb_hdr *ccbh; 1075 struct virtio_scsi_cmd_req *cmd_req; 1076 struct virtio_scsi_cmd_resp *cmd_resp; 1077 int readable, writable, error; 1078 1079 sg = sc->vtscsi_sglist; 1080 vq = sc->vtscsi_request_vq; 1081 csio = &req->vsr_ccb->csio; 1082 ccbh = &csio->ccb_h; 1083 cmd_req = &req->vsr_cmd_req; 1084 cmd_resp = &req->vsr_cmd_resp; 1085 1086 vtscsi_init_scsi_cmd_req(csio, cmd_req); 1087 1088 error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); 1089 if (error) 1090 return (error); 1091 1092 req->vsr_complete = vtscsi_complete_scsi_cmd; 1093 cmd_resp->response = -1; 1094 1095 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1096 if (error) { 1097 vtscsi_dprintf(sc, VTSCSI_ERROR, 1098 "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); 1099 1100 ccbh->status = CAM_REQUEUE_REQ; 1101 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); 1102 return (error); 1103 } 1104 1105 ccbh->status |= CAM_SIM_QUEUED; 1106 ccbh->ccbh_vtscsi_req = req; 1107 1108 virtqueue_notify(vq); 1109 1110 if (ccbh->timeout != CAM_TIME_INFINITY) { 1111 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; 1112 callout_reset_sbt(&req->vsr_callout, SBT_1MS * ccbh->timeout, 1113 0, vtscsi_timedout_scsi_cmd, req, 0); 1114 } 1115 1116 vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", 1117 req, ccbh); 1118 1119 return (0); 1120 } 1121 1122 static int 1123 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) 1124 { 1125 struct vtscsi_request *req; 1126 int error; 1127 1128 req = vtscsi_dequeue_request(sc); 1129 if (req == NULL) { 1130 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1131 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 1132 return (ENOBUFS); 1133 } 1134 1135 req->vsr_ccb = ccb; 1136 1137 error = vtscsi_execute_scsi_cmd(sc, req); 1138 if (error) 1139 vtscsi_enqueue_request(sc, req); 1140 1141 return (error); 1142 } 1143 1144 static void 1145 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1146 struct vtscsi_request *req) 1147 { 1148 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1149 struct vtscsi_request *to_req; 1150 uint8_t response; 1151 1152 tmf_resp = &req->vsr_tmf_resp; 1153 response = tmf_resp->response; 1154 to_req = req->vsr_timedout_req; 1155 1156 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", 1157 req, to_req, response); 1158 1159 vtscsi_enqueue_request(sc, req); 1160 1161 /* 1162 * The timedout request could have completed between when the 1163 * abort task was sent and when the host processed it. 1164 */ 1165 if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) 1166 return; 1167 1168 /* The timedout request was successfully aborted. */ 1169 if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) 1170 return; 1171 1172 /* Don't bother if the device is going away. */ 1173 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1174 return; 1175 1176 /* The timedout request will be aborted by the reset. */ 1177 if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) 1178 return; 1179 1180 vtscsi_reset_bus(sc); 1181 } 1182 1183 static int 1184 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1185 struct vtscsi_request *to_req) 1186 { 1187 struct sglist *sg; 1188 struct ccb_hdr *to_ccbh; 1189 struct vtscsi_request *req; 1190 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1191 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1192 int error; 1193 1194 sg = sc->vtscsi_sglist; 1195 to_ccbh = &to_req->vsr_ccb->ccb_h; 1196 1197 req = vtscsi_dequeue_request(sc); 1198 if (req == NULL) { 1199 error = ENOBUFS; 1200 goto fail; 1201 } 1202 1203 tmf_req = &req->vsr_tmf_req; 1204 tmf_resp = &req->vsr_tmf_resp; 1205 1206 vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1207 (uintptr_t) to_ccbh, tmf_req); 1208 1209 sglist_reset(sg); 1210 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1211 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1212 1213 req->vsr_timedout_req = to_req; 1214 req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; 1215 tmf_resp->response = -1; 1216 1217 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1218 VTSCSI_EXECUTE_ASYNC); 1219 if (error == 0) 1220 return (0); 1221 1222 vtscsi_enqueue_request(sc, req); 1223 1224 fail: 1225 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " 1226 "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); 1227 1228 return (error); 1229 } 1230 1231 static void 1232 vtscsi_timedout_scsi_cmd(void *xreq) 1233 { 1234 struct vtscsi_softc *sc; 1235 struct vtscsi_request *to_req; 1236 1237 to_req = xreq; 1238 sc = to_req->vsr_softc; 1239 1240 vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", 1241 to_req, to_req->vsr_ccb, to_req->vsr_state); 1242 1243 /* Don't bother if the device is going away. */ 1244 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1245 return; 1246 1247 /* 1248 * Bail if the request is not in use. We likely raced when 1249 * stopping the callout handler or it has already been aborted. 1250 */ 1251 if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || 1252 (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) 1253 return; 1254 1255 /* 1256 * Complete the request queue in case the timedout request is 1257 * actually just pending. 1258 */ 1259 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1260 if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) 1261 return; 1262 1263 sc->vtscsi_stats.scsi_cmd_timeouts++; 1264 to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; 1265 1266 if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) 1267 return; 1268 1269 vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); 1270 vtscsi_reset_bus(sc); 1271 } 1272 1273 static cam_status 1274 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) 1275 { 1276 cam_status status; 1277 1278 switch (cmd_resp->response) { 1279 case VIRTIO_SCSI_S_OK: 1280 status = CAM_REQ_CMP; 1281 break; 1282 case VIRTIO_SCSI_S_OVERRUN: 1283 status = CAM_DATA_RUN_ERR; 1284 break; 1285 case VIRTIO_SCSI_S_ABORTED: 1286 status = CAM_REQ_ABORTED; 1287 break; 1288 case VIRTIO_SCSI_S_BAD_TARGET: 1289 status = CAM_SEL_TIMEOUT; 1290 break; 1291 case VIRTIO_SCSI_S_RESET: 1292 status = CAM_SCSI_BUS_RESET; 1293 break; 1294 case VIRTIO_SCSI_S_BUSY: 1295 status = CAM_SCSI_BUSY; 1296 break; 1297 case VIRTIO_SCSI_S_TRANSPORT_FAILURE: 1298 case VIRTIO_SCSI_S_TARGET_FAILURE: 1299 case VIRTIO_SCSI_S_NEXUS_FAILURE: 1300 status = CAM_SCSI_IT_NEXUS_LOST; 1301 break; 1302 default: /* VIRTIO_SCSI_S_FAILURE */ 1303 status = CAM_REQ_CMP_ERR; 1304 break; 1305 } 1306 1307 return (status); 1308 } 1309 1310 static cam_status 1311 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, 1312 struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) 1313 { 1314 cam_status status; 1315 1316 csio->scsi_status = cmd_resp->status; 1317 csio->resid = cmd_resp->resid; 1318 1319 if (csio->scsi_status == SCSI_STATUS_OK) 1320 status = CAM_REQ_CMP; 1321 else 1322 status = CAM_SCSI_STATUS_ERROR; 1323 1324 if (cmd_resp->sense_len > 0) { 1325 status |= CAM_AUTOSNS_VALID; 1326 1327 if (cmd_resp->sense_len < csio->sense_len) 1328 csio->sense_resid = csio->sense_len - 1329 cmd_resp->sense_len; 1330 else 1331 csio->sense_resid = 0; 1332 1333 memcpy(&csio->sense_data, cmd_resp->sense, 1334 csio->sense_len - csio->sense_resid); 1335 } 1336 1337 vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, 1338 "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", 1339 csio, csio->scsi_status, csio->resid, csio->sense_resid); 1340 1341 return (status); 1342 } 1343 1344 static void 1345 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1346 { 1347 struct ccb_hdr *ccbh; 1348 struct ccb_scsiio *csio; 1349 struct virtio_scsi_cmd_resp *cmd_resp; 1350 cam_status status; 1351 1352 csio = &req->vsr_ccb->csio; 1353 ccbh = &csio->ccb_h; 1354 cmd_resp = &req->vsr_cmd_resp; 1355 1356 KASSERT(ccbh->ccbh_vtscsi_req == req, 1357 ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); 1358 1359 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1360 callout_stop(&req->vsr_callout); 1361 1362 status = vtscsi_scsi_cmd_cam_status(cmd_resp); 1363 if (status == CAM_REQ_ABORTED) { 1364 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) 1365 status = CAM_CMD_TIMEOUT; 1366 } else if (status == CAM_REQ_CMP) 1367 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); 1368 1369 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1370 status |= CAM_DEV_QFRZN; 1371 xpt_freeze_devq(ccbh->path, 1); 1372 } 1373 1374 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1375 status |= CAM_RELEASE_SIMQ; 1376 1377 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", 1378 req, ccbh, status); 1379 1380 ccbh->status = status; 1381 xpt_done(req->vsr_ccb); 1382 vtscsi_enqueue_request(sc, req); 1383 } 1384 1385 static void 1386 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) 1387 { 1388 1389 /* XXX We probably shouldn't poll forever. */ 1390 req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; 1391 do 1392 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1393 while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); 1394 1395 req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; 1396 } 1397 1398 static int 1399 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, 1400 struct sglist *sg, int readable, int writable, int flag) 1401 { 1402 struct virtqueue *vq; 1403 int error; 1404 1405 vq = sc->vtscsi_control_vq; 1406 1407 MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); 1408 1409 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1410 if (error) { 1411 /* 1412 * Return EAGAIN when the virtqueue does not have enough 1413 * descriptors available. 1414 */ 1415 if (error == ENOSPC || error == EMSGSIZE) 1416 error = EAGAIN; 1417 1418 return (error); 1419 } 1420 1421 virtqueue_notify(vq); 1422 if (flag == VTSCSI_EXECUTE_POLL) 1423 vtscsi_poll_ctrl_req(sc, req); 1424 1425 return (0); 1426 } 1427 1428 static void 1429 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, 1430 struct vtscsi_request *req) 1431 { 1432 union ccb *ccb; 1433 struct ccb_hdr *ccbh; 1434 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1435 1436 ccb = req->vsr_ccb; 1437 ccbh = &ccb->ccb_h; 1438 tmf_resp = &req->vsr_tmf_resp; 1439 1440 switch (tmf_resp->response) { 1441 case VIRTIO_SCSI_S_FUNCTION_COMPLETE: 1442 ccbh->status = CAM_REQ_CMP; 1443 break; 1444 case VIRTIO_SCSI_S_FUNCTION_REJECTED: 1445 ccbh->status = CAM_UA_ABORT; 1446 break; 1447 default: 1448 ccbh->status = CAM_REQ_CMP_ERR; 1449 break; 1450 } 1451 1452 xpt_done(ccb); 1453 vtscsi_enqueue_request(sc, req); 1454 } 1455 1456 static int 1457 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, 1458 struct vtscsi_request *req) 1459 { 1460 struct sglist *sg; 1461 struct ccb_abort *cab; 1462 struct ccb_hdr *ccbh; 1463 struct ccb_hdr *abort_ccbh; 1464 struct vtscsi_request *abort_req; 1465 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1466 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1467 int error; 1468 1469 sg = sc->vtscsi_sglist; 1470 cab = &req->vsr_ccb->cab; 1471 ccbh = &cab->ccb_h; 1472 tmf_req = &req->vsr_tmf_req; 1473 tmf_resp = &req->vsr_tmf_resp; 1474 1475 /* CCB header and request that's to be aborted. */ 1476 abort_ccbh = &cab->abort_ccb->ccb_h; 1477 abort_req = abort_ccbh->ccbh_vtscsi_req; 1478 1479 if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { 1480 error = EINVAL; 1481 goto fail; 1482 } 1483 1484 /* Only attempt to abort requests that could be in-flight. */ 1485 if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { 1486 error = EALREADY; 1487 goto fail; 1488 } 1489 1490 abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; 1491 if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1492 callout_stop(&abort_req->vsr_callout); 1493 1494 vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1495 (uintptr_t) abort_ccbh, tmf_req); 1496 1497 sglist_reset(sg); 1498 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1499 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1500 1501 req->vsr_complete = vtscsi_complete_abort_task_cmd; 1502 tmf_resp->response = -1; 1503 1504 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1505 VTSCSI_EXECUTE_ASYNC); 1506 1507 fail: 1508 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " 1509 "abort_req=%p\n", error, req, abort_ccbh, abort_req); 1510 1511 return (error); 1512 } 1513 1514 static void 1515 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, 1516 struct vtscsi_request *req) 1517 { 1518 union ccb *ccb; 1519 struct ccb_hdr *ccbh; 1520 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1521 1522 ccb = req->vsr_ccb; 1523 ccbh = &ccb->ccb_h; 1524 tmf_resp = &req->vsr_tmf_resp; 1525 1526 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", 1527 req, ccb, tmf_resp->response); 1528 1529 if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { 1530 ccbh->status = CAM_REQ_CMP; 1531 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, 1532 ccbh->target_lun); 1533 } else 1534 ccbh->status = CAM_REQ_CMP_ERR; 1535 1536 xpt_done(ccb); 1537 vtscsi_enqueue_request(sc, req); 1538 } 1539 1540 static int 1541 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, 1542 struct vtscsi_request *req) 1543 { 1544 struct sglist *sg; 1545 struct ccb_resetdev *crd; 1546 struct ccb_hdr *ccbh; 1547 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1548 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1549 uint32_t subtype; 1550 int error; 1551 1552 sg = sc->vtscsi_sglist; 1553 crd = &req->vsr_ccb->crd; 1554 ccbh = &crd->ccb_h; 1555 tmf_req = &req->vsr_tmf_req; 1556 tmf_resp = &req->vsr_tmf_resp; 1557 1558 if (ccbh->target_lun == CAM_LUN_WILDCARD) 1559 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; 1560 else 1561 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; 1562 1563 vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); 1564 1565 sglist_reset(sg); 1566 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1567 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1568 1569 req->vsr_complete = vtscsi_complete_reset_dev_cmd; 1570 tmf_resp->response = -1; 1571 1572 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1573 VTSCSI_EXECUTE_ASYNC); 1574 1575 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", 1576 error, req, ccbh); 1577 1578 return (error); 1579 } 1580 1581 static void 1582 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) 1583 { 1584 1585 *target_id = lun[1]; 1586 *lun_id = (lun[2] << 8) | lun[3]; 1587 } 1588 1589 static void 1590 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) 1591 { 1592 1593 lun[0] = 1; 1594 lun[1] = ccbh->target_id; 1595 lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); 1596 lun[3] = ccbh->target_lun & 0xFF; 1597 } 1598 1599 static void 1600 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, 1601 struct virtio_scsi_cmd_req *cmd_req) 1602 { 1603 uint8_t attr; 1604 1605 switch (csio->tag_action) { 1606 case MSG_HEAD_OF_Q_TAG: 1607 attr = VIRTIO_SCSI_S_HEAD; 1608 break; 1609 case MSG_ORDERED_Q_TAG: 1610 attr = VIRTIO_SCSI_S_ORDERED; 1611 break; 1612 case MSG_ACA_TASK: 1613 attr = VIRTIO_SCSI_S_ACA; 1614 break; 1615 default: /* MSG_SIMPLE_Q_TAG */ 1616 attr = VIRTIO_SCSI_S_SIMPLE; 1617 break; 1618 } 1619 1620 vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); 1621 cmd_req->tag = (uintptr_t) csio; 1622 cmd_req->task_attr = attr; 1623 1624 memcpy(cmd_req->cdb, 1625 csio->ccb_h.flags & CAM_CDB_POINTER ? 1626 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, 1627 csio->cdb_len); 1628 } 1629 1630 static void 1631 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, 1632 uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) 1633 { 1634 1635 vtscsi_set_request_lun(ccbh, tmf_req->lun); 1636 1637 tmf_req->type = VIRTIO_SCSI_T_TMF; 1638 tmf_req->subtype = subtype; 1639 tmf_req->tag = tag; 1640 } 1641 1642 static void 1643 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) 1644 { 1645 int frozen; 1646 1647 frozen = sc->vtscsi_frozen; 1648 1649 if (reason & VTSCSI_REQUEST && 1650 (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) 1651 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; 1652 1653 if (reason & VTSCSI_REQUEST_VQ && 1654 (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) 1655 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; 1656 1657 /* Freeze the SIMQ if transitioned to frozen. */ 1658 if (frozen == 0 && sc->vtscsi_frozen != 0) { 1659 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); 1660 xpt_freeze_simq(sc->vtscsi_sim, 1); 1661 } 1662 } 1663 1664 static int 1665 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) 1666 { 1667 int thawed; 1668 1669 if (sc->vtscsi_frozen == 0 || reason == 0) 1670 return (0); 1671 1672 if (reason & VTSCSI_REQUEST && 1673 sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) 1674 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; 1675 1676 if (reason & VTSCSI_REQUEST_VQ && 1677 sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) 1678 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; 1679 1680 thawed = sc->vtscsi_frozen == 0; 1681 if (thawed != 0) 1682 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); 1683 1684 return (thawed); 1685 } 1686 1687 static void 1688 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, 1689 target_id_t target_id, lun_id_t lun_id) 1690 { 1691 struct cam_path *path; 1692 1693 /* Use the wildcard path from our softc for bus announcements. */ 1694 if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { 1695 xpt_async(ac_code, sc->vtscsi_path, NULL); 1696 return; 1697 } 1698 1699 if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), 1700 target_id, lun_id) != CAM_REQ_CMP) { 1701 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); 1702 return; 1703 } 1704 1705 xpt_async(ac_code, path, NULL); 1706 xpt_free_path(path); 1707 } 1708 1709 static void 1710 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, 1711 lun_id_t lun_id) 1712 { 1713 union ccb *ccb; 1714 cam_status status; 1715 1716 ccb = xpt_alloc_ccb_nowait(); 1717 if (ccb == NULL) { 1718 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1719 return; 1720 } 1721 1722 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1723 cam_sim_path(sc->vtscsi_sim), target_id, lun_id); 1724 if (status != CAM_REQ_CMP) { 1725 xpt_free_ccb(ccb); 1726 return; 1727 } 1728 1729 xpt_rescan(ccb); 1730 } 1731 1732 static void 1733 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) 1734 { 1735 1736 vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); 1737 } 1738 1739 static void 1740 vtscsi_transport_reset_event(struct vtscsi_softc *sc, 1741 struct virtio_scsi_event *event) 1742 { 1743 target_id_t target_id; 1744 lun_id_t lun_id; 1745 1746 vtscsi_get_request_lun(event->lun, &target_id, &lun_id); 1747 1748 switch (event->reason) { 1749 case VIRTIO_SCSI_EVT_RESET_RESCAN: 1750 case VIRTIO_SCSI_EVT_RESET_REMOVED: 1751 vtscsi_execute_rescan(sc, target_id, lun_id); 1752 break; 1753 default: 1754 device_printf(sc->vtscsi_dev, 1755 "unhandled transport event reason: %d\n", event->reason); 1756 break; 1757 } 1758 } 1759 1760 static void 1761 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) 1762 { 1763 int error; 1764 1765 if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { 1766 switch (event->event) { 1767 case VIRTIO_SCSI_T_TRANSPORT_RESET: 1768 vtscsi_transport_reset_event(sc, event); 1769 break; 1770 default: 1771 device_printf(sc->vtscsi_dev, 1772 "unhandled event: %d\n", event->event); 1773 break; 1774 } 1775 } else 1776 vtscsi_execute_rescan_bus(sc); 1777 1778 /* 1779 * This should always be successful since the buffer 1780 * was just dequeued. 1781 */ 1782 error = vtscsi_enqueue_event_buf(sc, event); 1783 KASSERT(error == 0, 1784 ("cannot requeue event buffer: %d", error)); 1785 } 1786 1787 static int 1788 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, 1789 struct virtio_scsi_event *event) 1790 { 1791 struct sglist *sg; 1792 struct virtqueue *vq; 1793 int size, error; 1794 1795 sg = sc->vtscsi_sglist; 1796 vq = sc->vtscsi_event_vq; 1797 size = sc->vtscsi_event_buf_size; 1798 1799 bzero(event, size); 1800 1801 sglist_reset(sg); 1802 error = sglist_append(sg, event, size); 1803 if (error) 1804 return (error); 1805 1806 error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); 1807 if (error) 1808 return (error); 1809 1810 virtqueue_notify(vq); 1811 1812 return (0); 1813 } 1814 1815 static int 1816 vtscsi_init_event_vq(struct vtscsi_softc *sc) 1817 { 1818 struct virtio_scsi_event *event; 1819 int i, size, error; 1820 1821 /* 1822 * The first release of QEMU with VirtIO SCSI support would crash 1823 * when attempting to notify the event virtqueue. This was fixed 1824 * when hotplug support was added. 1825 */ 1826 if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) 1827 size = sc->vtscsi_event_buf_size; 1828 else 1829 size = 0; 1830 1831 if (size < sizeof(struct virtio_scsi_event)) 1832 return (0); 1833 1834 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1835 event = &sc->vtscsi_event_bufs[i]; 1836 1837 error = vtscsi_enqueue_event_buf(sc, event); 1838 if (error) 1839 break; 1840 } 1841 1842 /* 1843 * Even just one buffer is enough. Missed events are 1844 * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. 1845 */ 1846 if (i > 0) 1847 error = 0; 1848 1849 return (error); 1850 } 1851 1852 static void 1853 vtscsi_reinit_event_vq(struct vtscsi_softc *sc) 1854 { 1855 struct virtio_scsi_event *event; 1856 int i, error; 1857 1858 if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || 1859 sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) 1860 return; 1861 1862 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1863 event = &sc->vtscsi_event_bufs[i]; 1864 1865 error = vtscsi_enqueue_event_buf(sc, event); 1866 if (error) 1867 break; 1868 } 1869 1870 KASSERT(i > 0, ("cannot reinit event vq: %d", error)); 1871 } 1872 1873 static void 1874 vtscsi_drain_event_vq(struct vtscsi_softc *sc) 1875 { 1876 struct virtqueue *vq; 1877 int last; 1878 1879 vq = sc->vtscsi_event_vq; 1880 last = 0; 1881 1882 while (virtqueue_drain(vq, &last) != NULL) 1883 ; 1884 1885 KASSERT(virtqueue_empty(vq), ("eventvq not empty")); 1886 } 1887 1888 static void 1889 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) 1890 { 1891 1892 VTSCSI_LOCK_OWNED(sc); 1893 1894 if (sc->vtscsi_request_vq != NULL) 1895 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1896 if (sc->vtscsi_control_vq != NULL) 1897 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1898 } 1899 1900 static void 1901 vtscsi_complete_vqs(struct vtscsi_softc *sc) 1902 { 1903 1904 VTSCSI_LOCK(sc); 1905 vtscsi_complete_vqs_locked(sc); 1906 VTSCSI_UNLOCK(sc); 1907 } 1908 1909 static void 1910 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 1911 { 1912 union ccb *ccb; 1913 int detach; 1914 1915 ccb = req->vsr_ccb; 1916 1917 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); 1918 1919 /* 1920 * The callout must be drained when detaching since the request is 1921 * about to be freed. The VTSCSI_MTX must not be held for this in 1922 * case the callout is pending because there is a deadlock potential. 1923 * Otherwise, the virtqueue is being drained because of a bus reset 1924 * so we only need to attempt to stop the callouts. 1925 */ 1926 detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; 1927 if (detach != 0) 1928 VTSCSI_LOCK_NOTOWNED(sc); 1929 else 1930 VTSCSI_LOCK_OWNED(sc); 1931 1932 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { 1933 if (detach != 0) 1934 callout_drain(&req->vsr_callout); 1935 else 1936 callout_stop(&req->vsr_callout); 1937 } 1938 1939 if (ccb != NULL) { 1940 if (detach != 0) { 1941 VTSCSI_LOCK(sc); 1942 ccb->ccb_h.status = CAM_NO_HBA; 1943 } else 1944 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1945 xpt_done(ccb); 1946 if (detach != 0) 1947 VTSCSI_UNLOCK(sc); 1948 } 1949 1950 vtscsi_enqueue_request(sc, req); 1951 } 1952 1953 static void 1954 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 1955 { 1956 struct vtscsi_request *req; 1957 int last; 1958 1959 last = 0; 1960 1961 vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); 1962 1963 while ((req = virtqueue_drain(vq, &last)) != NULL) 1964 vtscsi_cancel_request(sc, req); 1965 1966 KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 1967 } 1968 1969 static void 1970 vtscsi_drain_vqs(struct vtscsi_softc *sc) 1971 { 1972 1973 if (sc->vtscsi_control_vq != NULL) 1974 vtscsi_drain_vq(sc, sc->vtscsi_control_vq); 1975 if (sc->vtscsi_request_vq != NULL) 1976 vtscsi_drain_vq(sc, sc->vtscsi_request_vq); 1977 if (sc->vtscsi_event_vq != NULL) 1978 vtscsi_drain_event_vq(sc); 1979 } 1980 1981 static void 1982 vtscsi_stop(struct vtscsi_softc *sc) 1983 { 1984 1985 vtscsi_disable_vqs_intr(sc); 1986 virtio_stop(sc->vtscsi_dev); 1987 } 1988 1989 static int 1990 vtscsi_reset_bus(struct vtscsi_softc *sc) 1991 { 1992 int error; 1993 1994 VTSCSI_LOCK_OWNED(sc); 1995 1996 if (vtscsi_bus_reset_disable != 0) { 1997 device_printf(sc->vtscsi_dev, "bus reset disabled\n"); 1998 return (0); 1999 } 2000 2001 sc->vtscsi_flags |= VTSCSI_FLAG_RESET; 2002 2003 /* 2004 * vtscsi_stop() will cause the in-flight requests to be canceled. 2005 * Those requests are then completed here so CAM will retry them 2006 * after the reset is complete. 2007 */ 2008 vtscsi_stop(sc); 2009 vtscsi_complete_vqs_locked(sc); 2010 2011 /* Rid the virtqueues of any remaining requests. */ 2012 vtscsi_drain_vqs(sc); 2013 2014 /* 2015 * Any resource shortage that froze the SIMQ cannot persist across 2016 * a bus reset so ensure it gets thawed here. 2017 */ 2018 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 2019 xpt_release_simq(sc->vtscsi_sim, 0); 2020 2021 error = vtscsi_reinit(sc); 2022 if (error) { 2023 device_printf(sc->vtscsi_dev, 2024 "reinitialization failed, stopping device...\n"); 2025 vtscsi_stop(sc); 2026 } else 2027 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, 2028 CAM_LUN_WILDCARD); 2029 2030 sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; 2031 2032 return (error); 2033 } 2034 2035 static void 2036 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2037 { 2038 2039 #ifdef INVARIANTS 2040 int req_nsegs, resp_nsegs; 2041 2042 req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2043 resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2044 2045 KASSERT(req_nsegs == 1, ("request crossed page boundary")); 2046 KASSERT(resp_nsegs == 1, ("response crossed page boundary")); 2047 #endif 2048 2049 req->vsr_softc = sc; 2050 callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0); 2051 } 2052 2053 static int 2054 vtscsi_alloc_requests(struct vtscsi_softc *sc) 2055 { 2056 struct vtscsi_request *req; 2057 int i, nreqs; 2058 2059 /* 2060 * Commands destined for either the request or control queues come 2061 * from the same SIM queue. Use the size of the request virtqueue 2062 * as it (should) be much more frequently used. Some additional 2063 * requests are allocated for internal (TMF) use. 2064 */ 2065 nreqs = virtqueue_size(sc->vtscsi_request_vq); 2066 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) 2067 nreqs /= VTSCSI_MIN_SEGMENTS; 2068 nreqs += VTSCSI_RESERVED_REQUESTS; 2069 2070 for (i = 0; i < nreqs; i++) { 2071 req = malloc(sizeof(struct vtscsi_request), M_DEVBUF, 2072 M_NOWAIT); 2073 if (req == NULL) 2074 return (ENOMEM); 2075 2076 vtscsi_init_request(sc, req); 2077 2078 sc->vtscsi_nrequests++; 2079 vtscsi_enqueue_request(sc, req); 2080 } 2081 2082 return (0); 2083 } 2084 2085 static void 2086 vtscsi_free_requests(struct vtscsi_softc *sc) 2087 { 2088 struct vtscsi_request *req; 2089 2090 while ((req = vtscsi_dequeue_request(sc)) != NULL) { 2091 KASSERT(callout_active(&req->vsr_callout) == 0, 2092 ("request callout still active")); 2093 2094 sc->vtscsi_nrequests--; 2095 free(req, M_DEVBUF); 2096 } 2097 2098 KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", 2099 sc->vtscsi_nrequests)); 2100 } 2101 2102 static void 2103 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2104 { 2105 2106 KASSERT(req->vsr_softc == sc, 2107 ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); 2108 2109 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2110 2111 /* A request is available so the SIMQ could be released. */ 2112 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) 2113 xpt_release_simq(sc->vtscsi_sim, 1); 2114 2115 req->vsr_ccb = NULL; 2116 req->vsr_complete = NULL; 2117 req->vsr_ptr0 = NULL; 2118 req->vsr_state = VTSCSI_REQ_STATE_FREE; 2119 req->vsr_flags = 0; 2120 2121 bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2122 bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2123 2124 /* 2125 * We insert at the tail of the queue in order to make it 2126 * very unlikely a request will be reused if we race with 2127 * stopping its callout handler. 2128 */ 2129 TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); 2130 } 2131 2132 static struct vtscsi_request * 2133 vtscsi_dequeue_request(struct vtscsi_softc *sc) 2134 { 2135 struct vtscsi_request *req; 2136 2137 req = TAILQ_FIRST(&sc->vtscsi_req_free); 2138 if (req != NULL) { 2139 req->vsr_state = VTSCSI_REQ_STATE_INUSE; 2140 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); 2141 } else 2142 sc->vtscsi_stats.dequeue_no_requests++; 2143 2144 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2145 2146 return (req); 2147 } 2148 2149 static void 2150 vtscsi_complete_request(struct vtscsi_request *req) 2151 { 2152 2153 if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) 2154 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; 2155 2156 if (req->vsr_complete != NULL) 2157 req->vsr_complete(req->vsr_softc, req); 2158 } 2159 2160 static void 2161 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2162 { 2163 struct vtscsi_request *req; 2164 2165 VTSCSI_LOCK_OWNED(sc); 2166 2167 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) 2168 vtscsi_complete_request(req); 2169 } 2170 2171 static void 2172 vtscsi_control_vq_intr(void *xsc) 2173 { 2174 struct vtscsi_softc *sc; 2175 struct virtqueue *vq; 2176 2177 sc = xsc; 2178 vq = sc->vtscsi_control_vq; 2179 2180 again: 2181 VTSCSI_LOCK(sc); 2182 2183 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 2184 2185 if (virtqueue_enable_intr(vq) != 0) { 2186 virtqueue_disable_intr(vq); 2187 VTSCSI_UNLOCK(sc); 2188 goto again; 2189 } 2190 2191 VTSCSI_UNLOCK(sc); 2192 } 2193 2194 static void 2195 vtscsi_event_vq_intr(void *xsc) 2196 { 2197 struct vtscsi_softc *sc; 2198 struct virtqueue *vq; 2199 struct virtio_scsi_event *event; 2200 2201 sc = xsc; 2202 vq = sc->vtscsi_event_vq; 2203 2204 again: 2205 VTSCSI_LOCK(sc); 2206 2207 while ((event = virtqueue_dequeue(vq, NULL)) != NULL) 2208 vtscsi_handle_event(sc, event); 2209 2210 if (virtqueue_enable_intr(vq) != 0) { 2211 virtqueue_disable_intr(vq); 2212 VTSCSI_UNLOCK(sc); 2213 goto again; 2214 } 2215 2216 VTSCSI_UNLOCK(sc); 2217 } 2218 2219 static void 2220 vtscsi_request_vq_intr(void *xsc) 2221 { 2222 struct vtscsi_softc *sc; 2223 struct virtqueue *vq; 2224 2225 sc = xsc; 2226 vq = sc->vtscsi_request_vq; 2227 2228 again: 2229 VTSCSI_LOCK(sc); 2230 2231 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 2232 2233 if (virtqueue_enable_intr(vq) != 0) { 2234 virtqueue_disable_intr(vq); 2235 VTSCSI_UNLOCK(sc); 2236 goto again; 2237 } 2238 2239 VTSCSI_UNLOCK(sc); 2240 } 2241 2242 static void 2243 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) 2244 { 2245 2246 virtqueue_disable_intr(sc->vtscsi_control_vq); 2247 virtqueue_disable_intr(sc->vtscsi_event_vq); 2248 virtqueue_disable_intr(sc->vtscsi_request_vq); 2249 } 2250 2251 static void 2252 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) 2253 { 2254 2255 virtqueue_enable_intr(sc->vtscsi_control_vq); 2256 virtqueue_enable_intr(sc->vtscsi_event_vq); 2257 virtqueue_enable_intr(sc->vtscsi_request_vq); 2258 } 2259 2260 static void 2261 vtscsi_get_tunables(struct vtscsi_softc *sc) 2262 { 2263 char tmpstr[64]; 2264 2265 TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); 2266 2267 snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", 2268 device_get_unit(sc->vtscsi_dev)); 2269 TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); 2270 } 2271 2272 static void 2273 vtscsi_add_sysctl(struct vtscsi_softc *sc) 2274 { 2275 device_t dev; 2276 struct vtscsi_statistics *stats; 2277 struct sysctl_ctx_list *ctx; 2278 struct sysctl_oid *tree; 2279 struct sysctl_oid_list *child; 2280 2281 dev = sc->vtscsi_dev; 2282 stats = &sc->vtscsi_stats; 2283 ctx = device_get_sysctl_ctx(dev); 2284 tree = device_get_sysctl_tree(dev); 2285 child = SYSCTL_CHILDREN(tree); 2286 2287 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 2288 CTLFLAG_RW, &sc->vtscsi_debug, 0, 2289 "Debug level"); 2290 2291 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", 2292 CTLFLAG_RD, &stats->scsi_cmd_timeouts, 2293 "SCSI command timeouts"); 2294 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", 2295 CTLFLAG_RD, &stats->dequeue_no_requests, 2296 "No available requests to dequeue"); 2297 } 2298 2299 static void 2300 vtscsi_printf_req(struct vtscsi_request *req, const char *func, 2301 const char *fmt, ...) 2302 { 2303 struct vtscsi_softc *sc; 2304 union ccb *ccb; 2305 struct sbuf sb; 2306 va_list ap; 2307 char str[192]; 2308 char path_str[64]; 2309 2310 if (req == NULL) 2311 return; 2312 2313 sc = req->vsr_softc; 2314 ccb = req->vsr_ccb; 2315 2316 va_start(ap, fmt); 2317 sbuf_new(&sb, str, sizeof(str), 0); 2318 2319 if (ccb == NULL) { 2320 sbuf_printf(&sb, "(noperiph:%s%d:%u): ", 2321 cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), 2322 cam_sim_bus(sc->vtscsi_sim)); 2323 } else { 2324 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); 2325 sbuf_cat(&sb, path_str); 2326 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2327 scsi_command_string(&ccb->csio, &sb); 2328 sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); 2329 } 2330 } 2331 2332 sbuf_vprintf(&sb, fmt, ap); 2333 va_end(ap); 2334 2335 sbuf_finish(&sb); 2336 printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, 2337 sbuf_data(&sb)); 2338 } 2339