1 /*- 2 * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD: head/sys/dev/virtio/scsi/virtio_scsi.c 311305 2017-01-04 20:26:42Z asomers $ 27 */ 28 29 /* Driver for VirtIO SCSI devices. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/kthread.h> 35 #include <sys/malloc.h> 36 #include <sys/module.h> 37 #include <sys/sglist.h> 38 #include <sys/sysctl.h> 39 #include <sys/lock.h> 40 #include <sys/callout.h> 41 #include <sys/queue.h> 42 #include <sys/sbuf.h> 43 44 #include <machine/stdarg.h> 45 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 49 #include <bus/cam/cam.h> 50 #include <bus/cam/cam_ccb.h> 51 #include <bus/cam/cam_sim.h> 52 #include <bus/cam/cam_periph.h> 53 #include <bus/cam/cam_xpt_periph.h> 54 #include <bus/cam/cam_xpt_sim.h> 55 #include <bus/cam/cam_debug.h> 56 #include <bus/cam/scsi/scsi_all.h> 57 #include <bus/cam/scsi/scsi_message.h> 58 59 #include <dev/virtual/virtio/virtio/virtio.h> 60 #include <dev/virtual/virtio/virtio/virtqueue.h> 61 #include <dev/virtual/virtio/scsi/virtio_scsi.h> 62 #include <dev/virtual/virtio/scsi/virtio_scsivar.h> 63 64 static int vtscsi_modevent(module_t, int, void *); 65 66 static int vtscsi_probe(device_t); 67 static int vtscsi_attach(device_t); 68 static int vtscsi_detach(device_t); 69 static int vtscsi_suspend(device_t); 70 static int vtscsi_resume(device_t); 71 72 static void vtscsi_negotiate_features(struct vtscsi_softc *); 73 static void vtscsi_read_config(struct vtscsi_softc *, 74 struct virtio_scsi_config *); 75 static int vtscsi_maximum_segments(struct vtscsi_softc *, int); 76 static int vtscsi_alloc_intrs(struct vtscsi_softc *); 77 static int vtscsi_alloc_virtqueues(struct vtscsi_softc *); 78 static void vtscsi_write_device_config(struct vtscsi_softc *); 79 static int vtscsi_reinit(struct vtscsi_softc *); 80 81 static int vtscsi_alloc_cam(struct vtscsi_softc *); 82 static int vtscsi_register_cam(struct vtscsi_softc *); 83 static void vtscsi_free_cam(struct vtscsi_softc *); 84 static void vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *); 85 static int vtscsi_register_async(struct vtscsi_softc *); 86 static void vtscsi_deregister_async(struct vtscsi_softc *); 87 static void vtscsi_cam_action(struct cam_sim *, union ccb *); 88 static void vtscsi_cam_poll(struct cam_sim *); 89 90 static void vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *, 91 union ccb *); 92 static void vtscsi_cam_get_tran_settings(struct vtscsi_softc *, 93 union ccb *); 94 static void vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *); 95 static void vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *); 96 static void vtscsi_cam_abort(struct vtscsi_softc *, union ccb *); 97 static void vtscsi_cam_path_inquiry(struct vtscsi_softc *, 98 struct cam_sim *, union ccb *); 99 100 static int vtscsi_sg_append_scsi_buf(struct vtscsi_softc *, 101 struct sglist *, struct ccb_scsiio *); 102 static int vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *, 103 struct vtscsi_request *, int *, int *); 104 static int vtscsi_execute_scsi_cmd(struct vtscsi_softc *, 105 struct vtscsi_request *); 106 static int vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *); 107 static void vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *, 108 struct vtscsi_request *); 109 static int vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *, 110 struct vtscsi_request *); 111 static void vtscsi_timedout_scsi_cmd(void *); 112 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *); 113 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *, 114 struct ccb_scsiio *, struct virtio_scsi_cmd_resp *); 115 static void vtscsi_complete_scsi_cmd(struct vtscsi_softc *, 116 struct vtscsi_request *); 117 118 static void vtscsi_poll_ctrl_req(struct vtscsi_softc *, 119 struct vtscsi_request *); 120 static int vtscsi_execute_ctrl_req(struct vtscsi_softc *, 121 struct vtscsi_request *, struct sglist *, int, int, int); 122 static void vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c, 123 struct vtscsi_request *); 124 static int vtscsi_execute_abort_task_cmd(struct vtscsi_softc *, 125 struct vtscsi_request *); 126 static int vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *, 127 struct vtscsi_request *); 128 129 static void vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *); 130 static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []); 131 static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, 132 struct virtio_scsi_cmd_req *); 133 static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, 134 uintptr_t, struct virtio_scsi_ctrl_tmf_req *); 135 136 static void vtscsi_freeze_simq(struct vtscsi_softc *, int); 137 static int vtscsi_thaw_simq(struct vtscsi_softc *, int); 138 139 static void vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t, 140 lun_id_t); 141 static void vtscsi_cam_rescan_callback(struct cam_periph *periph, 142 union ccb *ccb); 143 static void vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t, 144 lun_id_t); 145 static void vtscsi_execute_rescan_bus(struct vtscsi_softc *); 146 147 static void vtscsi_handle_event(struct vtscsi_softc *, 148 struct virtio_scsi_event *); 149 static int vtscsi_enqueue_event_buf(struct vtscsi_softc *, 150 struct virtio_scsi_event *); 151 static int vtscsi_init_event_vq(struct vtscsi_softc *); 152 static void vtscsi_reinit_event_vq(struct vtscsi_softc *); 153 static void vtscsi_drain_event_vq(struct vtscsi_softc *); 154 155 static void vtscsi_complete_vqs_locked(struct vtscsi_softc *); 156 static void vtscsi_complete_vqs(struct vtscsi_softc *); 157 static void vtscsi_drain_vqs(struct vtscsi_softc *); 158 static void vtscsi_cancel_request(struct vtscsi_softc *, 159 struct vtscsi_request *); 160 static void vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *); 161 static void vtscsi_stop(struct vtscsi_softc *); 162 static int vtscsi_reset_bus(struct vtscsi_softc *); 163 164 static void vtscsi_init_request(struct vtscsi_softc *, 165 struct vtscsi_request *); 166 static int vtscsi_alloc_requests(struct vtscsi_softc *); 167 static void vtscsi_free_requests(struct vtscsi_softc *); 168 static void vtscsi_enqueue_request(struct vtscsi_softc *, 169 struct vtscsi_request *); 170 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *); 171 172 static void vtscsi_complete_request(struct vtscsi_request *); 173 static void vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *); 174 175 static void vtscsi_control_vq_intr(void *); 176 static void vtscsi_event_vq_intr(void *); 177 static void vtscsi_request_vq_intr(void *); 178 static void vtscsi_disable_vqs_intr(struct vtscsi_softc *); 179 static void vtscsi_enable_vqs_intr(struct vtscsi_softc *); 180 181 static void vtscsi_get_tunables(struct vtscsi_softc *); 182 static void vtscsi_add_sysctl(struct vtscsi_softc *); 183 184 static void vtscsi_printf_req(struct vtscsi_request *, const char *, 185 const char *, ...) __printflike(3, 4); 186 187 /* Global tunables. */ 188 /* 189 * The current QEMU VirtIO SCSI implementation does not cancel in-flight 190 * IO during virtio_stop(). So in-flight requests still complete after the 191 * device reset. We would have to wait for all the in-flight IO to complete, 192 * which defeats the typical purpose of a bus reset. We could simulate the 193 * bus reset with either I_T_NEXUS_RESET of all the targets, or with 194 * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the 195 * control virtqueue). But this isn't very useful if things really go off 196 * the rails, so default to disabled for now. 197 */ 198 static int vtscsi_bus_reset_disable = 1; 199 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable); 200 201 static struct virtio_feature_desc vtscsi_feature_desc[] = { 202 { VIRTIO_SCSI_F_INOUT, "InOut" }, 203 { VIRTIO_SCSI_F_HOTPLUG, "Hotplug" }, 204 205 { 0, NULL } 206 }; 207 208 static device_method_t vtscsi_methods[] = { 209 /* Device methods. */ 210 DEVMETHOD(device_probe, vtscsi_probe), 211 DEVMETHOD(device_attach, vtscsi_attach), 212 DEVMETHOD(device_detach, vtscsi_detach), 213 DEVMETHOD(device_suspend, vtscsi_suspend), 214 DEVMETHOD(device_resume, vtscsi_resume), 215 216 DEVMETHOD_END 217 }; 218 219 static driver_t vtscsi_driver = { 220 "vtscsi", 221 vtscsi_methods, 222 sizeof(struct vtscsi_softc) 223 }; 224 static devclass_t vtscsi_devclass; 225 226 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, 227 vtscsi_modevent, NULL); 228 MODULE_VERSION(virtio_scsi, 1); 229 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1); 230 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1); 231 232 static int 233 vtscsi_modevent(module_t mod, int type, void *unused) 234 { 235 int error; 236 237 switch (type) { 238 case MOD_LOAD: 239 case MOD_UNLOAD: 240 case MOD_SHUTDOWN: 241 error = 0; 242 break; 243 default: 244 error = EOPNOTSUPP; 245 break; 246 } 247 248 return (error); 249 } 250 251 static int 252 vtscsi_probe(device_t dev) 253 { 254 255 if (virtio_get_device_type(dev) != VIRTIO_ID_SCSI) 256 return (ENXIO); 257 258 device_set_desc(dev, "VirtIO SCSI Adapter"); 259 260 return (BUS_PROBE_DEFAULT); 261 } 262 263 struct irqmap { 264 int irq; 265 driver_intr_t *handler; 266 }; 267 268 static int 269 vtscsi_attach(device_t dev) 270 { 271 struct vtscsi_softc *sc; 272 struct virtio_scsi_config scsicfg; 273 int i, error; 274 275 sc = device_get_softc(dev); 276 sc->vtscsi_dev = dev; 277 278 VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev)); 279 TAILQ_INIT(&sc->vtscsi_req_free); 280 281 vtscsi_get_tunables(sc); 282 vtscsi_add_sysctl(sc); 283 284 virtio_set_feature_desc(dev, vtscsi_feature_desc); 285 vtscsi_negotiate_features(sc); 286 287 if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) 288 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; 289 #ifndef __DragonFly__ /* XXX swildner */ 290 if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) 291 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; 292 #endif 293 if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) 294 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; 295 296 vtscsi_read_config(sc, &scsicfg); 297 298 sc->vtscsi_max_channel = scsicfg.max_channel; 299 sc->vtscsi_max_target = scsicfg.max_target; 300 sc->vtscsi_max_lun = scsicfg.max_lun; 301 sc->vtscsi_event_buf_size = scsicfg.event_info_size; 302 303 vtscsi_write_device_config(sc); 304 305 sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max); 306 sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT); 307 if (sc->vtscsi_sglist == NULL) { 308 error = ENOMEM; 309 device_printf(dev, "cannot allocate sglist\n"); 310 goto fail; 311 } 312 313 error = vtscsi_alloc_intrs(sc); 314 if (error) { 315 device_printf(dev, "cannot allocate interrupts\n"); 316 goto fail; 317 } 318 319 error = vtscsi_alloc_virtqueues(sc); 320 if (error) { 321 device_printf(dev, "cannot allocate virtqueues\n"); 322 goto fail; 323 } 324 325 /* XXX Separate function */ 326 struct irqmap info[3]; 327 328 /* Possible "Virtqueue <-> IRQ" configurations */ 329 switch (sc->vtscsi_nintr) { 330 case 1: 331 info[0] = (struct irqmap){0, vtscsi_control_vq_intr}; 332 info[1] = (struct irqmap){0, vtscsi_event_vq_intr}; 333 info[2] = (struct irqmap){0, vtscsi_request_vq_intr}; 334 break; 335 case 2: 336 info[0] = (struct irqmap){0, vtscsi_control_vq_intr}; 337 info[1] = (struct irqmap){0, vtscsi_event_vq_intr}; 338 info[2] = (struct irqmap){1, vtscsi_request_vq_intr}; 339 break; 340 case 3: 341 info[0] = (struct irqmap){0, vtscsi_control_vq_intr}; 342 info[1] = (struct irqmap){1, vtscsi_event_vq_intr}; 343 info[2] = (struct irqmap){2, vtscsi_request_vq_intr}; 344 break; 345 default: 346 device_printf(dev, "Invalid interrupt vector count: %d\n", 347 sc->vtscsi_nintr); 348 goto fail; 349 } 350 for (i = 0; i < 3; i++) { 351 error = virtio_bind_intr(sc->vtscsi_dev, info[i].irq, i, 352 info[i].handler, sc); 353 if (error) { 354 device_printf(dev, 355 "cannot bind virtqueue IRQs\n"); 356 goto fail; 357 } 358 } 359 360 error = vtscsi_init_event_vq(sc); 361 if (error) { 362 device_printf(dev, "cannot populate the eventvq\n"); 363 goto fail; 364 } 365 366 error = vtscsi_alloc_requests(sc); 367 if (error) { 368 device_printf(dev, "cannot allocate requests\n"); 369 goto fail; 370 } 371 372 error = vtscsi_alloc_cam(sc); 373 if (error) { 374 device_printf(dev, "cannot allocate CAM structures\n"); 375 goto fail; 376 } 377 378 for (i = 0; i < sc->vtscsi_nintr; i++) { 379 error = virtio_setup_intr(dev, i, NULL); 380 if (error) { 381 device_printf(dev, "cannot setup virtqueue " 382 "interrupts\n"); 383 goto fail; 384 } 385 } 386 387 vtscsi_enable_vqs_intr(sc); 388 389 /* 390 * Register with CAM after interrupts are enabled so we will get 391 * notified of the probe responses. 392 */ 393 error = vtscsi_register_cam(sc); 394 if (error) { 395 device_printf(dev, "cannot register with CAM\n"); 396 goto fail; 397 } 398 399 fail: 400 if (error) 401 vtscsi_detach(dev); 402 403 return (error); 404 } 405 406 static int 407 vtscsi_detach(device_t dev) 408 { 409 struct vtscsi_softc *sc; 410 int i; 411 412 sc = device_get_softc(dev); 413 414 for (i = 0; i < sc->vtscsi_nintr; i++) 415 virtio_teardown_intr(dev, i); 416 417 VTSCSI_LOCK(sc); 418 sc->vtscsi_flags |= VTSCSI_FLAG_DETACH; 419 if (device_is_attached(dev)) 420 vtscsi_stop(sc); 421 VTSCSI_UNLOCK(sc); 422 423 vtscsi_complete_vqs(sc); 424 vtscsi_drain_vqs(sc); 425 426 vtscsi_free_cam(sc); 427 vtscsi_free_requests(sc); 428 429 if (sc->vtscsi_sglist != NULL) { 430 sglist_free(sc->vtscsi_sglist); 431 sc->vtscsi_sglist = NULL; 432 } 433 434 VTSCSI_LOCK_DESTROY(sc); 435 436 return (0); 437 } 438 439 static int 440 vtscsi_suspend(device_t dev) 441 { 442 443 return (0); 444 } 445 446 static int 447 vtscsi_resume(device_t dev) 448 { 449 450 return (0); 451 } 452 453 static void 454 vtscsi_negotiate_features(struct vtscsi_softc *sc) 455 { 456 device_t dev; 457 uint64_t features; 458 459 dev = sc->vtscsi_dev; 460 features = virtio_negotiate_features(dev, VTSCSI_FEATURES); 461 sc->vtscsi_features = features; 462 } 463 464 #define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \ 465 virtio_read_device_config(_dev, \ 466 offsetof(struct virtio_scsi_config, _field), \ 467 &(_cfg)->_field, sizeof((_cfg)->_field)) \ 468 469 static void 470 vtscsi_read_config(struct vtscsi_softc *sc, 471 struct virtio_scsi_config *scsicfg) 472 { 473 device_t dev; 474 475 dev = sc->vtscsi_dev; 476 477 bzero(scsicfg, sizeof(struct virtio_scsi_config)); 478 479 VTSCSI_GET_CONFIG(dev, num_queues, scsicfg); 480 VTSCSI_GET_CONFIG(dev, seg_max, scsicfg); 481 VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg); 482 VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg); 483 VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg); 484 VTSCSI_GET_CONFIG(dev, sense_size, scsicfg); 485 VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg); 486 VTSCSI_GET_CONFIG(dev, max_channel, scsicfg); 487 VTSCSI_GET_CONFIG(dev, max_target, scsicfg); 488 VTSCSI_GET_CONFIG(dev, max_lun, scsicfg); 489 } 490 491 #undef VTSCSI_GET_CONFIG 492 493 static int 494 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max) 495 { 496 int nsegs; 497 498 nsegs = VTSCSI_MIN_SEGMENTS; 499 500 if (seg_max > 0) { 501 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1); 502 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) 503 nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT); 504 } else 505 nsegs += 1; 506 507 return (nsegs); 508 } 509 510 511 static int 512 vtscsi_alloc_intrs(struct vtscsi_softc *sc) 513 { 514 int intrcount = virtio_intr_count(sc->vtscsi_dev); 515 int cnt, i, error; 516 517 for (i = 0; i < NELEM(sc->vtscsi_cpus); i++) 518 sc->vtscsi_cpus[i] = -1; 519 520 intrcount = imin(intrcount, 3); 521 if (intrcount < 1) 522 return (ENXIO); 523 524 cnt = intrcount; 525 error = virtio_intr_alloc(sc->vtscsi_dev, &cnt, 0, sc->vtscsi_cpus); 526 if (error != 0) { 527 virtio_intr_release(sc->vtscsi_dev); 528 return (error); 529 } 530 sc->vtscsi_nintr = cnt; 531 return (0); 532 } 533 534 static int 535 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc) 536 { 537 device_t dev = sc->vtscsi_dev; 538 struct vq_alloc_info vq_info[3]; 539 int nvqs = 3; 540 541 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, &sc->vtscsi_control_vq, 542 "%s control", device_get_nameunit(dev)); 543 544 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, &sc->vtscsi_event_vq, 545 "%s event", device_get_nameunit(dev)); 546 547 VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs, 548 &sc->vtscsi_request_vq, "%s request", device_get_nameunit(dev)); 549 550 return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); 551 } 552 553 static void 554 vtscsi_write_device_config(struct vtscsi_softc *sc) 555 { 556 557 virtio_write_dev_config_4(sc->vtscsi_dev, 558 offsetof(struct virtio_scsi_config, sense_size), 559 VIRTIO_SCSI_SENSE_SIZE); 560 561 /* 562 * This is the size in the virtio_scsi_cmd_req structure. Note 563 * this value (32) is larger than the maximum CAM CDB size (16). 564 */ 565 virtio_write_dev_config_4(sc->vtscsi_dev, 566 offsetof(struct virtio_scsi_config, cdb_size), 567 VIRTIO_SCSI_CDB_SIZE); 568 } 569 570 static int 571 vtscsi_reinit(struct vtscsi_softc *sc) 572 { 573 device_t dev; 574 int error; 575 576 dev = sc->vtscsi_dev; 577 578 error = virtio_reinit(dev, sc->vtscsi_features); 579 if (error == 0) { 580 vtscsi_write_device_config(sc); 581 vtscsi_reinit_event_vq(sc); 582 virtio_reinit_complete(dev); 583 584 vtscsi_enable_vqs_intr(sc); 585 } 586 587 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error); 588 589 return (error); 590 } 591 592 static int 593 vtscsi_alloc_cam(struct vtscsi_softc *sc) 594 { 595 device_t dev; 596 struct cam_devq *devq; 597 int openings; 598 599 dev = sc->vtscsi_dev; 600 openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS; 601 602 devq = cam_simq_alloc(openings); 603 if (devq == NULL) { 604 device_printf(dev, "cannot allocate SIM queue\n"); 605 return (ENOMEM); 606 } 607 608 sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll, 609 "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1, 610 openings, devq); 611 cam_simq_release(devq); 612 if (sc->vtscsi_sim == NULL) { 613 device_printf(dev, "cannot allocate SIM\n"); 614 return (ENOMEM); 615 } 616 617 return (0); 618 } 619 620 static int 621 vtscsi_register_cam(struct vtscsi_softc *sc) 622 { 623 device_t dev; 624 int registered, error; 625 626 dev = sc->vtscsi_dev; 627 registered = 0; 628 629 VTSCSI_LOCK(sc); 630 631 if (xpt_bus_register(sc->vtscsi_sim, 0) != CAM_SUCCESS) { 632 error = ENOMEM; 633 device_printf(dev, "cannot register XPT bus\n"); 634 goto fail; 635 } 636 637 registered = 1; 638 639 if (xpt_create_path(&sc->vtscsi_path, NULL, 640 cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD, 641 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 642 error = ENOMEM; 643 device_printf(dev, "cannot create bus path\n"); 644 goto fail; 645 } 646 647 if (vtscsi_register_async(sc) != CAM_REQ_CMP) { 648 error = EIO; 649 device_printf(dev, "cannot register async callback\n"); 650 goto fail; 651 } 652 653 VTSCSI_UNLOCK(sc); 654 655 return (0); 656 657 fail: 658 if (sc->vtscsi_path != NULL) { 659 xpt_free_path(sc->vtscsi_path); 660 sc->vtscsi_path = NULL; 661 } 662 663 if (registered != 0) 664 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 665 666 VTSCSI_UNLOCK(sc); 667 668 return (error); 669 } 670 671 static void 672 vtscsi_free_cam(struct vtscsi_softc *sc) 673 { 674 675 VTSCSI_LOCK(sc); 676 677 if (sc->vtscsi_path != NULL) { 678 vtscsi_deregister_async(sc); 679 680 xpt_free_path(sc->vtscsi_path); 681 sc->vtscsi_path = NULL; 682 683 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim)); 684 } 685 686 if (sc->vtscsi_sim != NULL) { 687 cam_sim_free(sc->vtscsi_sim); 688 sc->vtscsi_sim = NULL; 689 } 690 691 VTSCSI_UNLOCK(sc); 692 } 693 694 static void 695 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg) 696 { 697 struct cam_sim *sim; 698 struct vtscsi_softc *sc; 699 700 sim = cb_arg; 701 sc = cam_sim_softc(sim); 702 703 vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code); 704 705 /* 706 * TODO Once QEMU supports event reporting, we should 707 * (un)subscribe to events here. 708 */ 709 switch (code) { 710 case AC_FOUND_DEVICE: 711 break; 712 case AC_LOST_DEVICE: 713 break; 714 } 715 } 716 717 static int 718 vtscsi_register_async(struct vtscsi_softc *sc) 719 { 720 struct ccb_setasync *csa; 721 u_int32_t status; 722 723 csa = &xpt_alloc_ccb()->csa; 724 725 xpt_setup_ccb(&csa->ccb_h, sc->vtscsi_path, 5); 726 csa->ccb_h.func_code = XPT_SASYNC_CB; 727 csa->event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE; 728 csa->callback = vtscsi_cam_async; 729 csa->callback_arg = sc->vtscsi_sim; 730 731 xpt_action((union ccb *)csa); 732 status = csa->ccb_h.status; 733 xpt_free_ccb(&csa->ccb_h); 734 735 return status; 736 } 737 738 static void 739 vtscsi_deregister_async(struct vtscsi_softc *sc) 740 { 741 struct ccb_setasync *csa; 742 743 csa = &xpt_alloc_ccb()->csa; 744 xpt_setup_ccb(&csa->ccb_h, sc->vtscsi_path, 5); 745 csa->ccb_h.func_code = XPT_SASYNC_CB; 746 csa->event_enable = 0; 747 csa->callback = vtscsi_cam_async; 748 csa->callback_arg = sc->vtscsi_sim; 749 xpt_action((union ccb *)csa); 750 xpt_free_ccb(&csa->ccb_h); 751 } 752 753 static void 754 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb) 755 { 756 struct vtscsi_softc *sc; 757 struct ccb_hdr *ccbh; 758 759 sc = cam_sim_softc(sim); 760 ccbh = &ccb->ccb_h; 761 762 VTSCSI_LOCK_OWNED(sc); 763 764 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) { 765 /* 766 * The VTSCSI_MTX is briefly dropped between setting 767 * VTSCSI_FLAG_DETACH and deregistering with CAM, so 768 * drop any CCBs that come in during that window. 769 */ 770 ccbh->status = CAM_NO_HBA; 771 xpt_done(ccb); 772 return; 773 } 774 775 switch (ccbh->func_code) { 776 case XPT_SCSI_IO: 777 vtscsi_cam_scsi_io(sc, sim, ccb); 778 break; 779 780 case XPT_SET_TRAN_SETTINGS: 781 ccbh->status = CAM_FUNC_NOTAVAIL; 782 xpt_done(ccb); 783 break; 784 785 case XPT_GET_TRAN_SETTINGS: 786 vtscsi_cam_get_tran_settings(sc, ccb); 787 break; 788 789 case XPT_RESET_BUS: 790 vtscsi_cam_reset_bus(sc, ccb); 791 break; 792 793 case XPT_RESET_DEV: 794 vtscsi_cam_reset_dev(sc, ccb); 795 break; 796 797 case XPT_ABORT: 798 vtscsi_cam_abort(sc, ccb); 799 break; 800 801 case XPT_CALC_GEOMETRY: 802 cam_calc_geometry(&ccb->ccg, 1); 803 xpt_done(ccb); 804 break; 805 806 case XPT_PATH_INQ: 807 vtscsi_cam_path_inquiry(sc, sim, ccb); 808 break; 809 810 default: 811 vtscsi_dprintf(sc, VTSCSI_ERROR, 812 "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code); 813 814 ccbh->status = CAM_REQ_INVALID; 815 xpt_done(ccb); 816 break; 817 } 818 } 819 820 static void 821 vtscsi_cam_poll(struct cam_sim *sim) 822 { 823 struct vtscsi_softc *sc; 824 825 sc = cam_sim_softc(sim); 826 827 vtscsi_complete_vqs_locked(sc); 828 } 829 830 static void 831 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim, 832 union ccb *ccb) 833 { 834 struct ccb_hdr *ccbh; 835 struct ccb_scsiio *csio; 836 int error; 837 838 ccbh = &ccb->ccb_h; 839 csio = &ccb->csio; 840 841 if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) { 842 error = EINVAL; 843 ccbh->status = CAM_REQ_INVALID; 844 goto done; 845 } 846 847 #ifndef __DragonFly__ /* XXX swildner */ 848 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH && 849 (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) { 850 error = EINVAL; 851 ccbh->status = CAM_REQ_INVALID; 852 goto done; 853 } 854 #endif 855 856 error = vtscsi_start_scsi_cmd(sc, ccb); 857 858 done: 859 if (error) { 860 vtscsi_dprintf(sc, VTSCSI_ERROR, 861 "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status); 862 xpt_done(ccb); 863 } 864 } 865 866 static void 867 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb) 868 { 869 struct ccb_trans_settings *cts; 870 struct ccb_trans_settings_scsi *scsi; 871 872 cts = &ccb->cts; 873 scsi = &cts->proto_specific.scsi; 874 875 cts->protocol = PROTO_SCSI; 876 cts->protocol_version = SCSI_REV_SPC3; 877 cts->transport = XPORT_SAS; 878 cts->transport_version = 0; 879 880 scsi->valid = CTS_SCSI_VALID_TQ; 881 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 882 883 ccb->ccb_h.status = CAM_REQ_CMP; 884 xpt_done(ccb); 885 } 886 887 static void 888 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb) 889 { 890 int error; 891 892 error = vtscsi_reset_bus(sc); 893 if (error == 0) 894 ccb->ccb_h.status = CAM_REQ_CMP; 895 else 896 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 897 898 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n", 899 error, ccb, ccb->ccb_h.status); 900 901 xpt_done(ccb); 902 } 903 904 static void 905 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb) 906 { 907 struct ccb_hdr *ccbh; 908 struct vtscsi_request *req; 909 int error; 910 911 ccbh = &ccb->ccb_h; 912 913 req = vtscsi_dequeue_request(sc); 914 if (req == NULL) { 915 error = EAGAIN; 916 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 917 goto fail; 918 } 919 920 req->vsr_ccb = ccb; 921 922 error = vtscsi_execute_reset_dev_cmd(sc, req); 923 if (error == 0) 924 return; 925 926 vtscsi_enqueue_request(sc, req); 927 928 fail: 929 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 930 error, req, ccb); 931 932 if (error == EAGAIN) 933 ccbh->status = CAM_RESRC_UNAVAIL; 934 else 935 ccbh->status = CAM_REQ_CMP_ERR; 936 937 xpt_done(ccb); 938 } 939 940 static void 941 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb) 942 { 943 struct vtscsi_request *req; 944 struct ccb_hdr *ccbh; 945 int error; 946 947 ccbh = &ccb->ccb_h; 948 949 req = vtscsi_dequeue_request(sc); 950 if (req == NULL) { 951 error = EAGAIN; 952 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 953 goto fail; 954 } 955 956 req->vsr_ccb = ccb; 957 958 error = vtscsi_execute_abort_task_cmd(sc, req); 959 if (error == 0) 960 return; 961 962 vtscsi_enqueue_request(sc, req); 963 964 fail: 965 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n", 966 error, req, ccb); 967 968 if (error == EAGAIN) 969 ccbh->status = CAM_RESRC_UNAVAIL; 970 else 971 ccbh->status = CAM_REQ_CMP_ERR; 972 973 xpt_done(ccb); 974 } 975 976 static void 977 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim, 978 union ccb *ccb) 979 { 980 device_t dev; 981 struct ccb_pathinq *cpi; 982 983 dev = sc->vtscsi_dev; 984 cpi = &ccb->cpi; 985 986 vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb); 987 988 cpi->version_num = 1; 989 cpi->hba_inquiry = PI_TAG_ABLE; 990 cpi->target_sprt = 0; 991 cpi->hba_misc = PIM_SEQSCAN; 992 if (vtscsi_bus_reset_disable != 0) 993 cpi->hba_misc |= PIM_NOBUSRESET; 994 cpi->hba_eng_cnt = 0; 995 996 cpi->max_target = sc->vtscsi_max_target; 997 cpi->max_lun = sc->vtscsi_max_lun; 998 cpi->initiator_id = VTSCSI_INITIATOR_ID; 999 1000 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 1001 strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN); 1002 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 1003 1004 cpi->unit_number = cam_sim_unit(sim); 1005 cpi->bus_id = cam_sim_bus(sim); 1006 1007 cpi->base_transfer_speed = 300000; 1008 1009 cpi->protocol = PROTO_SCSI; 1010 cpi->protocol_version = SCSI_REV_SPC3; 1011 cpi->transport = XPORT_SAS; 1012 cpi->transport_version = 0; 1013 1014 cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) * 1015 PAGE_SIZE; 1016 1017 #if 0 1018 cpi->hba_vendor = virtio_get_vendor(dev); 1019 cpi->hba_device = virtio_get_device(dev); 1020 cpi->hba_subvendor = virtio_get_subvendor(dev); 1021 cpi->hba_subdevice = virtio_get_subdevice(dev); 1022 #endif 1023 1024 ccb->ccb_h.status = CAM_REQ_CMP; 1025 xpt_done(ccb); 1026 } 1027 1028 static int 1029 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg, 1030 struct ccb_scsiio *csio) 1031 { 1032 struct ccb_hdr *ccbh; 1033 struct bus_dma_segment *dseg; 1034 int i, error; 1035 1036 ccbh = &csio->ccb_h; 1037 error = 0; 1038 1039 if ((ccbh->flags & CAM_SCATTER_VALID) == 0) { 1040 1041 if ((ccbh->flags & CAM_DATA_PHYS) == 0) 1042 error = sglist_append(sg, 1043 csio->data_ptr, csio->dxfer_len); 1044 else 1045 error = sglist_append_phys(sg, 1046 (vm_paddr_t)(vm_offset_t) csio->data_ptr, 1047 csio->dxfer_len); 1048 } else { 1049 1050 for (i = 0; i < csio->sglist_cnt && error == 0; i++) { 1051 dseg = &((struct bus_dma_segment *)csio->data_ptr)[i]; 1052 1053 if ((ccbh->flags & CAM_SG_LIST_PHYS) == 0) 1054 error = sglist_append(sg, 1055 (void *)(vm_offset_t) dseg->ds_addr, 1056 dseg->ds_len); 1057 else 1058 error = sglist_append_phys(sg, 1059 (vm_paddr_t) dseg->ds_addr, dseg->ds_len); 1060 } 1061 } 1062 1063 return (error); 1064 } 1065 1066 static int 1067 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req, 1068 int *readable, int *writable) 1069 { 1070 struct sglist *sg; 1071 struct ccb_hdr *ccbh; 1072 struct ccb_scsiio *csio; 1073 struct virtio_scsi_cmd_req *cmd_req; 1074 struct virtio_scsi_cmd_resp *cmd_resp; 1075 int error; 1076 1077 sg = sc->vtscsi_sglist; 1078 csio = &req->vsr_ccb->csio; 1079 ccbh = &csio->ccb_h; 1080 cmd_req = &req->vsr_cmd_req; 1081 cmd_resp = &req->vsr_cmd_resp; 1082 1083 sglist_reset(sg); 1084 1085 sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req)); 1086 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) { 1087 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1088 /* At least one segment must be left for the response. */ 1089 if (error || sg->sg_nseg == sg->sg_maxseg) 1090 goto fail; 1091 } 1092 1093 *readable = sg->sg_nseg; 1094 1095 sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp)); 1096 if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) { 1097 error = vtscsi_sg_append_scsi_buf(sc, sg, csio); 1098 if (error) 1099 goto fail; 1100 } 1101 1102 *writable = sg->sg_nseg - *readable; 1103 1104 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d " 1105 "writable=%d\n", req, ccbh, *readable, *writable); 1106 1107 return (0); 1108 1109 fail: 1110 /* 1111 * This should never happen unless maxio was incorrectly set. 1112 */ 1113 vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0); 1114 1115 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p " 1116 "nseg=%d maxseg=%d\n", 1117 error, req, ccbh, sg->sg_nseg, sg->sg_maxseg); 1118 1119 return (EFBIG); 1120 } 1121 1122 static int 1123 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1124 { 1125 struct sglist *sg; 1126 struct virtqueue *vq; 1127 struct ccb_scsiio *csio; 1128 struct ccb_hdr *ccbh; 1129 struct virtio_scsi_cmd_req *cmd_req; 1130 struct virtio_scsi_cmd_resp *cmd_resp; 1131 int readable, writable, error; 1132 1133 sg = sc->vtscsi_sglist; 1134 vq = sc->vtscsi_request_vq; 1135 csio = &req->vsr_ccb->csio; 1136 ccbh = &csio->ccb_h; 1137 cmd_req = &req->vsr_cmd_req; 1138 cmd_resp = &req->vsr_cmd_resp; 1139 1140 vtscsi_init_scsi_cmd_req(csio, cmd_req); 1141 1142 error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable); 1143 if (error) 1144 return (error); 1145 1146 req->vsr_complete = vtscsi_complete_scsi_cmd; 1147 cmd_resp->response = -1; 1148 1149 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1150 if (error) { 1151 vtscsi_dprintf(sc, VTSCSI_ERROR, 1152 "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh); 1153 1154 ccbh->status = CAM_REQUEUE_REQ; 1155 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ); 1156 return (error); 1157 } 1158 1159 ccbh->status |= CAM_SIM_QUEUED; 1160 ccbh->ccbh_vtscsi_req = req; 1161 1162 virtqueue_notify(vq, NULL); 1163 1164 if (ccbh->timeout != CAM_TIME_INFINITY) { 1165 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET; 1166 callout_reset(&req->vsr_callout, ccbh->timeout * hz / 1000, 1167 vtscsi_timedout_scsi_cmd, req); 1168 } 1169 1170 vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n", 1171 req, ccbh); 1172 1173 return (0); 1174 } 1175 1176 static int 1177 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb) 1178 { 1179 struct vtscsi_request *req; 1180 int error; 1181 1182 req = vtscsi_dequeue_request(sc); 1183 if (req == NULL) { 1184 ccb->ccb_h.status = CAM_REQUEUE_REQ; 1185 vtscsi_freeze_simq(sc, VTSCSI_REQUEST); 1186 return (ENOBUFS); 1187 } 1188 1189 req->vsr_ccb = ccb; 1190 1191 error = vtscsi_execute_scsi_cmd(sc, req); 1192 if (error) 1193 vtscsi_enqueue_request(sc, req); 1194 1195 return (error); 1196 } 1197 1198 static void 1199 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1200 struct vtscsi_request *req) 1201 { 1202 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1203 struct vtscsi_request *to_req; 1204 uint8_t response; 1205 1206 tmf_resp = &req->vsr_tmf_resp; 1207 response = tmf_resp->response; 1208 to_req = req->vsr_timedout_req; 1209 1210 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n", 1211 req, to_req, response); 1212 1213 vtscsi_enqueue_request(sc, req); 1214 1215 /* 1216 * The timedout request could have completed between when the 1217 * abort task was sent and when the host processed it. 1218 */ 1219 if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT) 1220 return; 1221 1222 /* The timedout request was successfully aborted. */ 1223 if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) 1224 return; 1225 1226 /* Don't bother if the device is going away. */ 1227 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1228 return; 1229 1230 /* The timedout request will be aborted by the reset. */ 1231 if (sc->vtscsi_flags & VTSCSI_FLAG_RESET) 1232 return; 1233 1234 vtscsi_reset_bus(sc); 1235 } 1236 1237 static int 1238 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc, 1239 struct vtscsi_request *to_req) 1240 { 1241 struct sglist *sg; 1242 struct ccb_hdr *to_ccbh; 1243 struct vtscsi_request *req; 1244 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1245 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1246 int error; 1247 1248 sg = sc->vtscsi_sglist; 1249 to_ccbh = &to_req->vsr_ccb->ccb_h; 1250 1251 req = vtscsi_dequeue_request(sc); 1252 if (req == NULL) { 1253 error = ENOBUFS; 1254 goto fail; 1255 } 1256 1257 tmf_req = &req->vsr_tmf_req; 1258 tmf_resp = &req->vsr_tmf_resp; 1259 1260 vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1261 (uintptr_t) to_ccbh, tmf_req); 1262 1263 sglist_reset(sg); 1264 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1265 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1266 1267 req->vsr_timedout_req = to_req; 1268 req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd; 1269 tmf_resp->response = -1; 1270 1271 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1272 VTSCSI_EXECUTE_ASYNC); 1273 if (error == 0) 1274 return (0); 1275 1276 vtscsi_enqueue_request(sc, req); 1277 1278 fail: 1279 vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p " 1280 "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh); 1281 1282 return (error); 1283 } 1284 1285 static void 1286 vtscsi_timedout_scsi_cmd(void *xreq) 1287 { 1288 struct vtscsi_softc *sc; 1289 struct vtscsi_request *to_req; 1290 1291 to_req = xreq; 1292 sc = to_req->vsr_softc; 1293 1294 vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n", 1295 to_req, to_req->vsr_ccb, to_req->vsr_state); 1296 1297 /* Don't bother if the device is going away. */ 1298 if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) 1299 return; 1300 1301 /* 1302 * Bail if the request is not in use. We likely raced when 1303 * stopping the callout handler or it has already been aborted. 1304 */ 1305 if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE || 1306 (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0) 1307 return; 1308 1309 /* 1310 * Complete the request queue in case the timedout request is 1311 * actually just pending. 1312 */ 1313 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1314 if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE) 1315 return; 1316 1317 sc->vtscsi_stats.scsi_cmd_timeouts++; 1318 to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT; 1319 1320 if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0) 1321 return; 1322 1323 vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n"); 1324 vtscsi_reset_bus(sc); 1325 } 1326 1327 static cam_status 1328 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp) 1329 { 1330 cam_status status; 1331 1332 switch (cmd_resp->response) { 1333 case VIRTIO_SCSI_S_OK: 1334 status = CAM_REQ_CMP; 1335 break; 1336 case VIRTIO_SCSI_S_OVERRUN: 1337 status = CAM_DATA_RUN_ERR; 1338 break; 1339 case VIRTIO_SCSI_S_ABORTED: 1340 status = CAM_REQ_ABORTED; 1341 break; 1342 case VIRTIO_SCSI_S_BAD_TARGET: 1343 /* 1344 * A CAM_SEL_TIMEOUT here will cause the entire device to 1345 * be lost, which is not desirable when scanning LUNs. 1346 * Use CAM_DEV_NOT_THERE instead. 1347 */ 1348 status = CAM_DEV_NOT_THERE; 1349 break; 1350 case VIRTIO_SCSI_S_RESET: 1351 status = CAM_SCSI_BUS_RESET; 1352 break; 1353 case VIRTIO_SCSI_S_BUSY: 1354 status = CAM_SCSI_BUSY; 1355 break; 1356 case VIRTIO_SCSI_S_TRANSPORT_FAILURE: 1357 case VIRTIO_SCSI_S_TARGET_FAILURE: 1358 case VIRTIO_SCSI_S_NEXUS_FAILURE: 1359 status = CAM_SCSI_IT_NEXUS_LOST; 1360 break; 1361 default: /* VIRTIO_SCSI_S_FAILURE */ 1362 status = CAM_REQ_CMP_ERR; 1363 break; 1364 } 1365 1366 return (status); 1367 } 1368 1369 static cam_status 1370 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc, 1371 struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp) 1372 { 1373 cam_status status; 1374 1375 csio->scsi_status = cmd_resp->status; 1376 csio->resid = cmd_resp->resid; 1377 1378 if (csio->scsi_status == SCSI_STATUS_OK) 1379 status = CAM_REQ_CMP; 1380 else 1381 status = CAM_SCSI_STATUS_ERROR; 1382 1383 if (cmd_resp->sense_len > 0) { 1384 status |= CAM_AUTOSNS_VALID; 1385 1386 if (cmd_resp->sense_len < csio->sense_len) 1387 csio->sense_resid = csio->sense_len - 1388 cmd_resp->sense_len; 1389 else 1390 csio->sense_resid = 0; 1391 1392 bzero(&csio->sense_data, sizeof(csio->sense_data)); 1393 memcpy(cmd_resp->sense, &csio->sense_data, 1394 csio->sense_len - csio->sense_resid); 1395 } 1396 1397 vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR, 1398 "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n", 1399 csio, csio->scsi_status, csio->resid, csio->sense_resid); 1400 1401 return (status); 1402 } 1403 1404 static void 1405 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req) 1406 { 1407 struct ccb_hdr *ccbh; 1408 struct ccb_scsiio *csio; 1409 struct virtio_scsi_cmd_resp *cmd_resp; 1410 cam_status status; 1411 1412 csio = &req->vsr_ccb->csio; 1413 ccbh = &csio->ccb_h; 1414 cmd_resp = &req->vsr_cmd_resp; 1415 1416 KASSERT(ccbh->ccbh_vtscsi_req == req, 1417 ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req)); 1418 1419 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1420 callout_stop(&req->vsr_callout); 1421 1422 status = vtscsi_scsi_cmd_cam_status(cmd_resp); 1423 if (status == CAM_REQ_ABORTED) { 1424 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT) 1425 status = CAM_CMD_TIMEOUT; 1426 } else if (status == CAM_REQ_CMP) 1427 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp); 1428 1429 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1430 status |= CAM_DEV_QFRZN; 1431 xpt_freeze_devq(ccbh->path, 1); 1432 } 1433 1434 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 1435 status |= CAM_RELEASE_SIMQ; 1436 1437 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n", 1438 req, ccbh, status); 1439 1440 ccbh->status = status; 1441 xpt_done(req->vsr_ccb); 1442 vtscsi_enqueue_request(sc, req); 1443 } 1444 1445 static void 1446 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req) 1447 { 1448 1449 /* XXX We probably shouldn't poll forever. */ 1450 req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED; 1451 do 1452 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1453 while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0); 1454 1455 req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED; 1456 } 1457 1458 static int 1459 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req, 1460 struct sglist *sg, int readable, int writable, int flag) 1461 { 1462 struct virtqueue *vq; 1463 int error; 1464 1465 vq = sc->vtscsi_control_vq; 1466 1467 KKASSERT(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL); 1468 1469 error = virtqueue_enqueue(vq, req, sg, readable, writable); 1470 if (error) { 1471 /* 1472 * Return EAGAIN when the virtqueue does not have enough 1473 * descriptors available. 1474 */ 1475 if (error == ENOSPC || error == EMSGSIZE) 1476 error = EAGAIN; 1477 1478 return (error); 1479 } 1480 1481 virtqueue_notify(vq, NULL); 1482 if (flag == VTSCSI_EXECUTE_POLL) 1483 vtscsi_poll_ctrl_req(sc, req); 1484 1485 return (0); 1486 } 1487 1488 static void 1489 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc, 1490 struct vtscsi_request *req) 1491 { 1492 union ccb *ccb; 1493 struct ccb_hdr *ccbh; 1494 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1495 1496 ccb = req->vsr_ccb; 1497 ccbh = &ccb->ccb_h; 1498 tmf_resp = &req->vsr_tmf_resp; 1499 1500 switch (tmf_resp->response) { 1501 case VIRTIO_SCSI_S_FUNCTION_COMPLETE: 1502 ccbh->status = CAM_REQ_CMP; 1503 break; 1504 case VIRTIO_SCSI_S_FUNCTION_REJECTED: 1505 ccbh->status = CAM_UA_ABORT; 1506 break; 1507 default: 1508 ccbh->status = CAM_REQ_CMP_ERR; 1509 break; 1510 } 1511 1512 xpt_done(ccb); 1513 vtscsi_enqueue_request(sc, req); 1514 } 1515 1516 static int 1517 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc, 1518 struct vtscsi_request *req) 1519 { 1520 struct sglist *sg; 1521 struct ccb_abort *cab; 1522 struct ccb_hdr *ccbh; 1523 struct ccb_hdr *abort_ccbh; 1524 struct vtscsi_request *abort_req; 1525 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1526 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1527 int error; 1528 1529 sg = sc->vtscsi_sglist; 1530 cab = &req->vsr_ccb->cab; 1531 ccbh = &cab->ccb_h; 1532 tmf_req = &req->vsr_tmf_req; 1533 tmf_resp = &req->vsr_tmf_resp; 1534 1535 /* CCB header and request that's to be aborted. */ 1536 abort_ccbh = &cab->abort_ccb->ccb_h; 1537 abort_req = abort_ccbh->ccbh_vtscsi_req; 1538 1539 if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) { 1540 error = EINVAL; 1541 goto fail; 1542 } 1543 1544 /* Only attempt to abort requests that could be in-flight. */ 1545 if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) { 1546 error = EALREADY; 1547 goto fail; 1548 } 1549 1550 abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED; 1551 if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) 1552 callout_stop(&abort_req->vsr_callout); 1553 1554 vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, 1555 (uintptr_t) abort_ccbh, tmf_req); 1556 1557 sglist_reset(sg); 1558 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1559 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1560 1561 req->vsr_complete = vtscsi_complete_abort_task_cmd; 1562 tmf_resp->response = -1; 1563 1564 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1565 VTSCSI_EXECUTE_ASYNC); 1566 1567 fail: 1568 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p " 1569 "abort_req=%p\n", error, req, abort_ccbh, abort_req); 1570 1571 return (error); 1572 } 1573 1574 static void 1575 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc, 1576 struct vtscsi_request *req) 1577 { 1578 union ccb *ccb; 1579 struct ccb_hdr *ccbh; 1580 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1581 1582 ccb = req->vsr_ccb; 1583 ccbh = &ccb->ccb_h; 1584 tmf_resp = &req->vsr_tmf_resp; 1585 1586 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n", 1587 req, ccb, tmf_resp->response); 1588 1589 if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) { 1590 ccbh->status = CAM_REQ_CMP; 1591 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id, 1592 ccbh->target_lun); 1593 } else 1594 ccbh->status = CAM_REQ_CMP_ERR; 1595 1596 xpt_done(ccb); 1597 vtscsi_enqueue_request(sc, req); 1598 } 1599 1600 static int 1601 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc, 1602 struct vtscsi_request *req) 1603 { 1604 struct sglist *sg; 1605 struct ccb_resetdev *crd; 1606 struct ccb_hdr *ccbh; 1607 struct virtio_scsi_ctrl_tmf_req *tmf_req; 1608 struct virtio_scsi_ctrl_tmf_resp *tmf_resp; 1609 uint32_t subtype; 1610 int error; 1611 1612 sg = sc->vtscsi_sglist; 1613 crd = &req->vsr_ccb->crd; 1614 ccbh = &crd->ccb_h; 1615 tmf_req = &req->vsr_tmf_req; 1616 tmf_resp = &req->vsr_tmf_resp; 1617 1618 if (ccbh->target_lun == CAM_LUN_WILDCARD) 1619 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET; 1620 else 1621 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET; 1622 1623 vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); 1624 1625 sglist_reset(sg); 1626 sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req)); 1627 sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp)); 1628 1629 req->vsr_complete = vtscsi_complete_reset_dev_cmd; 1630 tmf_resp->response = -1; 1631 1632 error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1, 1633 VTSCSI_EXECUTE_ASYNC); 1634 1635 vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n", 1636 error, req, ccbh); 1637 1638 return (error); 1639 } 1640 1641 static void 1642 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id) 1643 { 1644 1645 *target_id = lun[1]; 1646 *lun_id = (lun[2] << 8) | lun[3]; 1647 } 1648 1649 static void 1650 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[]) 1651 { 1652 1653 lun[0] = 1; 1654 lun[1] = ccbh->target_id; 1655 lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F); 1656 lun[3] = ccbh->target_lun & 0xFF; 1657 } 1658 1659 static void 1660 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, 1661 struct virtio_scsi_cmd_req *cmd_req) 1662 { 1663 uint8_t attr; 1664 1665 switch (csio->tag_action) { 1666 case MSG_HEAD_OF_Q_TAG: 1667 attr = VIRTIO_SCSI_S_HEAD; 1668 break; 1669 case MSG_ORDERED_Q_TAG: 1670 attr = VIRTIO_SCSI_S_ORDERED; 1671 break; 1672 case MSG_ACA_TASK: 1673 attr = VIRTIO_SCSI_S_ACA; 1674 break; 1675 default: /* MSG_SIMPLE_Q_TAG */ 1676 attr = VIRTIO_SCSI_S_SIMPLE; 1677 break; 1678 } 1679 1680 vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun); 1681 cmd_req->tag = (uintptr_t) csio; 1682 cmd_req->task_attr = attr; 1683 1684 memcpy(cmd_req->cdb, 1685 csio->ccb_h.flags & CAM_CDB_POINTER ? 1686 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes, 1687 csio->cdb_len); 1688 } 1689 1690 static void 1691 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, 1692 uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) 1693 { 1694 1695 vtscsi_set_request_lun(ccbh, tmf_req->lun); 1696 1697 tmf_req->type = VIRTIO_SCSI_T_TMF; 1698 tmf_req->subtype = subtype; 1699 tmf_req->tag = tag; 1700 } 1701 1702 static void 1703 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason) 1704 { 1705 int frozen; 1706 1707 frozen = sc->vtscsi_frozen; 1708 1709 if (reason & VTSCSI_REQUEST && 1710 (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0) 1711 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS; 1712 1713 if (reason & VTSCSI_REQUEST_VQ && 1714 (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0) 1715 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL; 1716 1717 /* Freeze the SIMQ if transitioned to frozen. */ 1718 if (frozen == 0 && sc->vtscsi_frozen != 0) { 1719 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n"); 1720 xpt_freeze_simq(sc->vtscsi_sim, 1); 1721 } 1722 } 1723 1724 static int 1725 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason) 1726 { 1727 int thawed; 1728 1729 if (sc->vtscsi_frozen == 0 || reason == 0) 1730 return (0); 1731 1732 if (reason & VTSCSI_REQUEST && 1733 sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) 1734 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS; 1735 1736 if (reason & VTSCSI_REQUEST_VQ && 1737 sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) 1738 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL; 1739 1740 thawed = sc->vtscsi_frozen == 0; 1741 if (thawed != 0) 1742 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n"); 1743 1744 return (thawed); 1745 } 1746 1747 static void 1748 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code, 1749 target_id_t target_id, lun_id_t lun_id) 1750 { 1751 struct cam_path *path; 1752 1753 /* Use the wildcard path from our softc for bus announcements. */ 1754 if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) { 1755 xpt_async(ac_code, sc->vtscsi_path, NULL); 1756 return; 1757 } 1758 1759 if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim), 1760 target_id, lun_id) != CAM_REQ_CMP) { 1761 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n"); 1762 return; 1763 } 1764 1765 xpt_async(ac_code, path, NULL); 1766 xpt_free_path(path); 1767 } 1768 1769 static void 1770 vtscsi_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb) 1771 { 1772 xpt_free_path(ccb->ccb_h.path); 1773 xpt_free_ccb(&ccb->ccb_h); 1774 } 1775 1776 static void 1777 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id, 1778 lun_id_t lun_id) 1779 { 1780 union ccb *ccb; 1781 cam_status status; 1782 1783 ccb = xpt_alloc_ccb(); 1784 if (ccb == NULL) { 1785 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1786 return; 1787 } 1788 1789 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1790 cam_sim_path(sc->vtscsi_sim), target_id, lun_id); 1791 if (status != CAM_REQ_CMP) { 1792 xpt_free_ccb(&ccb->ccb_h); 1793 return; 1794 } 1795 1796 xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/); 1797 ccb->ccb_h.func_code = XPT_SCAN_LUN; 1798 ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback; 1799 ccb->crcn.flags = CAM_FLAG_NONE; 1800 xpt_action(ccb); 1801 } 1802 1803 static void 1804 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc) 1805 { 1806 union ccb *ccb; 1807 cam_status status; 1808 1809 ccb = xpt_alloc_ccb(); 1810 if (ccb == NULL) { 1811 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n"); 1812 return; 1813 } 1814 1815 status = xpt_create_path(&ccb->ccb_h.path, NULL, 1816 cam_sim_path(sc->vtscsi_sim), 1817 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); 1818 if (status != CAM_REQ_CMP) { 1819 xpt_free_ccb(&ccb->ccb_h); 1820 return; 1821 } 1822 1823 xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/); 1824 ccb->ccb_h.func_code = XPT_SCAN_BUS; 1825 ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback; 1826 ccb->crcn.flags = CAM_FLAG_NONE; 1827 xpt_action(ccb); 1828 } 1829 1830 static void 1831 vtscsi_transport_reset_event(struct vtscsi_softc *sc, 1832 struct virtio_scsi_event *event) 1833 { 1834 target_id_t target_id; 1835 lun_id_t lun_id; 1836 1837 vtscsi_get_request_lun(event->lun, &target_id, &lun_id); 1838 1839 switch (event->reason) { 1840 case VIRTIO_SCSI_EVT_RESET_RESCAN: 1841 case VIRTIO_SCSI_EVT_RESET_REMOVED: 1842 vtscsi_execute_rescan(sc, target_id, lun_id); 1843 break; 1844 default: 1845 device_printf(sc->vtscsi_dev, 1846 "unhandled transport event reason: %d\n", event->reason); 1847 break; 1848 } 1849 } 1850 1851 static void 1852 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event) 1853 { 1854 int error; 1855 1856 if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) { 1857 switch (event->event) { 1858 case VIRTIO_SCSI_T_TRANSPORT_RESET: 1859 vtscsi_transport_reset_event(sc, event); 1860 break; 1861 default: 1862 device_printf(sc->vtscsi_dev, 1863 "unhandled event: %d\n", event->event); 1864 break; 1865 } 1866 } else 1867 vtscsi_execute_rescan_bus(sc); 1868 1869 /* 1870 * This should always be successful since the buffer 1871 * was just dequeued. 1872 */ 1873 error = vtscsi_enqueue_event_buf(sc, event); 1874 KASSERT(error == 0, 1875 ("cannot requeue event buffer: %d", error)); 1876 } 1877 1878 static int 1879 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc, 1880 struct virtio_scsi_event *event) 1881 { 1882 struct sglist *sg; 1883 struct virtqueue *vq; 1884 int size, error; 1885 1886 sg = sc->vtscsi_sglist; 1887 vq = sc->vtscsi_event_vq; 1888 size = sc->vtscsi_event_buf_size; 1889 1890 bzero(event, size); 1891 1892 sglist_reset(sg); 1893 error = sglist_append(sg, event, size); 1894 if (error) 1895 return (error); 1896 1897 error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg); 1898 if (error) 1899 return (error); 1900 1901 virtqueue_notify(vq, NULL); 1902 1903 return (0); 1904 } 1905 1906 static int 1907 vtscsi_init_event_vq(struct vtscsi_softc *sc) 1908 { 1909 struct virtio_scsi_event *event; 1910 int i, size, error; 1911 1912 /* 1913 * The first release of QEMU with VirtIO SCSI support would crash 1914 * when attempting to notify the event virtqueue. This was fixed 1915 * when hotplug support was added. 1916 */ 1917 if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) 1918 size = sc->vtscsi_event_buf_size; 1919 else 1920 size = 0; 1921 1922 if (size < sizeof(struct virtio_scsi_event)) 1923 return (0); 1924 1925 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1926 event = &sc->vtscsi_event_bufs[i]; 1927 1928 error = vtscsi_enqueue_event_buf(sc, event); 1929 if (error) 1930 break; 1931 } 1932 1933 /* 1934 * Even just one buffer is enough. Missed events are 1935 * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag. 1936 */ 1937 if (i > 0) 1938 error = 0; 1939 1940 return (error); 1941 } 1942 1943 static void 1944 vtscsi_reinit_event_vq(struct vtscsi_softc *sc) 1945 { 1946 struct virtio_scsi_event *event; 1947 int i, error; 1948 1949 if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 || 1950 sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event)) 1951 return; 1952 1953 for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) { 1954 event = &sc->vtscsi_event_bufs[i]; 1955 1956 error = vtscsi_enqueue_event_buf(sc, event); 1957 if (error) 1958 break; 1959 } 1960 1961 KASSERT(i > 0, ("cannot reinit event vq: %d", error)); 1962 } 1963 1964 static void 1965 vtscsi_drain_event_vq(struct vtscsi_softc *sc) 1966 { 1967 struct virtqueue *vq; 1968 int last; 1969 1970 vq = sc->vtscsi_event_vq; 1971 last = 0; 1972 1973 while (virtqueue_drain(vq, &last) != NULL) 1974 ; 1975 1976 KASSERT(virtqueue_empty(vq), ("eventvq not empty")); 1977 } 1978 1979 static void 1980 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc) 1981 { 1982 1983 VTSCSI_LOCK_OWNED(sc); 1984 1985 if (sc->vtscsi_request_vq != NULL) 1986 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 1987 if (sc->vtscsi_control_vq != NULL) 1988 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 1989 } 1990 1991 static void 1992 vtscsi_complete_vqs(struct vtscsi_softc *sc) 1993 { 1994 1995 VTSCSI_LOCK(sc); 1996 vtscsi_complete_vqs_locked(sc); 1997 VTSCSI_UNLOCK(sc); 1998 } 1999 2000 static void 2001 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2002 { 2003 union ccb *ccb; 2004 int detach; 2005 2006 ccb = req->vsr_ccb; 2007 2008 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb); 2009 2010 /* 2011 * The callout must be drained when detaching since the request is 2012 * about to be freed. The VTSCSI_MTX must not be held for this in 2013 * case the callout is pending because there is a deadlock potential. 2014 * Otherwise, the virtqueue is being drained because of a bus reset 2015 * so we only need to attempt to stop the callouts. 2016 */ 2017 detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0; 2018 if (detach != 0) 2019 VTSCSI_LOCK_NOTOWNED(sc); 2020 else 2021 VTSCSI_LOCK_OWNED(sc); 2022 2023 if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) { 2024 if (detach != 0) 2025 callout_drain(&req->vsr_callout); 2026 else 2027 callout_stop(&req->vsr_callout); 2028 } 2029 2030 if (ccb != NULL) { 2031 if (detach != 0) { 2032 VTSCSI_LOCK(sc); 2033 ccb->ccb_h.status = CAM_NO_HBA; 2034 } else 2035 ccb->ccb_h.status = CAM_REQUEUE_REQ; 2036 xpt_done(ccb); 2037 if (detach != 0) 2038 VTSCSI_UNLOCK(sc); 2039 } 2040 2041 vtscsi_enqueue_request(sc, req); 2042 } 2043 2044 static void 2045 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2046 { 2047 struct vtscsi_request *req; 2048 int last; 2049 2050 last = 0; 2051 2052 vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq); 2053 2054 while ((req = virtqueue_drain(vq, &last)) != NULL) 2055 vtscsi_cancel_request(sc, req); 2056 2057 KASSERT(virtqueue_empty(vq), ("virtqueue not empty")); 2058 } 2059 2060 static void 2061 vtscsi_drain_vqs(struct vtscsi_softc *sc) 2062 { 2063 2064 if (sc->vtscsi_control_vq != NULL) 2065 vtscsi_drain_vq(sc, sc->vtscsi_control_vq); 2066 if (sc->vtscsi_request_vq != NULL) 2067 vtscsi_drain_vq(sc, sc->vtscsi_request_vq); 2068 if (sc->vtscsi_event_vq != NULL) 2069 vtscsi_drain_event_vq(sc); 2070 } 2071 2072 static void 2073 vtscsi_stop(struct vtscsi_softc *sc) 2074 { 2075 2076 vtscsi_disable_vqs_intr(sc); 2077 virtio_stop(sc->vtscsi_dev); 2078 } 2079 2080 static int 2081 vtscsi_reset_bus(struct vtscsi_softc *sc) 2082 { 2083 int error; 2084 2085 VTSCSI_LOCK_OWNED(sc); 2086 2087 if (vtscsi_bus_reset_disable != 0) { 2088 device_printf(sc->vtscsi_dev, "bus reset disabled\n"); 2089 return (0); 2090 } 2091 2092 sc->vtscsi_flags |= VTSCSI_FLAG_RESET; 2093 2094 /* 2095 * vtscsi_stop() will cause the in-flight requests to be canceled. 2096 * Those requests are then completed here so CAM will retry them 2097 * after the reset is complete. 2098 */ 2099 vtscsi_stop(sc); 2100 vtscsi_complete_vqs_locked(sc); 2101 2102 /* Rid the virtqueues of any remaining requests. */ 2103 vtscsi_drain_vqs(sc); 2104 2105 /* 2106 * Any resource shortage that froze the SIMQ cannot persist across 2107 * a bus reset so ensure it gets thawed here. 2108 */ 2109 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0) 2110 xpt_release_simq(sc->vtscsi_sim, 0); 2111 2112 error = vtscsi_reinit(sc); 2113 if (error) { 2114 device_printf(sc->vtscsi_dev, 2115 "reinitialization failed, stopping device...\n"); 2116 vtscsi_stop(sc); 2117 } else 2118 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD, 2119 CAM_LUN_WILDCARD); 2120 2121 sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET; 2122 2123 return (error); 2124 } 2125 2126 static void 2127 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2128 { 2129 2130 #ifdef INVARIANTS 2131 int req_nsegs, resp_nsegs; 2132 2133 req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2134 resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2135 2136 KASSERT(req_nsegs == 1, ("request crossed page boundary")); 2137 KASSERT(resp_nsegs == 1, ("response crossed page boundary")); 2138 #endif 2139 2140 req->vsr_softc = sc; 2141 callout_init_lk(&req->vsr_callout, VTSCSI_MTX(sc)); 2142 } 2143 2144 static int 2145 vtscsi_alloc_requests(struct vtscsi_softc *sc) 2146 { 2147 struct vtscsi_request *req; 2148 int i, nreqs; 2149 2150 /* 2151 * Commands destined for either the request or control queues come 2152 * from the same SIM queue. Use the size of the request virtqueue 2153 * as it (should) be much more frequently used. Some additional 2154 * requests are allocated for internal (TMF) use. 2155 */ 2156 nreqs = virtqueue_size(sc->vtscsi_request_vq); 2157 if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0) 2158 nreqs /= VTSCSI_MIN_SEGMENTS; 2159 nreqs += VTSCSI_RESERVED_REQUESTS; 2160 2161 for (i = 0; i < nreqs; i++) { 2162 req = contigmalloc(sizeof(struct vtscsi_request), M_DEVBUF, 2163 M_WAITOK, 0, BUS_SPACE_MAXADDR, 16, 0); 2164 if (req == NULL) 2165 return (ENOMEM); 2166 2167 vtscsi_init_request(sc, req); 2168 2169 sc->vtscsi_nrequests++; 2170 vtscsi_enqueue_request(sc, req); 2171 } 2172 2173 return (0); 2174 } 2175 2176 static void 2177 vtscsi_free_requests(struct vtscsi_softc *sc) 2178 { 2179 struct vtscsi_request *req; 2180 2181 while ((req = vtscsi_dequeue_request(sc)) != NULL) { 2182 KASSERT(callout_active(&req->vsr_callout) == 0, 2183 ("request callout still active")); 2184 2185 sc->vtscsi_nrequests--; 2186 contigfree(req, sizeof(struct vtscsi_request), M_DEVBUF); 2187 } 2188 2189 KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d", 2190 sc->vtscsi_nrequests)); 2191 } 2192 2193 static void 2194 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req) 2195 { 2196 2197 KASSERT(req->vsr_softc == sc, 2198 ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc)); 2199 2200 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2201 2202 /* A request is available so the SIMQ could be released. */ 2203 if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0) 2204 xpt_release_simq(sc->vtscsi_sim, 1); 2205 2206 req->vsr_ccb = NULL; 2207 req->vsr_complete = NULL; 2208 req->vsr_ptr0 = NULL; 2209 req->vsr_state = VTSCSI_REQ_STATE_FREE; 2210 req->vsr_flags = 0; 2211 2212 bzero(&req->vsr_ureq, sizeof(req->vsr_ureq)); 2213 bzero(&req->vsr_uresp, sizeof(req->vsr_uresp)); 2214 2215 /* 2216 * We insert at the tail of the queue in order to make it 2217 * very unlikely a request will be reused if we race with 2218 * stopping its callout handler. 2219 */ 2220 TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link); 2221 } 2222 2223 static struct vtscsi_request * 2224 vtscsi_dequeue_request(struct vtscsi_softc *sc) 2225 { 2226 struct vtscsi_request *req; 2227 2228 req = TAILQ_FIRST(&sc->vtscsi_req_free); 2229 if (req != NULL) { 2230 req->vsr_state = VTSCSI_REQ_STATE_INUSE; 2231 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link); 2232 } else 2233 sc->vtscsi_stats.dequeue_no_requests++; 2234 2235 vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req); 2236 2237 return (req); 2238 } 2239 2240 static void 2241 vtscsi_complete_request(struct vtscsi_request *req) 2242 { 2243 2244 if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED) 2245 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE; 2246 2247 if (req->vsr_complete != NULL) 2248 req->vsr_complete(req->vsr_softc, req); 2249 } 2250 2251 static void 2252 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq) 2253 { 2254 struct vtscsi_request *req; 2255 2256 VTSCSI_LOCK_OWNED(sc); 2257 2258 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) 2259 vtscsi_complete_request(req); 2260 } 2261 2262 static void 2263 vtscsi_control_vq_intr(void *xsc) 2264 { 2265 struct vtscsi_softc *sc; 2266 struct virtqueue *vq; 2267 2268 sc = xsc; 2269 vq = sc->vtscsi_control_vq; 2270 2271 again: 2272 VTSCSI_LOCK(sc); 2273 if (!virtqueue_pending(vq)) 2274 goto done; 2275 2276 vtscsi_complete_vq(sc, sc->vtscsi_control_vq); 2277 2278 if (virtqueue_enable_intr(vq) != 0) { 2279 virtqueue_disable_intr(vq); 2280 VTSCSI_UNLOCK(sc); 2281 goto again; 2282 } 2283 2284 done: 2285 VTSCSI_UNLOCK(sc); 2286 } 2287 2288 static void 2289 vtscsi_event_vq_intr(void *xsc) 2290 { 2291 struct vtscsi_softc *sc; 2292 struct virtqueue *vq; 2293 struct virtio_scsi_event *event; 2294 2295 sc = xsc; 2296 vq = sc->vtscsi_event_vq; 2297 2298 again: 2299 VTSCSI_LOCK(sc); 2300 if (!virtqueue_pending(vq)) 2301 goto done; 2302 2303 while ((event = virtqueue_dequeue(vq, NULL)) != NULL) 2304 vtscsi_handle_event(sc, event); 2305 2306 if (virtqueue_enable_intr(vq) != 0) { 2307 virtqueue_disable_intr(vq); 2308 VTSCSI_UNLOCK(sc); 2309 goto again; 2310 } 2311 2312 done: 2313 VTSCSI_UNLOCK(sc); 2314 } 2315 2316 static void 2317 vtscsi_request_vq_intr(void *xsc) 2318 { 2319 struct vtscsi_softc *sc; 2320 struct virtqueue *vq; 2321 2322 sc = xsc; 2323 vq = sc->vtscsi_request_vq; 2324 2325 again: 2326 VTSCSI_LOCK(sc); 2327 if (!virtqueue_pending(vq)) 2328 goto done; 2329 2330 vtscsi_complete_vq(sc, sc->vtscsi_request_vq); 2331 2332 if (virtqueue_enable_intr(vq) != 0) { 2333 virtqueue_disable_intr(vq); 2334 VTSCSI_UNLOCK(sc); 2335 goto again; 2336 } 2337 2338 done: 2339 VTSCSI_UNLOCK(sc); 2340 } 2341 2342 static void 2343 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc) 2344 { 2345 2346 virtqueue_disable_intr(sc->vtscsi_control_vq); 2347 virtqueue_disable_intr(sc->vtscsi_event_vq); 2348 virtqueue_disable_intr(sc->vtscsi_request_vq); 2349 } 2350 2351 static void 2352 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc) 2353 { 2354 2355 virtqueue_enable_intr(sc->vtscsi_control_vq); 2356 virtqueue_enable_intr(sc->vtscsi_event_vq); 2357 virtqueue_enable_intr(sc->vtscsi_request_vq); 2358 } 2359 2360 static void 2361 vtscsi_get_tunables(struct vtscsi_softc *sc) 2362 { 2363 char tmpstr[64]; 2364 2365 TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug); 2366 2367 ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level", 2368 device_get_unit(sc->vtscsi_dev)); 2369 TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug); 2370 } 2371 2372 static void 2373 vtscsi_add_sysctl(struct vtscsi_softc *sc) 2374 { 2375 device_t dev; 2376 struct vtscsi_statistics *stats; 2377 struct sysctl_ctx_list *ctx; 2378 struct sysctl_oid *tree; 2379 struct sysctl_oid_list *child; 2380 2381 dev = sc->vtscsi_dev; 2382 stats = &sc->vtscsi_stats; 2383 ctx = device_get_sysctl_ctx(dev); 2384 tree = device_get_sysctl_tree(dev); 2385 child = SYSCTL_CHILDREN(tree); 2386 2387 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 2388 CTLFLAG_RW, &sc->vtscsi_debug, 0, 2389 "Debug level"); 2390 2391 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts", 2392 CTLFLAG_RD, &stats->scsi_cmd_timeouts, 2393 "SCSI command timeouts"); 2394 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests", 2395 CTLFLAG_RD, &stats->dequeue_no_requests, 2396 "No available requests to dequeue"); 2397 } 2398 2399 static void 2400 vtscsi_printf_req(struct vtscsi_request *req, const char *func, 2401 const char *fmt, ...) 2402 { 2403 struct vtscsi_softc *sc; 2404 union ccb *ccb; 2405 struct sbuf sb; 2406 __va_list ap; 2407 char str[192]; 2408 char path_str[64]; 2409 2410 if (req == NULL) 2411 return; 2412 2413 sc = req->vsr_softc; 2414 ccb = req->vsr_ccb; 2415 2416 __va_start(ap, fmt); 2417 sbuf_new(&sb, str, sizeof(str), 0); 2418 2419 if (ccb == NULL) { 2420 sbuf_printf(&sb, "(noperiph:%s%d:%u): ", 2421 cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim), 2422 cam_sim_bus(sc->vtscsi_sim)); 2423 } else { 2424 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str)); 2425 sbuf_cat(&sb, path_str); 2426 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 2427 scsi_command_string(&ccb->csio, &sb); 2428 sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len); 2429 } 2430 } 2431 2432 sbuf_vprintf(&sb, fmt, ap); 2433 __va_end(ap); 2434 2435 sbuf_finish(&sb); 2436 kprintf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func, 2437 sbuf_data(&sb)); 2438 } 2439