1 /*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7 #include <sys/cdefs.h> 8 __FBSDID("$FreeBSD$"); 9 10 #include <sys/param.h> 11 #include <sys/bus.h> 12 #include <sys/errno.h> 13 #include <sys/kernel.h> 14 #include <sys/malloc.h> 15 #include <sys/module.h> 16 #include <sys/queue.h> 17 #include <sys/rman.h> 18 #include <sys/sysctl.h> 19 #include <sys/systm.h> 20 21 #include <machine/bus.h> 22 #include <machine/resource.h> 23 24 #include <dev/pci/pcireg.h> 25 #include <dev/pci/pcivar.h> 26 27 #include <cam/cam.h> 28 #include <cam/cam_ccb.h> 29 #include <cam/cam_debug.h> 30 #include <cam/cam_sim.h> 31 #include <cam/cam_xpt_sim.h> 32 #include <cam/scsi/scsi_message.h> 33 34 #include "pvscsi.h" 35 36 #define PVSCSI_DEFAULT_NUM_PAGES_REQ_RING 8 37 #define PVSCSI_SENSE_LENGTH 256 38 39 MALLOC_DECLARE(M_PVSCSI); 40 MALLOC_DEFINE(M_PVSCSI, "pvscsi", "PVSCSI memory"); 41 42 #ifdef PVSCSI_DEBUG_LOGGING 43 #define DEBUG_PRINTF(level, dev, fmt, ...) \ 44 do { \ 45 if (pvscsi_log_level >= (level)) { \ 46 device_printf((dev), (fmt), ##__VA_ARGS__); \ 47 } \ 48 } while(0) 49 #else 50 #define DEBUG_PRINTF(level, dev, fmt, ...) 51 #endif /* PVSCSI_DEBUG_LOGGING */ 52 53 #define ccb_pvscsi_hcb spriv_ptr0 54 #define ccb_pvscsi_sc spriv_ptr1 55 56 struct pvscsi_softc; 57 struct pvscsi_hcb; 58 struct pvscsi_dma; 59 60 static inline uint32_t pvscsi_reg_read(struct pvscsi_softc *sc, 61 uint32_t offset); 62 static inline void pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, 63 uint32_t val); 64 static inline uint32_t pvscsi_read_intr_status(struct pvscsi_softc *sc); 65 static inline void pvscsi_write_intr_status(struct pvscsi_softc *sc, 66 uint32_t val); 67 static inline void pvscsi_intr_enable(struct pvscsi_softc *sc); 68 static inline void pvscsi_intr_disable(struct pvscsi_softc *sc); 69 static void pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0); 70 static void pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, 71 uint32_t len); 72 static uint32_t pvscsi_get_max_targets(struct pvscsi_softc *sc); 73 static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable); 74 static void pvscsi_setup_rings(struct pvscsi_softc *sc); 75 static void pvscsi_setup_msg_ring(struct pvscsi_softc *sc); 76 static int pvscsi_hw_supports_msg(struct pvscsi_softc *sc); 77 78 static void pvscsi_timeout(void *arg); 79 static void pvscsi_freeze(struct pvscsi_softc *sc); 80 static void pvscsi_adapter_reset(struct pvscsi_softc *sc); 81 static void pvscsi_bus_reset(struct pvscsi_softc *sc); 82 static void pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target); 83 static void pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, 84 union ccb *ccb); 85 86 static void pvscsi_process_completion(struct pvscsi_softc *sc, 87 struct pvscsi_ring_cmp_desc *e); 88 static void pvscsi_process_cmp_ring(struct pvscsi_softc *sc); 89 static void pvscsi_process_msg(struct pvscsi_softc *sc, 90 struct pvscsi_ring_msg_desc *e); 91 static void pvscsi_process_msg_ring(struct pvscsi_softc *sc); 92 93 static void pvscsi_intr_locked(struct pvscsi_softc *sc); 94 static void pvscsi_intr(void *xsc); 95 static void pvscsi_poll(struct cam_sim *sim); 96 97 static void pvscsi_execute_ccb(void *arg, bus_dma_segment_t *segs, int nseg, 98 int error); 99 static void pvscsi_action(struct cam_sim *sim, union ccb *ccb); 100 101 static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, 102 struct pvscsi_hcb *hcb); 103 static inline struct pvscsi_hcb* pvscsi_context_to_hcb(struct pvscsi_softc *sc, 104 uint64_t context); 105 static struct pvscsi_hcb * pvscsi_hcb_get(struct pvscsi_softc *sc); 106 static void pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb); 107 108 static void pvscsi_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, 109 int error); 110 static void pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma); 111 static int pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 112 bus_size_t size, bus_size_t alignment); 113 static int pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, 114 struct pvscsi_dma *dma, uint64_t *ppn_list, uint32_t num_pages); 115 static void pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, 116 uint32_t hcbs_allocated); 117 static int pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc); 118 static void pvscsi_free_rings(struct pvscsi_softc *sc); 119 static int pvscsi_allocate_rings(struct pvscsi_softc *sc); 120 static void pvscsi_free_interrupts(struct pvscsi_softc *sc); 121 static int pvscsi_setup_interrupts(struct pvscsi_softc *sc); 122 static void pvscsi_free_all(struct pvscsi_softc *sc); 123 124 static int pvscsi_attach(device_t dev); 125 static int pvscsi_detach(device_t dev); 126 static int pvscsi_probe(device_t dev); 127 static int pvscsi_shutdown(device_t dev); 128 static int pvscsi_get_tunable(struct pvscsi_softc *sc, char *name, int value); 129 130 131 #ifdef PVSCSI_DEBUG_LOGGING 132 static int pvscsi_log_level = 0; 133 static SYSCTL_NODE(_hw, OID_AUTO, pvscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 134 "PVSCSI driver parameters"); 135 SYSCTL_INT(_hw_pvscsi, OID_AUTO, log_level, CTLFLAG_RWTUN, &pvscsi_log_level, 136 0, "PVSCSI debug log level"); 137 #endif 138 139 static int pvscsi_request_ring_pages = 0; 140 TUNABLE_INT("hw.pvscsi.request_ring_pages", &pvscsi_request_ring_pages); 141 142 static int pvscsi_use_msg = 1; 143 TUNABLE_INT("hw.pvscsi.use_msg", &pvscsi_use_msg); 144 145 static int pvscsi_use_msi = 1; 146 TUNABLE_INT("hw.pvscsi.use_msi", &pvscsi_use_msi); 147 148 static int pvscsi_use_msix = 1; 149 TUNABLE_INT("hw.pvscsi.use_msix", &pvscsi_use_msix); 150 151 static int pvscsi_use_req_call_threshold = 1; 152 TUNABLE_INT("hw.pvscsi.use_req_call_threshold", &pvscsi_use_req_call_threshold); 153 154 static int pvscsi_max_queue_depth = 0; 155 TUNABLE_INT("hw.pvscsi.max_queue_depth", &pvscsi_max_queue_depth); 156 157 158 struct pvscsi_sg_list { 159 struct pvscsi_sg_element sge[PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT]; 160 }; 161 162 163 #define PVSCSI_ABORT_TIMEOUT 2 164 #define PVSCSI_RESET_TIMEOUT 10 165 166 #define PVSCSI_HCB_NONE 0 167 #define PVSCSI_HCB_ABORT 1 168 #define PVSCSI_HCB_DEVICE_RESET 2 169 #define PVSCSI_HCB_BUS_RESET 3 170 171 struct pvscsi_hcb { 172 union ccb *ccb; 173 struct pvscsi_ring_req_desc *e; 174 int recovery; 175 SLIST_ENTRY(pvscsi_hcb) links; 176 177 struct callout callout; 178 bus_dmamap_t dma_map; 179 void *sense_buffer; 180 bus_addr_t sense_buffer_paddr; 181 struct pvscsi_sg_list *sg_list; 182 bus_addr_t sg_list_paddr; 183 }; 184 185 struct pvscsi_dma 186 { 187 bus_dma_tag_t tag; 188 bus_dmamap_t map; 189 void *vaddr; 190 bus_addr_t paddr; 191 bus_size_t size; 192 }; 193 194 struct pvscsi_softc { 195 device_t dev; 196 struct mtx lock; 197 struct cam_sim *sim; 198 struct cam_path *bus_path; 199 int frozen; 200 struct pvscsi_rings_state *rings_state; 201 struct pvscsi_ring_req_desc *req_ring; 202 struct pvscsi_ring_cmp_desc *cmp_ring; 203 struct pvscsi_ring_msg_desc *msg_ring; 204 uint32_t hcb_cnt; 205 struct pvscsi_hcb *hcbs; 206 SLIST_HEAD(, pvscsi_hcb) free_list; 207 bus_dma_tag_t parent_dmat; 208 bus_dma_tag_t buffer_dmat; 209 210 bool use_msg; 211 uint32_t max_targets; 212 int mm_rid; 213 struct resource *mm_res; 214 int irq_id; 215 struct resource *irq_res; 216 void *irq_handler; 217 int use_req_call_threshold; 218 int use_msi_or_msix; 219 220 uint64_t rings_state_ppn; 221 uint32_t req_ring_num_pages; 222 uint64_t req_ring_ppn[PVSCSI_MAX_NUM_PAGES_REQ_RING]; 223 uint32_t cmp_ring_num_pages; 224 uint64_t cmp_ring_ppn[PVSCSI_MAX_NUM_PAGES_CMP_RING]; 225 uint32_t msg_ring_num_pages; 226 uint64_t msg_ring_ppn[PVSCSI_MAX_NUM_PAGES_MSG_RING]; 227 228 struct pvscsi_dma rings_state_dma; 229 struct pvscsi_dma req_ring_dma; 230 struct pvscsi_dma cmp_ring_dma; 231 struct pvscsi_dma msg_ring_dma; 232 233 struct pvscsi_dma sg_list_dma; 234 struct pvscsi_dma sense_buffer_dma; 235 }; 236 237 static int pvscsi_get_tunable(struct pvscsi_softc *sc, char *name, int value) 238 { 239 char cfg[64]; 240 241 snprintf(cfg, sizeof(cfg), "hw.pvscsi.%d.%s", device_get_unit(sc->dev), 242 name); 243 TUNABLE_INT_FETCH(cfg, &value); 244 245 return (value); 246 } 247 248 static void 249 pvscsi_freeze(struct pvscsi_softc *sc) 250 { 251 252 if (!sc->frozen) { 253 xpt_freeze_simq(sc->sim, 1); 254 sc->frozen = 1; 255 } 256 } 257 258 static inline uint32_t 259 pvscsi_reg_read(struct pvscsi_softc *sc, uint32_t offset) 260 { 261 262 return (bus_read_4(sc->mm_res, offset)); 263 } 264 265 static inline void 266 pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, uint32_t val) 267 { 268 269 bus_write_4(sc->mm_res, offset, val); 270 } 271 272 static inline uint32_t 273 pvscsi_read_intr_status(struct pvscsi_softc *sc) 274 { 275 276 return (pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_INTR_STATUS)); 277 } 278 279 static inline void 280 pvscsi_write_intr_status(struct pvscsi_softc *sc, uint32_t val) 281 { 282 283 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_STATUS, val); 284 } 285 286 static inline void 287 pvscsi_intr_enable(struct pvscsi_softc *sc) 288 { 289 uint32_t mask; 290 291 mask = PVSCSI_INTR_CMPL_MASK; 292 if (sc->use_msg) { 293 mask |= PVSCSI_INTR_MSG_MASK; 294 } 295 296 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, mask); 297 } 298 299 static inline void 300 pvscsi_intr_disable(struct pvscsi_softc *sc) 301 { 302 303 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, 0); 304 } 305 306 static void 307 pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0) 308 { 309 struct pvscsi_rings_state *s; 310 311 if (cdb0 == READ_6 || cdb0 == READ_10 || 312 cdb0 == READ_12 || cdb0 == READ_16 || 313 cdb0 == WRITE_6 || cdb0 == WRITE_10 || 314 cdb0 == WRITE_12 || cdb0 == WRITE_16) { 315 s = sc->rings_state; 316 317 if (!sc->use_req_call_threshold || 318 (s->req_prod_idx - s->req_cons_idx) >= 319 s->req_call_threshold) { 320 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_RW_IO, 0); 321 } 322 } else { 323 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0); 324 } 325 } 326 327 static void 328 pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, 329 uint32_t len) 330 { 331 uint32_t *data_ptr; 332 int i; 333 334 KASSERT(len % sizeof(uint32_t) == 0, 335 ("command size not a multiple of 4")); 336 337 data_ptr = data; 338 len /= sizeof(uint32_t); 339 340 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, cmd); 341 for (i = 0; i < len; ++i) { 342 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND_DATA, 343 data_ptr[i]); 344 } 345 } 346 347 static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, 348 struct pvscsi_hcb *hcb) 349 { 350 351 /* Offset by 1 because context must not be 0 */ 352 return (hcb - sc->hcbs + 1); 353 } 354 355 static inline struct pvscsi_hcb* pvscsi_context_to_hcb(struct pvscsi_softc *sc, 356 uint64_t context) 357 { 358 359 return (sc->hcbs + (context - 1)); 360 } 361 362 static struct pvscsi_hcb * 363 pvscsi_hcb_get(struct pvscsi_softc *sc) 364 { 365 struct pvscsi_hcb *hcb; 366 367 mtx_assert(&sc->lock, MA_OWNED); 368 369 hcb = SLIST_FIRST(&sc->free_list); 370 if (hcb) { 371 SLIST_REMOVE_HEAD(&sc->free_list, links); 372 } 373 374 return (hcb); 375 } 376 377 static void 378 pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb) 379 { 380 381 mtx_assert(&sc->lock, MA_OWNED); 382 hcb->ccb = NULL; 383 hcb->e = NULL; 384 hcb->recovery = PVSCSI_HCB_NONE; 385 SLIST_INSERT_HEAD(&sc->free_list, hcb, links); 386 } 387 388 static uint32_t 389 pvscsi_get_max_targets(struct pvscsi_softc *sc) 390 { 391 uint32_t max_targets; 392 393 pvscsi_write_cmd(sc, PVSCSI_CMD_GET_MAX_TARGETS, NULL, 0); 394 395 max_targets = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 396 397 if (max_targets == ~0) { 398 max_targets = 16; 399 } 400 401 return (max_targets); 402 } 403 404 static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable) 405 { 406 uint32_t status; 407 struct pvscsi_cmd_desc_setup_req_call cmd; 408 409 if (!pvscsi_get_tunable(sc, "pvscsi_use_req_call_threshold", 410 pvscsi_use_req_call_threshold)) { 411 return (0); 412 } 413 414 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, 415 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD); 416 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 417 418 if (status != -1) { 419 bzero(&cmd, sizeof(cmd)); 420 cmd.enable = enable; 421 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_REQCALLTHRESHOLD, 422 &cmd, sizeof(cmd)); 423 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 424 425 return (status != 0); 426 } else { 427 return (0); 428 } 429 } 430 431 static void 432 pvscsi_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 433 { 434 bus_addr_t *dest; 435 436 KASSERT(nseg == 1, ("more than one segment")); 437 438 dest = arg; 439 440 if (!error) { 441 *dest = segs->ds_addr; 442 } 443 } 444 445 static void 446 pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma) 447 { 448 449 if (dma->tag != NULL) { 450 if (dma->paddr != 0) { 451 bus_dmamap_unload(dma->tag, dma->map); 452 } 453 454 if (dma->vaddr != NULL) { 455 bus_dmamem_free(dma->tag, dma->vaddr, dma->map); 456 } 457 458 bus_dma_tag_destroy(dma->tag); 459 } 460 461 bzero(dma, sizeof(*dma)); 462 } 463 464 static int 465 pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 466 bus_size_t size, bus_size_t alignment) 467 { 468 int error; 469 470 bzero(dma, sizeof(*dma)); 471 472 error = bus_dma_tag_create(sc->parent_dmat, alignment, 0, 473 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, size, 1, size, 474 BUS_DMA_ALLOCNOW, NULL, NULL, &dma->tag); 475 if (error) { 476 device_printf(sc->dev, "error creating dma tag, error %d\n", 477 error); 478 goto fail; 479 } 480 481 error = bus_dmamem_alloc(dma->tag, &dma->vaddr, 482 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &dma->map); 483 if (error) { 484 device_printf(sc->dev, "error allocating dma mem, error %d\n", 485 error); 486 goto fail; 487 } 488 489 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size, 490 pvscsi_dma_cb, &dma->paddr, BUS_DMA_NOWAIT); 491 if (error) { 492 device_printf(sc->dev, "error mapping dma mam, error %d\n", 493 error); 494 goto fail; 495 } 496 497 dma->size = size; 498 499 fail: 500 if (error) { 501 pvscsi_dma_free(sc, dma); 502 } 503 return (error); 504 } 505 506 static int 507 pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 508 uint64_t *ppn_list, uint32_t num_pages) 509 { 510 int error; 511 uint32_t i; 512 uint64_t ppn; 513 514 error = pvscsi_dma_alloc(sc, dma, num_pages * PAGE_SIZE, PAGE_SIZE); 515 if (error) { 516 device_printf(sc->dev, "Error allocating pages, error %d\n", 517 error); 518 return (error); 519 } 520 521 ppn = dma->paddr >> PAGE_SHIFT; 522 for (i = 0; i < num_pages; i++) { 523 ppn_list[i] = ppn + i; 524 } 525 526 return (0); 527 } 528 529 static void 530 pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, uint32_t hcbs_allocated) 531 { 532 int i; 533 int lock_owned; 534 struct pvscsi_hcb *hcb; 535 536 lock_owned = mtx_owned(&sc->lock); 537 538 if (lock_owned) { 539 mtx_unlock(&sc->lock); 540 } 541 for (i = 0; i < hcbs_allocated; ++i) { 542 hcb = sc->hcbs + i; 543 callout_drain(&hcb->callout); 544 }; 545 if (lock_owned) { 546 mtx_lock(&sc->lock); 547 } 548 549 for (i = 0; i < hcbs_allocated; ++i) { 550 hcb = sc->hcbs + i; 551 bus_dmamap_destroy(sc->buffer_dmat, hcb->dma_map); 552 }; 553 554 pvscsi_dma_free(sc, &sc->sense_buffer_dma); 555 pvscsi_dma_free(sc, &sc->sg_list_dma); 556 } 557 558 static int 559 pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc) 560 { 561 int i; 562 int error; 563 struct pvscsi_hcb *hcb; 564 565 i = 0; 566 567 error = pvscsi_dma_alloc(sc, &sc->sg_list_dma, 568 sizeof(struct pvscsi_sg_list) * sc->hcb_cnt, 1); 569 if (error) { 570 device_printf(sc->dev, 571 "Error allocation sg list DMA memory, error %d\n", error); 572 goto fail; 573 } 574 575 error = pvscsi_dma_alloc(sc, &sc->sense_buffer_dma, 576 PVSCSI_SENSE_LENGTH * sc->hcb_cnt, 1); 577 if (error) { 578 device_printf(sc->dev, 579 "Error allocation sg list DMA memory, error %d\n", error); 580 goto fail; 581 } 582 583 for (i = 0; i < sc->hcb_cnt; ++i) { 584 hcb = sc->hcbs + i; 585 586 error = bus_dmamap_create(sc->buffer_dmat, 0, &hcb->dma_map); 587 if (error) { 588 device_printf(sc->dev, 589 "Error creating dma map for hcb %d, error %d\n", 590 i, error); 591 goto fail; 592 } 593 594 hcb->sense_buffer = 595 (void *)((caddr_t)sc->sense_buffer_dma.vaddr + 596 PVSCSI_SENSE_LENGTH * i); 597 hcb->sense_buffer_paddr = 598 sc->sense_buffer_dma.paddr + PVSCSI_SENSE_LENGTH * i; 599 600 hcb->sg_list = 601 (struct pvscsi_sg_list *)((caddr_t)sc->sg_list_dma.vaddr + 602 sizeof(struct pvscsi_sg_list) * i); 603 hcb->sg_list_paddr = 604 sc->sg_list_dma.paddr + sizeof(struct pvscsi_sg_list) * i; 605 606 callout_init_mtx(&hcb->callout, &sc->lock, 0); 607 } 608 609 SLIST_INIT(&sc->free_list); 610 for (i = (sc->hcb_cnt - 1); i >= 0; --i) { 611 hcb = sc->hcbs + i; 612 SLIST_INSERT_HEAD(&sc->free_list, hcb, links); 613 } 614 615 fail: 616 if (error) { 617 pvscsi_dma_free_per_hcb(sc, i); 618 } 619 620 return (error); 621 } 622 623 static void 624 pvscsi_free_rings(struct pvscsi_softc *sc) 625 { 626 627 pvscsi_dma_free(sc, &sc->rings_state_dma); 628 pvscsi_dma_free(sc, &sc->req_ring_dma); 629 pvscsi_dma_free(sc, &sc->cmp_ring_dma); 630 if (sc->use_msg) { 631 pvscsi_dma_free(sc, &sc->msg_ring_dma); 632 } 633 } 634 635 static int 636 pvscsi_allocate_rings(struct pvscsi_softc *sc) 637 { 638 int error; 639 640 error = pvscsi_dma_alloc_ppns(sc, &sc->rings_state_dma, 641 &sc->rings_state_ppn, 1); 642 if (error) { 643 device_printf(sc->dev, 644 "Error allocating rings state, error = %d\n", error); 645 goto fail; 646 } 647 sc->rings_state = sc->rings_state_dma.vaddr; 648 649 error = pvscsi_dma_alloc_ppns(sc, &sc->req_ring_dma, sc->req_ring_ppn, 650 sc->req_ring_num_pages); 651 if (error) { 652 device_printf(sc->dev, 653 "Error allocating req ring pages, error = %d\n", error); 654 goto fail; 655 } 656 sc->req_ring = sc->req_ring_dma.vaddr; 657 658 error = pvscsi_dma_alloc_ppns(sc, &sc->cmp_ring_dma, sc->cmp_ring_ppn, 659 sc->cmp_ring_num_pages); 660 if (error) { 661 device_printf(sc->dev, 662 "Error allocating cmp ring pages, error = %d\n", error); 663 goto fail; 664 } 665 sc->cmp_ring = sc->cmp_ring_dma.vaddr; 666 667 sc->msg_ring = NULL; 668 if (sc->use_msg) { 669 error = pvscsi_dma_alloc_ppns(sc, &sc->msg_ring_dma, 670 sc->msg_ring_ppn, sc->msg_ring_num_pages); 671 if (error) { 672 device_printf(sc->dev, 673 "Error allocating cmp ring pages, error = %d\n", 674 error); 675 goto fail; 676 } 677 sc->msg_ring = sc->msg_ring_dma.vaddr; 678 } 679 680 DEBUG_PRINTF(1, sc->dev, "rings_state: %p\n", sc->rings_state); 681 DEBUG_PRINTF(1, sc->dev, "req_ring: %p - %u pages\n", sc->req_ring, 682 sc->req_ring_num_pages); 683 DEBUG_PRINTF(1, sc->dev, "cmp_ring: %p - %u pages\n", sc->cmp_ring, 684 sc->cmp_ring_num_pages); 685 DEBUG_PRINTF(1, sc->dev, "msg_ring: %p - %u pages\n", sc->msg_ring, 686 sc->msg_ring_num_pages); 687 688 fail: 689 if (error) { 690 pvscsi_free_rings(sc); 691 } 692 return (error); 693 } 694 695 static void 696 pvscsi_setup_rings(struct pvscsi_softc *sc) 697 { 698 struct pvscsi_cmd_desc_setup_rings cmd; 699 uint32_t i; 700 701 bzero(&cmd, sizeof(cmd)); 702 703 cmd.rings_state_ppn = sc->rings_state_ppn; 704 705 cmd.req_ring_num_pages = sc->req_ring_num_pages; 706 for (i = 0; i < sc->req_ring_num_pages; ++i) { 707 cmd.req_ring_ppns[i] = sc->req_ring_ppn[i]; 708 } 709 710 cmd.cmp_ring_num_pages = sc->cmp_ring_num_pages; 711 for (i = 0; i < sc->cmp_ring_num_pages; ++i) { 712 cmd.cmp_ring_ppns[i] = sc->cmp_ring_ppn[i]; 713 } 714 715 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof(cmd)); 716 } 717 718 static int 719 pvscsi_hw_supports_msg(struct pvscsi_softc *sc) 720 { 721 uint32_t status; 722 723 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, 724 PVSCSI_CMD_SETUP_MSG_RING); 725 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 726 727 return (status != -1); 728 } 729 730 static void 731 pvscsi_setup_msg_ring(struct pvscsi_softc *sc) 732 { 733 struct pvscsi_cmd_desc_setup_msg_ring cmd; 734 uint32_t i; 735 736 KASSERT(sc->use_msg, ("msg is not being used")); 737 738 bzero(&cmd, sizeof(cmd)); 739 740 cmd.num_pages = sc->msg_ring_num_pages; 741 for (i = 0; i < sc->msg_ring_num_pages; ++i) { 742 cmd.ring_ppns[i] = sc->msg_ring_ppn[i]; 743 } 744 745 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_MSG_RING, &cmd, sizeof(cmd)); 746 } 747 748 static void 749 pvscsi_adapter_reset(struct pvscsi_softc *sc) 750 { 751 uint32_t val; 752 753 device_printf(sc->dev, "Adapter Reset\n"); 754 755 pvscsi_write_cmd(sc, PVSCSI_CMD_ADAPTER_RESET, NULL, 0); 756 val = pvscsi_read_intr_status(sc); 757 758 DEBUG_PRINTF(2, sc->dev, "adapter reset done: %u\n", val); 759 } 760 761 static void 762 pvscsi_bus_reset(struct pvscsi_softc *sc) 763 { 764 765 device_printf(sc->dev, "Bus Reset\n"); 766 767 pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_BUS, NULL, 0); 768 pvscsi_process_cmp_ring(sc); 769 770 DEBUG_PRINTF(2, sc->dev, "bus reset done\n"); 771 } 772 773 static void 774 pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target) 775 { 776 struct pvscsi_cmd_desc_reset_device cmd; 777 778 memset(&cmd, 0, sizeof(cmd)); 779 780 cmd.target = target; 781 782 device_printf(sc->dev, "Device reset for target %u\n", target); 783 784 pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof cmd); 785 pvscsi_process_cmp_ring(sc); 786 787 DEBUG_PRINTF(2, sc->dev, "device reset done\n"); 788 } 789 790 static void 791 pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, union ccb *ccb) 792 { 793 struct pvscsi_cmd_desc_abort_cmd cmd; 794 struct pvscsi_hcb *hcb; 795 uint64_t context; 796 797 pvscsi_process_cmp_ring(sc); 798 799 hcb = ccb->ccb_h.ccb_pvscsi_hcb; 800 801 if (hcb != NULL) { 802 context = pvscsi_hcb_to_context(sc, hcb); 803 804 memset(&cmd, 0, sizeof cmd); 805 cmd.target = target; 806 cmd.context = context; 807 808 device_printf(sc->dev, "Abort for target %u context %llx\n", 809 target, (unsigned long long)context); 810 811 pvscsi_write_cmd(sc, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd)); 812 pvscsi_process_cmp_ring(sc); 813 814 DEBUG_PRINTF(2, sc->dev, "abort done\n"); 815 } else { 816 DEBUG_PRINTF(1, sc->dev, 817 "Target %u ccb %p not found for abort\n", target, ccb); 818 } 819 } 820 821 static int 822 pvscsi_probe(device_t dev) 823 { 824 825 if (pci_get_vendor(dev) == PCI_VENDOR_ID_VMWARE && 826 pci_get_device(dev) == PCI_DEVICE_ID_VMWARE_PVSCSI) { 827 device_set_desc(dev, "VMware Paravirtual SCSI Controller"); 828 return (BUS_PROBE_DEFAULT); 829 } 830 return (ENXIO); 831 } 832 833 static int 834 pvscsi_shutdown(device_t dev) 835 { 836 837 return (0); 838 } 839 840 static void 841 pvscsi_timeout(void *arg) 842 { 843 struct pvscsi_hcb *hcb; 844 struct pvscsi_softc *sc; 845 union ccb *ccb; 846 847 hcb = arg; 848 ccb = hcb->ccb; 849 850 if (ccb == NULL) { 851 /* Already completed */ 852 return; 853 } 854 855 sc = ccb->ccb_h.ccb_pvscsi_sc; 856 mtx_assert(&sc->lock, MA_OWNED); 857 858 device_printf(sc->dev, "Command timed out hcb=%p ccb=%p.\n", hcb, ccb); 859 860 switch (hcb->recovery) { 861 case PVSCSI_HCB_NONE: 862 hcb->recovery = PVSCSI_HCB_ABORT; 863 pvscsi_abort(sc, ccb->ccb_h.target_id, ccb); 864 callout_reset_sbt(&hcb->callout, PVSCSI_ABORT_TIMEOUT * SBT_1S, 865 0, pvscsi_timeout, hcb, 0); 866 break; 867 case PVSCSI_HCB_ABORT: 868 hcb->recovery = PVSCSI_HCB_DEVICE_RESET; 869 pvscsi_freeze(sc); 870 pvscsi_device_reset(sc, ccb->ccb_h.target_id); 871 callout_reset_sbt(&hcb->callout, PVSCSI_RESET_TIMEOUT * SBT_1S, 872 0, pvscsi_timeout, hcb, 0); 873 break; 874 case PVSCSI_HCB_DEVICE_RESET: 875 hcb->recovery = PVSCSI_HCB_BUS_RESET; 876 pvscsi_freeze(sc); 877 pvscsi_bus_reset(sc); 878 callout_reset_sbt(&hcb->callout, PVSCSI_RESET_TIMEOUT * SBT_1S, 879 0, pvscsi_timeout, hcb, 0); 880 break; 881 case PVSCSI_HCB_BUS_RESET: 882 pvscsi_freeze(sc); 883 pvscsi_adapter_reset(sc); 884 break; 885 }; 886 } 887 888 static void 889 pvscsi_process_completion(struct pvscsi_softc *sc, 890 struct pvscsi_ring_cmp_desc *e) 891 { 892 struct pvscsi_hcb *hcb; 893 union ccb *ccb; 894 uint32_t status; 895 uint32_t btstat; 896 uint32_t sdstat; 897 bus_dmasync_op_t op; 898 899 hcb = pvscsi_context_to_hcb(sc, e->context); 900 901 callout_stop(&hcb->callout); 902 903 ccb = hcb->ccb; 904 905 btstat = e->host_status; 906 sdstat = e->scsi_status; 907 908 ccb->csio.scsi_status = sdstat; 909 ccb->csio.resid = ccb->csio.dxfer_len - e->data_len; 910 911 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 912 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { 913 op = BUS_DMASYNC_POSTREAD; 914 } else { 915 op = BUS_DMASYNC_POSTWRITE; 916 } 917 bus_dmamap_sync(sc->buffer_dmat, hcb->dma_map, op); 918 bus_dmamap_unload(sc->buffer_dmat, hcb->dma_map); 919 } 920 921 if (btstat == BTSTAT_SUCCESS && sdstat == SCSI_STATUS_OK) { 922 DEBUG_PRINTF(3, sc->dev, 923 "completing command context %llx success\n", 924 (unsigned long long)e->context); 925 ccb->csio.resid = 0; 926 status = CAM_REQ_CMP; 927 } else { 928 switch (btstat) { 929 case BTSTAT_SUCCESS: 930 case BTSTAT_LINKED_COMMAND_COMPLETED: 931 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG: 932 switch (sdstat) { 933 case SCSI_STATUS_OK: 934 ccb->csio.resid = 0; 935 status = CAM_REQ_CMP; 936 break; 937 case SCSI_STATUS_CHECK_COND: 938 status = CAM_SCSI_STATUS_ERROR; 939 940 if (ccb->csio.sense_len != 0) { 941 status |= CAM_AUTOSNS_VALID; 942 943 memset(&ccb->csio.sense_data, 0, 944 sizeof(ccb->csio.sense_data)); 945 memcpy(&ccb->csio.sense_data, 946 hcb->sense_buffer, 947 MIN(ccb->csio.sense_len, 948 e->sense_len)); 949 } 950 break; 951 case SCSI_STATUS_BUSY: 952 case SCSI_STATUS_QUEUE_FULL: 953 status = CAM_REQUEUE_REQ; 954 break; 955 case SCSI_STATUS_CMD_TERMINATED: 956 case SCSI_STATUS_TASK_ABORTED: 957 status = CAM_REQ_ABORTED; 958 break; 959 default: 960 DEBUG_PRINTF(1, sc->dev, 961 "ccb: %p sdstat=0x%x\n", ccb, sdstat); 962 status = CAM_SCSI_STATUS_ERROR; 963 break; 964 } 965 break; 966 case BTSTAT_SELTIMEO: 967 status = CAM_SEL_TIMEOUT; 968 break; 969 case BTSTAT_DATARUN: 970 case BTSTAT_DATA_UNDERRUN: 971 status = CAM_DATA_RUN_ERR; 972 break; 973 case BTSTAT_ABORTQUEUE: 974 case BTSTAT_HATIMEOUT: 975 status = CAM_REQUEUE_REQ; 976 break; 977 case BTSTAT_NORESPONSE: 978 case BTSTAT_SENTRST: 979 case BTSTAT_RECVRST: 980 case BTSTAT_BUSRESET: 981 status = CAM_SCSI_BUS_RESET; 982 break; 983 case BTSTAT_SCSIPARITY: 984 status = CAM_UNCOR_PARITY; 985 break; 986 case BTSTAT_BUSFREE: 987 status = CAM_UNEXP_BUSFREE; 988 break; 989 case BTSTAT_INVPHASE: 990 status = CAM_SEQUENCE_FAIL; 991 break; 992 case BTSTAT_SENSFAILED: 993 status = CAM_AUTOSENSE_FAIL; 994 break; 995 case BTSTAT_LUNMISMATCH: 996 case BTSTAT_TAGREJECT: 997 case BTSTAT_DISCONNECT: 998 case BTSTAT_BADMSG: 999 case BTSTAT_INVPARAM: 1000 status = CAM_REQ_CMP_ERR; 1001 break; 1002 case BTSTAT_HASOFTWARE: 1003 case BTSTAT_HAHARDWARE: 1004 status = CAM_NO_HBA; 1005 break; 1006 default: 1007 device_printf(sc->dev, "unknown hba status: 0x%x\n", 1008 btstat); 1009 status = CAM_NO_HBA; 1010 break; 1011 } 1012 1013 DEBUG_PRINTF(3, sc->dev, 1014 "completing command context %llx btstat %x sdstat %x - status %x\n", 1015 (unsigned long long)e->context, btstat, sdstat, status); 1016 } 1017 1018 ccb->ccb_h.ccb_pvscsi_hcb = NULL; 1019 ccb->ccb_h.ccb_pvscsi_sc = NULL; 1020 pvscsi_hcb_put(sc, hcb); 1021 1022 ccb->ccb_h.status = 1023 status | (ccb->ccb_h.status & ~(CAM_STATUS_MASK | CAM_SIM_QUEUED)); 1024 1025 if (sc->frozen) { 1026 ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1027 sc->frozen = 0; 1028 } 1029 1030 if (status != CAM_REQ_CMP) { 1031 ccb->ccb_h.status |= CAM_DEV_QFRZN; 1032 xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1); 1033 } 1034 xpt_done(ccb); 1035 } 1036 1037 static void 1038 pvscsi_process_cmp_ring(struct pvscsi_softc *sc) 1039 { 1040 struct pvscsi_ring_cmp_desc *ring; 1041 struct pvscsi_rings_state *s; 1042 struct pvscsi_ring_cmp_desc *e; 1043 uint32_t mask; 1044 1045 mtx_assert(&sc->lock, MA_OWNED); 1046 1047 s = sc->rings_state; 1048 ring = sc->cmp_ring; 1049 mask = MASK(s->cmp_num_entries_log2); 1050 1051 while (s->cmp_cons_idx != s->cmp_prod_idx) { 1052 e = ring + (s->cmp_cons_idx & mask); 1053 1054 pvscsi_process_completion(sc, e); 1055 1056 mb(); 1057 s->cmp_cons_idx++; 1058 } 1059 } 1060 1061 static void 1062 pvscsi_process_msg(struct pvscsi_softc *sc, struct pvscsi_ring_msg_desc *e) 1063 { 1064 struct pvscsi_ring_msg_dev_status_changed *desc; 1065 1066 union ccb *ccb; 1067 switch (e->type) { 1068 case PVSCSI_MSG_DEV_ADDED: 1069 case PVSCSI_MSG_DEV_REMOVED: { 1070 desc = (struct pvscsi_ring_msg_dev_status_changed *)e; 1071 1072 device_printf(sc->dev, "MSG: device %s at scsi%u:%u:%u\n", 1073 desc->type == PVSCSI_MSG_DEV_ADDED ? "addition" : "removal", 1074 desc->bus, desc->target, desc->lun[1]); 1075 1076 ccb = xpt_alloc_ccb_nowait(); 1077 if (ccb == NULL) { 1078 device_printf(sc->dev, 1079 "Error allocating CCB for dev change.\n"); 1080 break; 1081 } 1082 1083 if (xpt_create_path(&ccb->ccb_h.path, NULL, 1084 cam_sim_path(sc->sim), desc->target, desc->lun[1]) 1085 != CAM_REQ_CMP) { 1086 device_printf(sc->dev, 1087 "Error creating path for dev change.\n"); 1088 xpt_free_ccb(ccb); 1089 break; 1090 } 1091 1092 xpt_rescan(ccb); 1093 } break; 1094 default: 1095 device_printf(sc->dev, "Unknown msg type 0x%x\n", e->type); 1096 }; 1097 } 1098 1099 static void 1100 pvscsi_process_msg_ring(struct pvscsi_softc *sc) 1101 { 1102 struct pvscsi_ring_msg_desc *ring; 1103 struct pvscsi_rings_state *s; 1104 struct pvscsi_ring_msg_desc *e; 1105 uint32_t mask; 1106 1107 mtx_assert(&sc->lock, MA_OWNED); 1108 1109 s = sc->rings_state; 1110 ring = sc->msg_ring; 1111 mask = MASK(s->msg_num_entries_log2); 1112 1113 while (s->msg_cons_idx != s->msg_prod_idx) { 1114 e = ring + (s->msg_cons_idx & mask); 1115 1116 pvscsi_process_msg(sc, e); 1117 1118 mb(); 1119 s->msg_cons_idx++; 1120 } 1121 } 1122 1123 static void 1124 pvscsi_intr_locked(struct pvscsi_softc *sc) 1125 { 1126 uint32_t val; 1127 1128 mtx_assert(&sc->lock, MA_OWNED); 1129 1130 val = pvscsi_read_intr_status(sc); 1131 1132 if ((val & PVSCSI_INTR_ALL_SUPPORTED) != 0) { 1133 pvscsi_write_intr_status(sc, val & PVSCSI_INTR_ALL_SUPPORTED); 1134 pvscsi_process_cmp_ring(sc); 1135 if (sc->use_msg) { 1136 pvscsi_process_msg_ring(sc); 1137 } 1138 } 1139 } 1140 1141 static void 1142 pvscsi_intr(void *xsc) 1143 { 1144 struct pvscsi_softc *sc; 1145 1146 sc = xsc; 1147 1148 mtx_assert(&sc->lock, MA_NOTOWNED); 1149 1150 mtx_lock(&sc->lock); 1151 pvscsi_intr_locked(xsc); 1152 mtx_unlock(&sc->lock); 1153 } 1154 1155 static void 1156 pvscsi_poll(struct cam_sim *sim) 1157 { 1158 struct pvscsi_softc *sc; 1159 1160 sc = cam_sim_softc(sim); 1161 1162 mtx_assert(&sc->lock, MA_OWNED); 1163 pvscsi_intr_locked(sc); 1164 } 1165 1166 static void 1167 pvscsi_execute_ccb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1168 { 1169 struct pvscsi_hcb *hcb; 1170 struct pvscsi_ring_req_desc *e; 1171 union ccb *ccb; 1172 struct pvscsi_softc *sc; 1173 struct pvscsi_rings_state *s; 1174 uint8_t cdb0; 1175 bus_dmasync_op_t op; 1176 1177 hcb = arg; 1178 ccb = hcb->ccb; 1179 e = hcb->e; 1180 sc = ccb->ccb_h.ccb_pvscsi_sc; 1181 s = sc->rings_state; 1182 1183 mtx_assert(&sc->lock, MA_OWNED); 1184 1185 if (error) { 1186 device_printf(sc->dev, "pvscsi_execute_ccb error %d\n", error); 1187 1188 if (error == EFBIG) { 1189 ccb->ccb_h.status = CAM_REQ_TOO_BIG; 1190 } else { 1191 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 1192 } 1193 1194 pvscsi_hcb_put(sc, hcb); 1195 xpt_done(ccb); 1196 return; 1197 } 1198 1199 e->flags = 0; 1200 op = 0; 1201 switch (ccb->ccb_h.flags & CAM_DIR_MASK) { 1202 case CAM_DIR_NONE: 1203 e->flags |= PVSCSI_FLAG_CMD_DIR_NONE; 1204 break; 1205 case CAM_DIR_IN: 1206 e->flags |= PVSCSI_FLAG_CMD_DIR_TOHOST; 1207 op = BUS_DMASYNC_PREREAD; 1208 break; 1209 case CAM_DIR_OUT: 1210 e->flags |= PVSCSI_FLAG_CMD_DIR_TODEVICE; 1211 op = BUS_DMASYNC_PREWRITE; 1212 break; 1213 case CAM_DIR_BOTH: 1214 /* TODO: does this need handling? */ 1215 break; 1216 } 1217 1218 if (nseg != 0) { 1219 if (nseg > 1) { 1220 int i; 1221 struct pvscsi_sg_element *sge; 1222 1223 KASSERT(nseg <= PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT, 1224 ("too many sg segments")); 1225 1226 sge = hcb->sg_list->sge; 1227 e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST; 1228 1229 for (i = 0; i < nseg; ++i) { 1230 sge[i].addr = segs[i].ds_addr; 1231 sge[i].length = segs[i].ds_len; 1232 sge[i].flags = 0; 1233 } 1234 1235 e->data_addr = hcb->sg_list_paddr; 1236 } else { 1237 e->data_addr = segs->ds_addr; 1238 } 1239 1240 bus_dmamap_sync(sc->buffer_dmat, hcb->dma_map, op); 1241 } else { 1242 e->data_addr = 0; 1243 } 1244 1245 cdb0 = e->cdb[0]; 1246 ccb->ccb_h.status |= CAM_SIM_QUEUED; 1247 1248 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) { 1249 callout_reset_sbt(&hcb->callout, ccb->ccb_h.timeout * SBT_1MS, 1250 0, pvscsi_timeout, hcb, 0); 1251 } 1252 1253 mb(); 1254 s->req_prod_idx++; 1255 pvscsi_kick_io(sc, cdb0); 1256 } 1257 1258 static void 1259 pvscsi_action(struct cam_sim *sim, union ccb *ccb) 1260 { 1261 struct pvscsi_softc *sc; 1262 struct ccb_hdr *ccb_h; 1263 1264 sc = cam_sim_softc(sim); 1265 ccb_h = &ccb->ccb_h; 1266 1267 mtx_assert(&sc->lock, MA_OWNED); 1268 1269 switch (ccb_h->func_code) { 1270 case XPT_SCSI_IO: 1271 { 1272 struct ccb_scsiio *csio; 1273 uint32_t req_num_entries_log2; 1274 struct pvscsi_ring_req_desc *ring; 1275 struct pvscsi_ring_req_desc *e; 1276 struct pvscsi_rings_state *s; 1277 struct pvscsi_hcb *hcb; 1278 1279 csio = &ccb->csio; 1280 ring = sc->req_ring; 1281 s = sc->rings_state; 1282 1283 hcb = NULL; 1284 1285 /* 1286 * Check if it was completed already (such as aborted 1287 * by upper layers) 1288 */ 1289 if ((ccb_h->status & CAM_STATUS_MASK) != CAM_REQ_INPROG) { 1290 xpt_done(ccb); 1291 return; 1292 } 1293 1294 req_num_entries_log2 = s->req_num_entries_log2; 1295 1296 if (s->req_prod_idx - s->cmp_cons_idx >= 1297 (1 << req_num_entries_log2)) { 1298 device_printf(sc->dev, 1299 "Not enough room on completion ring.\n"); 1300 pvscsi_freeze(sc); 1301 ccb_h->status = CAM_REQUEUE_REQ; 1302 goto finish_ccb; 1303 } 1304 1305 hcb = pvscsi_hcb_get(sc); 1306 if (hcb == NULL) { 1307 device_printf(sc->dev, "No free hcbs.\n"); 1308 pvscsi_freeze(sc); 1309 ccb_h->status = CAM_REQUEUE_REQ; 1310 goto finish_ccb; 1311 } 1312 1313 hcb->ccb = ccb; 1314 ccb_h->ccb_pvscsi_hcb = hcb; 1315 ccb_h->ccb_pvscsi_sc = sc; 1316 1317 if (csio->cdb_len > sizeof(e->cdb)) { 1318 DEBUG_PRINTF(2, sc->dev, "cdb length %u too large\n", 1319 csio->cdb_len); 1320 ccb_h->status = CAM_REQ_INVALID; 1321 goto finish_ccb; 1322 } 1323 1324 if (ccb_h->flags & CAM_CDB_PHYS) { 1325 DEBUG_PRINTF(2, sc->dev, 1326 "CAM_CDB_PHYS not implemented\n"); 1327 ccb_h->status = CAM_REQ_INVALID; 1328 goto finish_ccb; 1329 } 1330 1331 e = ring + (s->req_prod_idx & MASK(req_num_entries_log2)); 1332 1333 e->bus = cam_sim_bus(sim); 1334 e->target = ccb_h->target_id; 1335 memset(e->lun, 0, sizeof(e->lun)); 1336 e->lun[1] = ccb_h->target_lun; 1337 e->data_addr = 0; 1338 e->data_len = csio->dxfer_len; 1339 e->vcpu_hint = curcpu; 1340 1341 e->cdb_len = csio->cdb_len; 1342 memcpy(e->cdb, scsiio_cdb_ptr(csio), csio->cdb_len); 1343 1344 e->sense_addr = 0; 1345 e->sense_len = csio->sense_len; 1346 if (e->sense_len > 0) { 1347 e->sense_addr = hcb->sense_buffer_paddr; 1348 } 1349 1350 e->tag = MSG_SIMPLE_Q_TAG; 1351 if (ccb_h->flags & CAM_TAG_ACTION_VALID) { 1352 e->tag = csio->tag_action; 1353 } 1354 1355 e->context = pvscsi_hcb_to_context(sc, hcb); 1356 hcb->e = e; 1357 1358 DEBUG_PRINTF(3, sc->dev, 1359 " queuing command %02x context %llx\n", e->cdb[0], 1360 (unsigned long long)e->context); 1361 bus_dmamap_load_ccb(sc->buffer_dmat, hcb->dma_map, ccb, 1362 pvscsi_execute_ccb, hcb, 0); 1363 break; 1364 1365 finish_ccb: 1366 if (hcb != NULL) { 1367 pvscsi_hcb_put(sc, hcb); 1368 } 1369 xpt_done(ccb); 1370 } break; 1371 case XPT_ABORT: 1372 { 1373 struct pvscsi_hcb *abort_hcb; 1374 union ccb *abort_ccb; 1375 1376 abort_ccb = ccb->cab.abort_ccb; 1377 abort_hcb = abort_ccb->ccb_h.ccb_pvscsi_hcb; 1378 1379 if (abort_hcb->ccb != NULL && abort_hcb->ccb == abort_ccb) { 1380 if (abort_ccb->ccb_h.func_code == XPT_SCSI_IO) { 1381 pvscsi_abort(sc, ccb_h->target_id, abort_ccb); 1382 ccb_h->status = CAM_REQ_CMP; 1383 } else { 1384 ccb_h->status = CAM_UA_ABORT; 1385 } 1386 } else { 1387 device_printf(sc->dev, 1388 "Could not find hcb for ccb %p (tgt %u)\n", 1389 ccb, ccb_h->target_id); 1390 ccb_h->status = CAM_REQ_CMP; 1391 } 1392 xpt_done(ccb); 1393 } break; 1394 case XPT_RESET_DEV: 1395 { 1396 pvscsi_device_reset(sc, ccb_h->target_id); 1397 ccb_h->status = CAM_REQ_CMP; 1398 xpt_done(ccb); 1399 } break; 1400 case XPT_RESET_BUS: 1401 { 1402 pvscsi_bus_reset(sc); 1403 ccb_h->status = CAM_REQ_CMP; 1404 xpt_done(ccb); 1405 } break; 1406 case XPT_PATH_INQ: 1407 { 1408 struct ccb_pathinq *cpi; 1409 1410 cpi = &ccb->cpi; 1411 1412 cpi->version_num = 1; 1413 cpi->hba_inquiry = PI_TAG_ABLE; 1414 cpi->target_sprt = 0; 1415 cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED; 1416 cpi->hba_eng_cnt = 0; 1417 /* cpi->vuhba_flags = 0; */ 1418 cpi->max_target = sc->max_targets; 1419 cpi->max_lun = 0; 1420 cpi->async_flags = 0; 1421 cpi->hpath_id = 0; 1422 cpi->unit_number = cam_sim_unit(sim); 1423 cpi->bus_id = cam_sim_bus(sim); 1424 cpi->initiator_id = 7; 1425 cpi->base_transfer_speed = 750000; 1426 strlcpy(cpi->sim_vid, "VMware", SIM_IDLEN); 1427 strlcpy(cpi->hba_vid, "VMware", HBA_IDLEN); 1428 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 1429 cpi->maxio = PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT * PAGE_SIZE; 1430 cpi->protocol = PROTO_SCSI; 1431 cpi->protocol_version = SCSI_REV_SPC2; 1432 cpi->transport = XPORT_SAS; 1433 cpi->transport_version = 0; 1434 1435 ccb_h->status = CAM_REQ_CMP; 1436 xpt_done(ccb); 1437 } break; 1438 case XPT_GET_TRAN_SETTINGS: 1439 { 1440 struct ccb_trans_settings *cts; 1441 1442 cts = &ccb->cts; 1443 1444 cts->protocol = PROTO_SCSI; 1445 cts->protocol_version = SCSI_REV_SPC2; 1446 cts->transport = XPORT_SAS; 1447 cts->transport_version = 0; 1448 1449 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB; 1450 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ; 1451 1452 ccb_h->status = CAM_REQ_CMP; 1453 xpt_done(ccb); 1454 } break; 1455 case XPT_CALC_GEOMETRY: 1456 { 1457 cam_calc_geometry(&ccb->ccg, 1); 1458 xpt_done(ccb); 1459 } break; 1460 default: 1461 ccb_h->status = CAM_REQ_INVALID; 1462 xpt_done(ccb); 1463 break; 1464 } 1465 } 1466 1467 static void 1468 pvscsi_free_interrupts(struct pvscsi_softc *sc) 1469 { 1470 1471 if (sc->irq_handler != NULL) { 1472 bus_teardown_intr(sc->dev, sc->irq_res, sc->irq_handler); 1473 } 1474 if (sc->irq_res != NULL) { 1475 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_id, 1476 sc->irq_res); 1477 } 1478 if (sc->use_msi_or_msix) { 1479 pci_release_msi(sc->dev); 1480 } 1481 } 1482 1483 static int 1484 pvscsi_setup_interrupts(struct pvscsi_softc *sc) 1485 { 1486 int error; 1487 int flags; 1488 int use_msix; 1489 int use_msi; 1490 int count; 1491 1492 sc->use_msi_or_msix = 0; 1493 1494 use_msix = pvscsi_get_tunable(sc, "use_msix", pvscsi_use_msix); 1495 use_msi = pvscsi_get_tunable(sc, "use_msi", pvscsi_use_msi); 1496 1497 if (use_msix && pci_msix_count(sc->dev) > 0) { 1498 count = 1; 1499 if (pci_alloc_msix(sc->dev, &count) == 0 && count == 1) { 1500 sc->use_msi_or_msix = 1; 1501 device_printf(sc->dev, "Interrupt: MSI-X\n"); 1502 } else { 1503 pci_release_msi(sc->dev); 1504 } 1505 } 1506 1507 if (sc->use_msi_or_msix == 0 && use_msi && pci_msi_count(sc->dev) > 0) { 1508 count = 1; 1509 if (pci_alloc_msi(sc->dev, &count) == 0 && count == 1) { 1510 sc->use_msi_or_msix = 1; 1511 device_printf(sc->dev, "Interrupt: MSI\n"); 1512 } else { 1513 pci_release_msi(sc->dev); 1514 } 1515 } 1516 1517 flags = RF_ACTIVE; 1518 if (sc->use_msi_or_msix) { 1519 sc->irq_id = 1; 1520 } else { 1521 device_printf(sc->dev, "Interrupt: INT\n"); 1522 sc->irq_id = 0; 1523 flags |= RF_SHAREABLE; 1524 } 1525 1526 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_id, 1527 flags); 1528 if (sc->irq_res == NULL) { 1529 device_printf(sc->dev, "IRQ allocation failed\n"); 1530 if (sc->use_msi_or_msix) { 1531 pci_release_msi(sc->dev); 1532 } 1533 return (ENXIO); 1534 } 1535 1536 error = bus_setup_intr(sc->dev, sc->irq_res, 1537 INTR_TYPE_CAM | INTR_MPSAFE, NULL, pvscsi_intr, sc, 1538 &sc->irq_handler); 1539 if (error) { 1540 device_printf(sc->dev, "IRQ handler setup failed\n"); 1541 pvscsi_free_interrupts(sc); 1542 return (error); 1543 } 1544 1545 return (0); 1546 } 1547 1548 static void 1549 pvscsi_free_all(struct pvscsi_softc *sc) 1550 { 1551 1552 if (sc->sim) { 1553 int32_t status; 1554 1555 if (sc->bus_path) { 1556 xpt_free_path(sc->bus_path); 1557 } 1558 1559 status = xpt_bus_deregister(cam_sim_path(sc->sim)); 1560 if (status != CAM_REQ_CMP) { 1561 device_printf(sc->dev, 1562 "Error deregistering bus, status=%d\n", status); 1563 } 1564 1565 cam_sim_free(sc->sim, TRUE); 1566 } 1567 1568 pvscsi_dma_free_per_hcb(sc, sc->hcb_cnt); 1569 1570 if (sc->hcbs) { 1571 free(sc->hcbs, M_PVSCSI); 1572 } 1573 1574 pvscsi_free_rings(sc); 1575 1576 pvscsi_free_interrupts(sc); 1577 1578 if (sc->buffer_dmat != NULL) { 1579 bus_dma_tag_destroy(sc->buffer_dmat); 1580 } 1581 1582 if (sc->parent_dmat != NULL) { 1583 bus_dma_tag_destroy(sc->parent_dmat); 1584 } 1585 1586 if (sc->mm_res != NULL) { 1587 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->mm_rid, 1588 sc->mm_res); 1589 } 1590 } 1591 1592 static int 1593 pvscsi_attach(device_t dev) 1594 { 1595 struct pvscsi_softc *sc; 1596 int rid; 1597 int barid; 1598 int error; 1599 int max_queue_depth; 1600 int adapter_queue_size; 1601 struct cam_devq *devq; 1602 1603 sc = device_get_softc(dev); 1604 sc->dev = dev; 1605 1606 mtx_init(&sc->lock, "pvscsi", NULL, MTX_DEF); 1607 1608 pci_enable_busmaster(dev); 1609 1610 sc->mm_rid = -1; 1611 for (barid = 0; barid <= PCIR_MAX_BAR_0; ++barid) { 1612 rid = PCIR_BAR(barid); 1613 1614 sc->mm_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 1615 RF_ACTIVE); 1616 if (sc->mm_res != NULL) { 1617 sc->mm_rid = rid; 1618 break; 1619 } 1620 } 1621 1622 if (sc->mm_res == NULL) { 1623 device_printf(dev, "could not map device memory\n"); 1624 return (ENXIO); 1625 } 1626 1627 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 1628 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, 1629 BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, 0, NULL, NULL, 1630 &sc->parent_dmat); 1631 if (error) { 1632 device_printf(dev, "parent dma tag create failure, error %d\n", 1633 error); 1634 pvscsi_free_all(sc); 1635 return (ENXIO); 1636 } 1637 1638 error = bus_dma_tag_create(sc->parent_dmat, 1, 0, 1639 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 1640 PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT * PAGE_SIZE, 1641 PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT, PAGE_SIZE, BUS_DMA_ALLOCNOW, 1642 NULL, NULL, &sc->buffer_dmat); 1643 if (error) { 1644 device_printf(dev, "parent dma tag create failure, error %d\n", 1645 error); 1646 pvscsi_free_all(sc); 1647 return (ENXIO); 1648 } 1649 1650 error = pvscsi_setup_interrupts(sc); 1651 if (error) { 1652 device_printf(dev, "Interrupt setup failed\n"); 1653 pvscsi_free_all(sc); 1654 return (error); 1655 } 1656 1657 sc->max_targets = pvscsi_get_max_targets(sc); 1658 1659 sc->use_msg = pvscsi_get_tunable(sc, "use_msg", pvscsi_use_msg) && 1660 pvscsi_hw_supports_msg(sc); 1661 sc->msg_ring_num_pages = sc->use_msg ? 1 : 0; 1662 1663 sc->req_ring_num_pages = pvscsi_get_tunable(sc, "request_ring_pages", 1664 pvscsi_request_ring_pages); 1665 if (sc->req_ring_num_pages <= 0) { 1666 if (sc->max_targets <= 16) { 1667 sc->req_ring_num_pages = 1668 PVSCSI_DEFAULT_NUM_PAGES_REQ_RING; 1669 } else { 1670 sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; 1671 } 1672 } else if (sc->req_ring_num_pages > PVSCSI_MAX_NUM_PAGES_REQ_RING) { 1673 sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; 1674 } 1675 sc->cmp_ring_num_pages = sc->req_ring_num_pages; 1676 1677 max_queue_depth = pvscsi_get_tunable(sc, "max_queue_depth", 1678 pvscsi_max_queue_depth); 1679 1680 adapter_queue_size = (sc->req_ring_num_pages * PAGE_SIZE) / 1681 sizeof(struct pvscsi_ring_req_desc); 1682 if (max_queue_depth > 0) { 1683 adapter_queue_size = MIN(adapter_queue_size, max_queue_depth); 1684 } 1685 adapter_queue_size = MIN(adapter_queue_size, 1686 PVSCSI_MAX_REQ_QUEUE_DEPTH); 1687 1688 device_printf(sc->dev, "Use Msg: %d\n", sc->use_msg); 1689 device_printf(sc->dev, "REQ num pages: %d\n", sc->req_ring_num_pages); 1690 device_printf(sc->dev, "CMP num pages: %d\n", sc->cmp_ring_num_pages); 1691 device_printf(sc->dev, "MSG num pages: %d\n", sc->msg_ring_num_pages); 1692 device_printf(sc->dev, "Queue size: %d\n", adapter_queue_size); 1693 1694 if (pvscsi_allocate_rings(sc)) { 1695 device_printf(dev, "ring allocation failed\n"); 1696 pvscsi_free_all(sc); 1697 return (ENXIO); 1698 } 1699 1700 sc->hcb_cnt = adapter_queue_size; 1701 sc->hcbs = malloc(sc->hcb_cnt * sizeof(*sc->hcbs), M_PVSCSI, 1702 M_NOWAIT | M_ZERO); 1703 if (sc->hcbs == NULL) { 1704 device_printf(dev, "error allocating hcb array\n"); 1705 pvscsi_free_all(sc); 1706 return (ENXIO); 1707 } 1708 1709 if (pvscsi_dma_alloc_per_hcb(sc)) { 1710 device_printf(dev, "error allocating per hcb dma memory\n"); 1711 pvscsi_free_all(sc); 1712 return (ENXIO); 1713 } 1714 1715 pvscsi_adapter_reset(sc); 1716 1717 devq = cam_simq_alloc(adapter_queue_size); 1718 if (devq == NULL) { 1719 device_printf(dev, "cam devq alloc failed\n"); 1720 pvscsi_free_all(sc); 1721 return (ENXIO); 1722 } 1723 1724 sc->sim = cam_sim_alloc(pvscsi_action, pvscsi_poll, "pvscsi", sc, 1725 device_get_unit(dev), &sc->lock, 1, adapter_queue_size, devq); 1726 if (sc->sim == NULL) { 1727 device_printf(dev, "cam sim alloc failed\n"); 1728 cam_simq_free(devq); 1729 pvscsi_free_all(sc); 1730 return (ENXIO); 1731 } 1732 1733 mtx_lock(&sc->lock); 1734 1735 if (xpt_bus_register(sc->sim, dev, 0) != CAM_SUCCESS) { 1736 device_printf(dev, "xpt bus register failed\n"); 1737 pvscsi_free_all(sc); 1738 mtx_unlock(&sc->lock); 1739 return (ENXIO); 1740 } 1741 1742 if (xpt_create_path(&sc->bus_path, NULL, cam_sim_path(sc->sim), 1743 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1744 device_printf(dev, "xpt create path failed\n"); 1745 pvscsi_free_all(sc); 1746 mtx_unlock(&sc->lock); 1747 return (ENXIO); 1748 } 1749 1750 pvscsi_setup_rings(sc); 1751 if (sc->use_msg) { 1752 pvscsi_setup_msg_ring(sc); 1753 } 1754 1755 sc->use_req_call_threshold = pvscsi_setup_req_call(sc, 1); 1756 1757 pvscsi_intr_enable(sc); 1758 1759 mtx_unlock(&sc->lock); 1760 1761 return (0); 1762 } 1763 1764 static int 1765 pvscsi_detach(device_t dev) 1766 { 1767 struct pvscsi_softc *sc; 1768 1769 sc = device_get_softc(dev); 1770 1771 pvscsi_intr_disable(sc); 1772 pvscsi_adapter_reset(sc); 1773 1774 if (sc->irq_handler != NULL) { 1775 bus_teardown_intr(dev, sc->irq_res, sc->irq_handler); 1776 } 1777 1778 mtx_lock(&sc->lock); 1779 pvscsi_free_all(sc); 1780 mtx_unlock(&sc->lock); 1781 1782 mtx_destroy(&sc->lock); 1783 1784 return (0); 1785 } 1786 1787 static device_method_t pvscsi_methods[] = { 1788 DEVMETHOD(device_probe, pvscsi_probe), 1789 DEVMETHOD(device_shutdown, pvscsi_shutdown), 1790 DEVMETHOD(device_attach, pvscsi_attach), 1791 DEVMETHOD(device_detach, pvscsi_detach), 1792 DEVMETHOD_END 1793 }; 1794 1795 static driver_t pvscsi_driver = { 1796 "pvscsi", pvscsi_methods, sizeof(struct pvscsi_softc) 1797 }; 1798 1799 static devclass_t pvscsi_devclass; 1800 DRIVER_MODULE(pvscsi, pci, pvscsi_driver, pvscsi_devclass, 0, 0); 1801 1802 MODULE_DEPEND(pvscsi, pci, 1, 1, 1); 1803 MODULE_DEPEND(pvscsi, cam, 1, 1, 1); 1804