1 /*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7 /* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */ 8 9 #include <sys/cdefs.h> 10 __FBSDID("$FreeBSD$"); 11 12 #include <sys/param.h> 13 #include <sys/bus.h> 14 #include <sys/kernel.h> 15 #include <sys/malloc.h> 16 #include <sys/module.h> 17 #include <sys/rman.h> 18 #include <sys/systm.h> 19 20 #include <dev/pci/pcireg.h> 21 #include <dev/pci/pcivar.h> 22 23 #include <machine/bus.h> 24 25 #include "vmci.h" 26 #include "vmci_doorbell.h" 27 #include "vmci_driver.h" 28 #include "vmci_kernel_defs.h" 29 #include "vmci_queue_pair.h" 30 31 static int vmci_probe(device_t); 32 static int vmci_attach(device_t); 33 static int vmci_detach(device_t); 34 static int vmci_shutdown(device_t); 35 36 static int vmci_map_bars(struct vmci_softc *); 37 static void vmci_unmap_bars(struct vmci_softc *); 38 39 static int vmci_config_capabilities(struct vmci_softc *); 40 41 static int vmci_dma_malloc_int(struct vmci_softc *, bus_size_t, 42 bus_size_t, struct vmci_dma_alloc *); 43 static void vmci_dma_free_int(struct vmci_softc *, 44 struct vmci_dma_alloc *); 45 46 static int vmci_config_interrupts(struct vmci_softc *); 47 static int vmci_config_interrupt(struct vmci_softc *); 48 static int vmci_check_intr_cnt(struct vmci_softc *); 49 static int vmci_allocate_interrupt_resources(struct vmci_softc *); 50 static int vmci_setup_interrupts(struct vmci_softc *); 51 static void vmci_dismantle_interrupts(struct vmci_softc *); 52 static void vmci_interrupt(void *); 53 static void vmci_interrupt_bm(void *); 54 static void dispatch_datagrams(void *, int); 55 static void process_bitmap(void *, int); 56 57 static void vmci_delayed_work_fn_cb(void *context, int data); 58 59 static device_method_t vmci_methods[] = { 60 /* Device interface. */ 61 DEVMETHOD(device_probe, vmci_probe), 62 DEVMETHOD(device_attach, vmci_attach), 63 DEVMETHOD(device_detach, vmci_detach), 64 DEVMETHOD(device_shutdown, vmci_shutdown), 65 66 DEVMETHOD_END 67 }; 68 69 static driver_t vmci_driver = { 70 "vmci", vmci_methods, sizeof(struct vmci_softc) 71 }; 72 73 static devclass_t vmci_devclass; 74 DRIVER_MODULE(vmci, pci, vmci_driver, vmci_devclass, 0, 0); 75 MODULE_VERSION(vmci, VMCI_VERSION); 76 const struct { 77 uint16_t vendor; 78 uint16_t device; 79 const char *desc; 80 } vmci_ids[] = { 81 { VMCI_VMWARE_VENDOR_ID, VMCI_VMWARE_DEVICE_ID, 82 "VMware Virtual Machine Communication Interface" }, 83 }; 84 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmci, vmci_ids, 85 nitems(vmci_ids)); 86 87 MODULE_DEPEND(vmci, pci, 1, 1, 1); 88 89 static struct vmci_softc *vmci_sc; 90 91 #define LGPFX "vmci: " 92 /* 93 * Allocate a buffer for incoming datagrams globally to avoid repeated 94 * allocation in the interrupt handler's atomic context. 95 */ 96 static uint8_t *data_buffer = NULL; 97 static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE; 98 99 struct vmci_delayed_work_info { 100 vmci_work_fn *work_fn; 101 void *data; 102 vmci_list_item(vmci_delayed_work_info) entry; 103 }; 104 105 /* 106 *------------------------------------------------------------------------------ 107 * 108 * vmci_probe -- 109 * 110 * Probe to see if the VMCI device is present. 111 * 112 * Results: 113 * BUS_PROBE_DEFAULT if device exists, ENXIO otherwise. 114 * 115 * Side effects: 116 * None. 117 * 118 *------------------------------------------------------------------------------ 119 */ 120 121 static int 122 vmci_probe(device_t dev) 123 { 124 125 if (pci_get_vendor(dev) == vmci_ids[0].vendor && 126 pci_get_device(dev) == vmci_ids[0].device) { 127 device_set_desc(dev, vmci_ids[0].desc); 128 129 return (BUS_PROBE_DEFAULT); 130 } 131 132 return (ENXIO); 133 } 134 135 /* 136 *------------------------------------------------------------------------------ 137 * 138 * vmci_attach -- 139 * 140 * Attach VMCI device to the system after vmci_probe() has been called and 141 * the device has been detected. 142 * 143 * Results: 144 * 0 if success, ENXIO otherwise. 145 * 146 * Side effects: 147 * None. 148 * 149 *------------------------------------------------------------------------------ 150 */ 151 152 static int 153 vmci_attach(device_t dev) 154 { 155 struct vmci_softc *sc; 156 int error, i; 157 158 sc = device_get_softc(dev); 159 sc->vmci_dev = dev; 160 vmci_sc = sc; 161 162 data_buffer = NULL; 163 sc->vmci_num_intr = 0; 164 for (i = 0; i < VMCI_MAX_INTRS; i++) { 165 sc->vmci_intrs[i].vmci_irq = NULL; 166 sc->vmci_intrs[i].vmci_handler = NULL; 167 } 168 169 TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc); 170 TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc); 171 172 TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc); 173 174 pci_enable_busmaster(dev); 175 176 mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN); 177 mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock", 178 NULL, MTX_DEF); 179 180 error = vmci_map_bars(sc); 181 if (error) { 182 VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n"); 183 goto fail; 184 } 185 186 error = vmci_config_capabilities(sc); 187 if (error) { 188 VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n"); 189 goto fail; 190 } 191 192 vmci_list_init(&sc->vmci_delayed_work_infos); 193 194 vmci_components_init(); 195 vmci_util_init(); 196 error = vmci_qp_guest_endpoints_init(); 197 if (error) { 198 VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n"); 199 goto fail; 200 } 201 202 error = vmci_config_interrupts(sc); 203 if (error) 204 VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n"); 205 206 fail: 207 if (error) { 208 vmci_detach(dev); 209 return (ENXIO); 210 } 211 212 return (0); 213 } 214 215 /* 216 *------------------------------------------------------------------------------ 217 * 218 * vmci_detach -- 219 * 220 * Detach the VMCI device. 221 * 222 * Results: 223 * 0 224 * 225 * Side effects: 226 * None. 227 * 228 *------------------------------------------------------------------------------ 229 */ 230 231 static int 232 vmci_detach(device_t dev) 233 { 234 struct vmci_softc *sc; 235 236 sc = device_get_softc(dev); 237 238 vmci_qp_guest_endpoints_exit(); 239 vmci_util_exit(); 240 241 vmci_dismantle_interrupts(sc); 242 243 vmci_components_cleanup(); 244 245 if mtx_initialized(&sc->vmci_spinlock) { 246 taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task); 247 mtx_destroy(&sc->vmci_delayed_work_lock); 248 } 249 250 if (sc->vmci_res0 != NULL) 251 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 252 VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET); 253 254 if (sc->vmci_notifications_bitmap.dma_vaddr != NULL) 255 vmci_dma_free(&sc->vmci_notifications_bitmap); 256 257 vmci_unmap_bars(sc); 258 259 if mtx_initialized(&sc->vmci_spinlock) 260 mtx_destroy(&sc->vmci_spinlock); 261 262 pci_disable_busmaster(dev); 263 264 return (0); 265 } 266 267 /* 268 *------------------------------------------------------------------------------ 269 * 270 * vmci_shutdown -- 271 * 272 * This function is called during system shutdown. We don't do anything. 273 * 274 * Results: 275 * 0 276 * 277 * Side effects: 278 * None. 279 * 280 *------------------------------------------------------------------------------ 281 */ 282 283 static int 284 vmci_shutdown(device_t dev) 285 { 286 287 return (0); 288 } 289 290 /* 291 *------------------------------------------------------------------------------ 292 * 293 * vmci_map_bars -- 294 * 295 * Maps the PCI I/O and MMIO BARs. 296 * 297 * Results: 298 * 0 on success, ENXIO otherwise. 299 * 300 * Side effects: 301 * None. 302 * 303 *------------------------------------------------------------------------------ 304 */ 305 306 static int 307 vmci_map_bars(struct vmci_softc *sc) 308 { 309 int rid; 310 311 /* Map the PCI I/O BAR: BAR0 */ 312 rid = PCIR_BAR(0); 313 sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT, 314 &rid, RF_ACTIVE); 315 if (sc->vmci_res0 == NULL) { 316 VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n"); 317 return (ENXIO); 318 } 319 320 sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0); 321 sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0); 322 sc->vmci_ioaddr = rman_get_start(sc->vmci_res0); 323 324 /* Map the PCI MMIO BAR: BAR1 */ 325 rid = PCIR_BAR(1); 326 sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY, 327 &rid, RF_ACTIVE); 328 if (sc->vmci_res1 == NULL) { 329 VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n"); 330 return (ENXIO); 331 } 332 333 sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1); 334 sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1); 335 336 return (0); 337 } 338 339 /* 340 *------------------------------------------------------------------------------ 341 * 342 * vmci_unmap_bars -- 343 * 344 * Unmaps the VMCI PCI I/O and MMIO BARs. 345 * 346 * Results: 347 * None. 348 * 349 * Side effects: 350 * None. 351 * 352 *------------------------------------------------------------------------------ 353 */ 354 355 static void 356 vmci_unmap_bars(struct vmci_softc *sc) 357 { 358 int rid; 359 360 if (sc->vmci_res0 != NULL) { 361 rid = PCIR_BAR(0); 362 bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid, 363 sc->vmci_res0); 364 sc->vmci_res0 = NULL; 365 } 366 367 if (sc->vmci_res1 != NULL) { 368 rid = PCIR_BAR(1); 369 bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid, 370 sc->vmci_res1); 371 sc->vmci_res1 = NULL; 372 } 373 } 374 375 /* 376 *------------------------------------------------------------------------------ 377 * 378 * vmci_config_capabilities -- 379 * 380 * Check the VMCI device capabilities and configure the device accordingly. 381 * 382 * Results: 383 * 0 if success, ENODEV otherwise. 384 * 385 * Side effects: 386 * Device capabilities are enabled. 387 * 388 *------------------------------------------------------------------------------ 389 */ 390 391 static int 392 vmci_config_capabilities(struct vmci_softc *sc) 393 { 394 unsigned long bitmap_PPN; 395 int error; 396 397 /* 398 * Verify that the VMCI device supports the capabilities that we 399 * need. Datagrams are necessary and notifications will be used 400 * if the device supports it. 401 */ 402 sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0, 403 VMCI_CAPS_ADDR); 404 405 if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) { 406 VMCI_LOG_ERROR(LGPFX"VMCI device does not support " 407 "datagrams.\n"); 408 return (ENODEV); 409 } 410 411 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) { 412 sc->capabilities = VMCI_CAPS_DATAGRAM; 413 error = vmci_dma_malloc(PAGE_SIZE, 1, 414 &sc->vmci_notifications_bitmap); 415 if (error) 416 VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for " 417 "notification bitmap.\n"); 418 else { 419 memset(sc->vmci_notifications_bitmap.dma_vaddr, 0, 420 PAGE_SIZE); 421 sc->capabilities |= VMCI_CAPS_NOTIFICATIONS; 422 } 423 } else 424 sc->capabilities = VMCI_CAPS_DATAGRAM; 425 426 /* Let the host know which capabilities we intend to use. */ 427 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 428 VMCI_CAPS_ADDR, sc->capabilities); 429 430 /* 431 * Register notification bitmap with device if that capability is 432 * used. 433 */ 434 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) { 435 bitmap_PPN = 436 sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT; 437 vmci_register_notification_bitmap(bitmap_PPN); 438 } 439 440 /* Check host capabilities. */ 441 if (!vmci_check_host_capabilities()) 442 return (ENODEV); 443 444 return (0); 445 } 446 447 /* 448 *------------------------------------------------------------------------------ 449 * 450 * vmci_dmamap_cb -- 451 * 452 * Callback to receive mapping information resulting from the load of a 453 * bus_dmamap_t via bus_dmamap_load() 454 * 455 * Results: 456 * None. 457 * 458 * Side effects: 459 * None. 460 * 461 *------------------------------------------------------------------------------ 462 */ 463 464 static void 465 vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 466 { 467 bus_addr_t *baddr = arg; 468 469 if (error == 0) 470 *baddr = segs->ds_addr; 471 } 472 473 /* 474 *------------------------------------------------------------------------------ 475 * 476 * vmci_dma_malloc_int -- 477 * 478 * Internal function that allocates DMA memory. 479 * 480 * Results: 481 * 0 if success. 482 * ENOMEM if insufficient memory. 483 * EINPROGRESS if mapping is deferred. 484 * EINVAL if the request was invalid. 485 * 486 * Side effects: 487 * DMA memory is allocated. 488 * 489 *------------------------------------------------------------------------------ 490 */ 491 492 static int 493 vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align, 494 struct vmci_dma_alloc *dma) 495 { 496 int error; 497 498 bzero(dma, sizeof(struct vmci_dma_alloc)); 499 500 error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev), 501 align, 0, /* alignment, bounds */ 502 BUS_SPACE_MAXADDR, /* lowaddr */ 503 BUS_SPACE_MAXADDR, /* highaddr */ 504 NULL, NULL, /* filter, filterarg */ 505 size, /* maxsize */ 506 1, /* nsegments */ 507 size, /* maxsegsize */ 508 BUS_DMA_ALLOCNOW, /* flags */ 509 NULL, /* lockfunc */ 510 NULL, /* lockfuncarg */ 511 &dma->dma_tag); 512 if (error) { 513 VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error); 514 goto fail; 515 } 516 517 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 518 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map); 519 if (error) { 520 VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error); 521 goto fail; 522 } 523 524 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 525 size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT); 526 if (error) { 527 VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error); 528 goto fail; 529 } 530 531 dma->dma_size = size; 532 533 fail: 534 if (error) 535 vmci_dma_free(dma); 536 537 return (error); 538 } 539 540 /* 541 *------------------------------------------------------------------------------ 542 * 543 * vmci_dma_malloc -- 544 * 545 * This function is a wrapper around vmci_dma_malloc_int for callers 546 * outside of this module. Since we only support a single VMCI device, this 547 * wrapper provides access to the device softc structure. 548 * 549 * Results: 550 * 0 if success. 551 * ENOMEM if insufficient memory. 552 * EINPROGRESS if mapping is deferred. 553 * EINVAL if the request was invalid. 554 * 555 * Side effects: 556 * DMA memory is allocated. 557 * 558 *------------------------------------------------------------------------------ 559 */ 560 561 int 562 vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma) 563 { 564 565 return (vmci_dma_malloc_int(vmci_sc, size, align, dma)); 566 } 567 568 /* 569 *------------------------------------------------------------------------------ 570 * 571 * vmci_dma_free_int -- 572 * 573 * Internal function that frees DMA memory. 574 * 575 * Results: 576 * None. 577 * 578 * Side effects: 579 * Frees DMA memory. 580 * 581 *------------------------------------------------------------------------------ 582 */ 583 584 static void 585 vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma) 586 { 587 588 if (dma->dma_tag != NULL) { 589 if (dma->dma_paddr != 0) { 590 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 591 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 592 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 593 } 594 595 if (dma->dma_vaddr != NULL) 596 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, 597 dma->dma_map); 598 599 bus_dma_tag_destroy(dma->dma_tag); 600 } 601 bzero(dma, sizeof(struct vmci_dma_alloc)); 602 } 603 604 /* 605 *------------------------------------------------------------------------------ 606 * 607 * vmci_dma_free -- 608 * 609 * This function is a wrapper around vmci_dma_free_int for callers outside 610 * of this module. Since we only support a single VMCI device, this wrapper 611 * provides access to the device softc structure. 612 * 613 * Results: 614 * None. 615 * 616 * Side effects: 617 * Frees DMA memory. 618 * 619 *------------------------------------------------------------------------------ 620 */ 621 622 void 623 vmci_dma_free(struct vmci_dma_alloc *dma) 624 { 625 626 vmci_dma_free_int(vmci_sc, dma); 627 } 628 629 /* 630 *------------------------------------------------------------------------------ 631 * 632 * vmci_config_interrupts -- 633 * 634 * Configures and enables interrupts. Try to configure MSI-X. If this fails, 635 * try to configure MSI. If even this fails, try legacy interrupts. 636 * 637 * Results: 638 * 0 if success. 639 * ENOMEM if insufficient memory. 640 * ENODEV if the device doesn't support interrupts. 641 * ENXIO if the device configuration failed. 642 * 643 * Side effects: 644 * Interrupts get enabled if successful. 645 * 646 *------------------------------------------------------------------------------ 647 */ 648 649 static int 650 vmci_config_interrupts(struct vmci_softc *sc) 651 { 652 int error; 653 654 data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT); 655 if (data_buffer == NULL) 656 return (ENOMEM); 657 658 sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX; 659 error = vmci_config_interrupt(sc); 660 if (error) { 661 sc->vmci_intr_type = VMCI_INTR_TYPE_MSI; 662 error = vmci_config_interrupt(sc); 663 } 664 if (error) { 665 sc->vmci_intr_type = VMCI_INTR_TYPE_INTX; 666 error = vmci_config_interrupt(sc); 667 } 668 if (error) 669 return (error); 670 671 /* Enable specific interrupt bits. */ 672 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) 673 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 674 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION); 675 else 676 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 677 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM); 678 679 /* Enable interrupts. */ 680 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0, 681 VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE); 682 683 return (0); 684 } 685 686 /* 687 *------------------------------------------------------------------------------ 688 * 689 * vmci_config_interrupt -- 690 * 691 * Check the number of interrupts supported, allocate resources and setup 692 * interrupts. 693 * 694 * Results: 695 * 0 if success. 696 * ENOMEM if insufficient memory. 697 * ENODEV if the device doesn't support interrupts. 698 * ENXIO if the device configuration failed. 699 * 700 * Side effects: 701 * Resources get allocated and interrupts get setup (but not enabled) if 702 * successful. 703 * 704 *------------------------------------------------------------------------------ 705 */ 706 707 static int 708 vmci_config_interrupt(struct vmci_softc *sc) 709 { 710 int error; 711 712 error = vmci_check_intr_cnt(sc); 713 if (error) 714 return (error); 715 716 error = vmci_allocate_interrupt_resources(sc); 717 if (error) 718 return (error); 719 720 error = vmci_setup_interrupts(sc); 721 if (error) 722 return (error); 723 724 return (0); 725 } 726 727 /* 728 *------------------------------------------------------------------------------ 729 * 730 * vmci_check_intr_cnt -- 731 * 732 * Check the number of interrupts supported by the device and ask PCI bus 733 * to allocate appropriate number of interrupts. 734 * 735 * Results: 736 * 0 if success. 737 * ENODEV if the device doesn't support any interrupts. 738 * ENXIO if the device configuration failed. 739 * 740 * Side effects: 741 * Resources get allocated on success. 742 * 743 *------------------------------------------------------------------------------ 744 */ 745 746 static int 747 vmci_check_intr_cnt(struct vmci_softc *sc) 748 { 749 750 if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) { 751 sc->vmci_num_intr = 1; 752 return (0); 753 } 754 755 /* 756 * Make sure that the device supports the required number of MSI/MSI-X 757 * messages. We try for 2 MSI-X messages but 1 is good too. We need at 758 * least 1 MSI message. 759 */ 760 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ? 761 pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev); 762 763 if (!sc->vmci_num_intr) { 764 VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt" 765 " messages"); 766 return (ENODEV); 767 } 768 769 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ? 770 VMCI_MAX_INTRS : 1; 771 if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) { 772 if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr)) 773 return (ENXIO); 774 } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) { 775 if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr)) 776 return (ENXIO); 777 } 778 779 return (0); 780 } 781 782 /* 783 *------------------------------------------------------------------------------ 784 * 785 * vmci_allocate_interrupt_resources -- 786 * 787 * Allocate resources necessary for interrupts. 788 * 789 * Results: 790 * 0 if success, ENXIO otherwise. 791 * 792 * Side effects: 793 * Resources get allocated on success. 794 * 795 *------------------------------------------------------------------------------ 796 */ 797 798 static int 799 vmci_allocate_interrupt_resources(struct vmci_softc *sc) 800 { 801 struct resource *irq; 802 int flags, i, rid; 803 804 flags = RF_ACTIVE; 805 flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0; 806 rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1; 807 808 for (i = 0; i < sc->vmci_num_intr; i++, rid++) { 809 irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid, 810 flags); 811 if (irq == NULL) 812 return (ENXIO); 813 sc->vmci_intrs[i].vmci_irq = irq; 814 sc->vmci_intrs[i].vmci_rid = rid; 815 } 816 817 return (0); 818 } 819 820 /* 821 *------------------------------------------------------------------------------ 822 * 823 * vmci_setup_interrupts -- 824 * 825 * Sets up the interrupts. 826 * 827 * Results: 828 * 0 if success, appropriate error code from bus_setup_intr otherwise. 829 * 830 * Side effects: 831 * Interrupt handler gets attached. 832 * 833 *------------------------------------------------------------------------------ 834 */ 835 836 static int 837 vmci_setup_interrupts(struct vmci_softc *sc) 838 { 839 struct vmci_interrupt *intr; 840 int error, flags; 841 842 flags = INTR_TYPE_NET | INTR_MPSAFE; 843 if (sc->vmci_num_intr > 1) 844 flags |= INTR_EXCL; 845 846 intr = &sc->vmci_intrs[0]; 847 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL, 848 vmci_interrupt, NULL, &intr->vmci_handler); 849 if (error) 850 return (error); 851 852 if (sc->vmci_num_intr == 2) { 853 bus_describe_intr(sc->vmci_dev, intr->vmci_irq, 854 intr->vmci_handler, "dg"); 855 intr = &sc->vmci_intrs[1]; 856 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, 857 NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler); 858 if (error) 859 return (error); 860 bus_describe_intr(sc->vmci_dev, intr->vmci_irq, 861 intr->vmci_handler, "bm"); 862 } 863 864 return (0); 865 } 866 867 /* 868 *------------------------------------------------------------------------------ 869 * 870 * vmci_interrupt -- 871 * 872 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X 873 * interrupt (vector VMCI_INTR_DATAGRAM). 874 * 875 * Results: 876 * None. 877 * 878 * Side effects: 879 * None. 880 * 881 *------------------------------------------------------------------------------ 882 */ 883 884 static void 885 vmci_interrupt(void *arg) 886 { 887 888 if (vmci_sc->vmci_num_intr == 2) 889 taskqueue_enqueue(taskqueue_swi, 890 &vmci_sc->vmci_interrupt_dq_task); 891 else { 892 unsigned int icr; 893 894 icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR); 895 if (icr == 0 || icr == 0xffffffff) 896 return; 897 if (icr & VMCI_ICR_DATAGRAM) { 898 taskqueue_enqueue(taskqueue_swi, 899 &vmci_sc->vmci_interrupt_dq_task); 900 icr &= ~VMCI_ICR_DATAGRAM; 901 } 902 if (icr & VMCI_ICR_NOTIFICATION) { 903 taskqueue_enqueue(taskqueue_swi, 904 &vmci_sc->vmci_interrupt_bm_task); 905 icr &= ~VMCI_ICR_NOTIFICATION; 906 } 907 if (icr != 0) 908 VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt " 909 "cause"); 910 } 911 } 912 913 /* 914 *------------------------------------------------------------------------------ 915 * 916 * vmci_interrupt_bm -- 917 * 918 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION, 919 * which is for the notification bitmap. Will only get called if we are 920 * using MSI-X with exclusive vectors. 921 * 922 * Results: 923 * None. 924 * 925 * Side effects: 926 * None. 927 * 928 *------------------------------------------------------------------------------ 929 */ 930 931 static void 932 vmci_interrupt_bm(void *arg) 933 { 934 935 ASSERT(vmci_sc->vmci_num_intr == 2); 936 taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task); 937 } 938 939 /* 940 *------------------------------------------------------------------------------ 941 * 942 * dispatch_datagrams -- 943 * 944 * Reads and dispatches incoming datagrams. 945 * 946 * Results: 947 * None. 948 * 949 * Side effects: 950 * Reads data from the device. 951 * 952 *------------------------------------------------------------------------------ 953 */ 954 955 static void 956 dispatch_datagrams(void *context, int data) 957 { 958 959 if (data_buffer == NULL) 960 VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer " 961 "present"); 962 963 vmci_read_datagrams_from_port((vmci_io_handle) 0, 964 vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR, 965 data_buffer, data_buffer_size); 966 } 967 968 /* 969 *------------------------------------------------------------------------------ 970 * 971 * process_bitmap -- 972 * 973 * Scans the notification bitmap for raised flags, clears them and handles 974 * the notifications. 975 * 976 * Results: 977 * None. 978 * 979 * Side effects: 980 * None. 981 * 982 *------------------------------------------------------------------------------ 983 */ 984 985 static void 986 process_bitmap(void *context, int data) 987 { 988 989 if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL) 990 VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present"); 991 992 vmci_scan_notification_bitmap( 993 vmci_sc->vmci_notifications_bitmap.dma_vaddr); 994 } 995 996 /* 997 *------------------------------------------------------------------------------ 998 * 999 * vmci_dismantle_interrupts -- 1000 * 1001 * Releases resources, detaches the interrupt handler and drains the task 1002 * queue. 1003 * 1004 * Results: 1005 * None. 1006 * 1007 * Side effects: 1008 * No more interrupts. 1009 * 1010 *------------------------------------------------------------------------------ 1011 */ 1012 1013 static void 1014 vmci_dismantle_interrupts(struct vmci_softc *sc) 1015 { 1016 struct vmci_interrupt *intr; 1017 int i; 1018 1019 for (i = 0; i < sc->vmci_num_intr; i++) { 1020 intr = &sc->vmci_intrs[i]; 1021 if (intr->vmci_handler != NULL) { 1022 bus_teardown_intr(sc->vmci_dev, intr->vmci_irq, 1023 intr->vmci_handler); 1024 intr->vmci_handler = NULL; 1025 } 1026 if (intr->vmci_irq != NULL) { 1027 bus_release_resource(sc->vmci_dev, SYS_RES_IRQ, 1028 intr->vmci_rid, intr->vmci_irq); 1029 intr->vmci_irq = NULL; 1030 intr->vmci_rid = -1; 1031 } 1032 } 1033 1034 if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) && 1035 (sc->vmci_num_intr)) 1036 pci_release_msi(sc->vmci_dev); 1037 1038 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task); 1039 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task); 1040 1041 if (data_buffer != NULL) 1042 free(data_buffer, M_DEVBUF); 1043 } 1044 1045 /* 1046 *------------------------------------------------------------------------------ 1047 * 1048 * vmci_delayed_work_fn_cb -- 1049 * 1050 * Callback function that executes the queued up delayed work functions. 1051 * 1052 * Results: 1053 * None. 1054 * 1055 * Side effects: 1056 * None. 1057 * 1058 *------------------------------------------------------------------------------ 1059 */ 1060 1061 static void 1062 vmci_delayed_work_fn_cb(void *context, int data) 1063 { 1064 vmci_list(vmci_delayed_work_info) temp_list; 1065 1066 vmci_list_init(&temp_list); 1067 1068 /* 1069 * Swap vmci_delayed_work_infos list with the empty temp_list while 1070 * holding a lock. vmci_delayed_work_infos would then be an empty list 1071 * and temp_list would contain the elements from the original 1072 * vmci_delayed_work_infos. Finally, iterate through temp_list 1073 * executing the delayed callbacks. 1074 */ 1075 1076 mtx_lock(&vmci_sc->vmci_delayed_work_lock); 1077 vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos, 1078 vmci_delayed_work_info, entry); 1079 mtx_unlock(&vmci_sc->vmci_delayed_work_lock); 1080 1081 while (!vmci_list_empty(&temp_list)) { 1082 struct vmci_delayed_work_info *delayed_work_info = 1083 vmci_list_first(&temp_list); 1084 1085 delayed_work_info->work_fn(delayed_work_info->data); 1086 1087 vmci_list_remove(delayed_work_info, entry); 1088 vmci_free_kernel_mem(delayed_work_info, 1089 sizeof(*delayed_work_info)); 1090 } 1091 } 1092 1093 /* 1094 *------------------------------------------------------------------------------ 1095 * 1096 * vmci_schedule_delayed_work_fn -- 1097 * 1098 * Schedule the specified callback. 1099 * 1100 * Results: 1101 * 0 if success, error code otherwise. 1102 * 1103 * Side effects: 1104 * None. 1105 * 1106 *------------------------------------------------------------------------------ 1107 */ 1108 1109 int 1110 vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data) 1111 { 1112 struct vmci_delayed_work_info *delayed_work_info; 1113 1114 delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info), 1115 VMCI_MEMORY_ATOMIC); 1116 1117 if (!delayed_work_info) 1118 return (VMCI_ERROR_NO_MEM); 1119 1120 delayed_work_info->work_fn = work_fn; 1121 delayed_work_info->data = data; 1122 mtx_lock(&vmci_sc->vmci_delayed_work_lock); 1123 vmci_list_insert(&vmci_sc->vmci_delayed_work_infos, 1124 delayed_work_info, entry); 1125 mtx_unlock(&vmci_sc->vmci_delayed_work_lock); 1126 1127 taskqueue_enqueue(taskqueue_thread, 1128 &vmci_sc->vmci_delayed_work_task); 1129 1130 return (VMCI_SUCCESS); 1131 } 1132 1133 /* 1134 *------------------------------------------------------------------------------ 1135 * 1136 * vmci_send_datagram -- 1137 * 1138 * VM to hypervisor call mechanism. 1139 * 1140 * Results: 1141 * The result of the hypercall. 1142 * 1143 * Side effects: 1144 * None. 1145 * 1146 *------------------------------------------------------------------------------ 1147 */ 1148 1149 int 1150 vmci_send_datagram(struct vmci_datagram *dg) 1151 { 1152 int result; 1153 1154 if (dg == NULL) 1155 return (VMCI_ERROR_INVALID_ARGS); 1156 1157 /* 1158 * Need to acquire spinlock on the device because 1159 * the datagram data may be spread over multiple pages and the monitor 1160 * may interleave device user rpc calls from multiple VCPUs. Acquiring 1161 * the spinlock precludes that possibility. Disabling interrupts to 1162 * avoid incoming datagrams during a "rep out" and possibly landing up 1163 * in this function. 1164 */ 1165 mtx_lock_spin(&vmci_sc->vmci_spinlock); 1166 1167 /* 1168 * Send the datagram and retrieve the return value from the result 1169 * register. 1170 */ 1171 __asm__ __volatile__( 1172 "cld\n\t" 1173 "rep outsb\n\t" 1174 : /* No output. */ 1175 : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR), 1176 "c"(VMCI_DG_SIZE(dg)), "S"(dg) 1177 ); 1178 1179 /* 1180 * XXX: Should read result high port as well when updating handlers to 1181 * return 64bit. 1182 */ 1183 1184 result = bus_space_read_4(vmci_sc->vmci_iot0, 1185 vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR); 1186 mtx_unlock_spin(&vmci_sc->vmci_spinlock); 1187 1188 return (result); 1189 } 1190