1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* Driver for the VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/module.h> 39 #include <sys/malloc.h> 40 41 #include <machine/bus.h> 42 #include <machine/resource.h> 43 #include <sys/bus.h> 44 #include <sys/rman.h> 45 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pcireg.h> 48 49 #include <dev/virtio/virtio.h> 50 #include <dev/virtio/virtqueue.h> 51 #include <dev/virtio/pci/virtio_pci.h> 52 53 #include "virtio_bus_if.h" 54 #include "virtio_if.h" 55 56 struct vtpci_interrupt { 57 struct resource *vti_irq; 58 int vti_rid; 59 void *vti_handler; 60 }; 61 62 struct vtpci_virtqueue { 63 struct virtqueue *vtv_vq; 64 int vtv_no_intr; 65 }; 66 67 struct vtpci_softc { 68 device_t vtpci_dev; 69 struct resource *vtpci_res; 70 struct resource *vtpci_msix_res; 71 uint64_t vtpci_features; 72 uint32_t vtpci_flags; 73 #define VTPCI_FLAG_NO_MSI 0x0001 74 #define VTPCI_FLAG_NO_MSIX 0x0002 75 #define VTPCI_FLAG_LEGACY 0x1000 76 #define VTPCI_FLAG_MSI 0x2000 77 #define VTPCI_FLAG_MSIX 0x4000 78 #define VTPCI_FLAG_SHARED_MSIX 0x8000 79 #define VTPCI_FLAG_ITYPE_MASK 0xF000 80 81 /* This "bus" will only ever have one child. */ 82 device_t vtpci_child_dev; 83 struct virtio_feature_desc *vtpci_child_feat_desc; 84 85 int vtpci_nvqs; 86 struct vtpci_virtqueue *vtpci_vqs; 87 88 /* 89 * Ideally, each virtqueue that the driver provides a callback for will 90 * receive its own MSIX vector. If there are not sufficient vectors 91 * available, then attempt to have all the VQs share one vector. For 92 * MSIX, the configuration changed notifications must be on their own 93 * vector. 94 * 95 * If MSIX is not available, we will attempt to have the whole device 96 * share one MSI vector, and then, finally, one legacy interrupt. 97 */ 98 struct vtpci_interrupt vtpci_device_interrupt; 99 struct vtpci_interrupt *vtpci_msix_vq_interrupts; 100 int vtpci_nmsix_resources; 101 }; 102 103 static int vtpci_probe(device_t); 104 static int vtpci_attach(device_t); 105 static int vtpci_detach(device_t); 106 static int vtpci_suspend(device_t); 107 static int vtpci_resume(device_t); 108 static int vtpci_shutdown(device_t); 109 static void vtpci_driver_added(device_t, driver_t *); 110 static void vtpci_child_detached(device_t, device_t); 111 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 112 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 113 114 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 115 static int vtpci_with_feature(device_t, uint64_t); 116 static int vtpci_alloc_virtqueues(device_t, int, int, 117 struct vq_alloc_info *); 118 static int vtpci_setup_intr(device_t, enum intr_type); 119 static void vtpci_stop(device_t); 120 static int vtpci_reinit(device_t, uint64_t); 121 static void vtpci_reinit_complete(device_t); 122 static void vtpci_notify_virtqueue(device_t, uint16_t); 123 static uint8_t vtpci_get_status(device_t); 124 static void vtpci_set_status(device_t, uint8_t); 125 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 126 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 127 128 static void vtpci_describe_features(struct vtpci_softc *, const char *, 129 uint64_t); 130 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 131 132 static int vtpci_alloc_msix(struct vtpci_softc *, int); 133 static int vtpci_alloc_msi(struct vtpci_softc *); 134 static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); 135 static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); 136 static int vtpci_alloc_intr_msi(struct vtpci_softc *); 137 static int vtpci_alloc_intr_legacy(struct vtpci_softc *); 138 static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, 139 struct vtpci_interrupt *); 140 static int vtpci_alloc_intr_resources(struct vtpci_softc *); 141 142 static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, 143 enum intr_type); 144 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, 145 enum intr_type); 146 static int vtpci_setup_msix_interrupts(struct vtpci_softc *, 147 enum intr_type); 148 static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); 149 150 static int vtpci_register_msix_vector(struct vtpci_softc *, int, 151 struct vtpci_interrupt *); 152 static int vtpci_set_host_msix_vectors(struct vtpci_softc *); 153 static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); 154 155 static void vtpci_free_interrupt(struct vtpci_softc *, 156 struct vtpci_interrupt *); 157 static void vtpci_free_interrupts(struct vtpci_softc *); 158 static void vtpci_free_virtqueues(struct vtpci_softc *); 159 static void vtpci_release_child_resources(struct vtpci_softc *); 160 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); 161 static void vtpci_reset(struct vtpci_softc *); 162 163 static void vtpci_select_virtqueue(struct vtpci_softc *, int); 164 165 static void vtpci_legacy_intr(void *); 166 static int vtpci_vq_shared_intr_filter(void *); 167 static void vtpci_vq_shared_intr(void *); 168 static int vtpci_vq_intr_filter(void *); 169 static void vtpci_vq_intr(void *); 170 static void vtpci_config_intr(void *); 171 172 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt 173 174 #define VIRTIO_PCI_CONFIG(_sc) \ 175 VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) 176 177 /* 178 * I/O port read/write wrappers. 179 */ 180 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 181 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 182 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 183 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 184 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 185 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 186 187 /* Tunables. */ 188 static int vtpci_disable_msix = 0; 189 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 190 191 static device_method_t vtpci_methods[] = { 192 /* Device interface. */ 193 DEVMETHOD(device_probe, vtpci_probe), 194 DEVMETHOD(device_attach, vtpci_attach), 195 DEVMETHOD(device_detach, vtpci_detach), 196 DEVMETHOD(device_suspend, vtpci_suspend), 197 DEVMETHOD(device_resume, vtpci_resume), 198 DEVMETHOD(device_shutdown, vtpci_shutdown), 199 200 /* Bus interface. */ 201 DEVMETHOD(bus_driver_added, vtpci_driver_added), 202 DEVMETHOD(bus_child_detached, vtpci_child_detached), 203 DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), 204 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 205 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 206 207 /* VirtIO bus interface. */ 208 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 209 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 210 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 211 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 212 DEVMETHOD(virtio_bus_stop, vtpci_stop), 213 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 214 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 215 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 216 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 217 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 218 219 DEVMETHOD_END 220 }; 221 222 static driver_t vtpci_driver = { 223 "virtio_pci", 224 vtpci_methods, 225 sizeof(struct vtpci_softc) 226 }; 227 228 devclass_t vtpci_devclass; 229 230 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 231 MODULE_VERSION(virtio_pci, 1); 232 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 233 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 234 235 static int 236 vtpci_probe(device_t dev) 237 { 238 char desc[36]; 239 const char *name; 240 241 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 242 return (ENXIO); 243 244 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 245 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 246 return (ENXIO); 247 248 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 249 return (ENXIO); 250 251 name = virtio_device_name(pci_get_subdevice(dev)); 252 if (name == NULL) 253 name = "Unknown"; 254 255 snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 256 device_set_desc_copy(dev, desc); 257 258 return (BUS_PROBE_DEFAULT); 259 } 260 261 static int 262 vtpci_attach(device_t dev) 263 { 264 struct vtpci_softc *sc; 265 device_t child; 266 int rid; 267 268 sc = device_get_softc(dev); 269 sc->vtpci_dev = dev; 270 271 pci_enable_busmaster(dev); 272 273 rid = PCIR_BAR(0); 274 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 275 RF_ACTIVE); 276 if (sc->vtpci_res == NULL) { 277 device_printf(dev, "cannot map I/O space\n"); 278 return (ENXIO); 279 } 280 281 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0) 282 sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; 283 284 if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { 285 rid = PCIR_BAR(1); 286 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 287 SYS_RES_MEMORY, &rid, RF_ACTIVE); 288 } 289 290 if (sc->vtpci_msix_res == NULL) 291 sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; 292 293 vtpci_reset(sc); 294 295 /* Tell the host we've noticed this device. */ 296 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 297 298 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 299 device_printf(dev, "cannot create child device\n"); 300 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 301 vtpci_detach(dev); 302 return (ENOMEM); 303 } 304 305 sc->vtpci_child_dev = child; 306 vtpci_probe_and_attach_child(sc); 307 308 return (0); 309 } 310 311 static int 312 vtpci_detach(device_t dev) 313 { 314 struct vtpci_softc *sc; 315 device_t child; 316 int error; 317 318 sc = device_get_softc(dev); 319 320 if ((child = sc->vtpci_child_dev) != NULL) { 321 error = device_delete_child(dev, child); 322 if (error) 323 return (error); 324 sc->vtpci_child_dev = NULL; 325 } 326 327 vtpci_reset(sc); 328 329 if (sc->vtpci_msix_res != NULL) { 330 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 331 sc->vtpci_msix_res); 332 sc->vtpci_msix_res = NULL; 333 } 334 335 if (sc->vtpci_res != NULL) { 336 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 337 sc->vtpci_res); 338 sc->vtpci_res = NULL; 339 } 340 341 return (0); 342 } 343 344 static int 345 vtpci_suspend(device_t dev) 346 { 347 348 return (bus_generic_suspend(dev)); 349 } 350 351 static int 352 vtpci_resume(device_t dev) 353 { 354 355 return (bus_generic_resume(dev)); 356 } 357 358 static int 359 vtpci_shutdown(device_t dev) 360 { 361 362 (void) bus_generic_shutdown(dev); 363 /* Forcibly stop the host device. */ 364 vtpci_stop(dev); 365 366 return (0); 367 } 368 369 static void 370 vtpci_driver_added(device_t dev, driver_t *driver) 371 { 372 struct vtpci_softc *sc; 373 374 sc = device_get_softc(dev); 375 376 vtpci_probe_and_attach_child(sc); 377 } 378 379 static void 380 vtpci_child_detached(device_t dev, device_t child) 381 { 382 struct vtpci_softc *sc; 383 384 sc = device_get_softc(dev); 385 386 vtpci_reset(sc); 387 vtpci_release_child_resources(sc); 388 } 389 390 static int 391 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 392 { 393 struct vtpci_softc *sc; 394 395 sc = device_get_softc(dev); 396 397 if (sc->vtpci_child_dev != child) 398 return (ENOENT); 399 400 switch (index) { 401 case VIRTIO_IVAR_DEVTYPE: 402 case VIRTIO_IVAR_SUBDEVICE: 403 *result = pci_get_subdevice(dev); 404 break; 405 case VIRTIO_IVAR_VENDOR: 406 *result = pci_get_vendor(dev); 407 break; 408 case VIRTIO_IVAR_DEVICE: 409 *result = pci_get_device(dev); 410 break; 411 case VIRTIO_IVAR_SUBVENDOR: 412 *result = pci_get_subvendor(dev); 413 break; 414 default: 415 return (ENOENT); 416 } 417 418 return (0); 419 } 420 421 static int 422 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 423 { 424 struct vtpci_softc *sc; 425 426 sc = device_get_softc(dev); 427 428 if (sc->vtpci_child_dev != child) 429 return (ENOENT); 430 431 switch (index) { 432 case VIRTIO_IVAR_FEATURE_DESC: 433 sc->vtpci_child_feat_desc = (void *) value; 434 break; 435 default: 436 return (ENOENT); 437 } 438 439 return (0); 440 } 441 442 static uint64_t 443 vtpci_negotiate_features(device_t dev, uint64_t child_features) 444 { 445 struct vtpci_softc *sc; 446 uint64_t host_features, features; 447 448 sc = device_get_softc(dev); 449 450 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 451 vtpci_describe_features(sc, "host", host_features); 452 453 /* 454 * Limit negotiated features to what the driver, virtqueue, and 455 * host all support. 456 */ 457 features = host_features & child_features; 458 features = virtqueue_filter_features(features); 459 sc->vtpci_features = features; 460 461 vtpci_describe_features(sc, "negotiated", features); 462 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 463 464 return (features); 465 } 466 467 static int 468 vtpci_with_feature(device_t dev, uint64_t feature) 469 { 470 struct vtpci_softc *sc; 471 472 sc = device_get_softc(dev); 473 474 return ((sc->vtpci_features & feature) != 0); 475 } 476 477 static int 478 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 479 struct vq_alloc_info *vq_info) 480 { 481 struct vtpci_softc *sc; 482 struct virtqueue *vq; 483 struct vtpci_virtqueue *vqx; 484 struct vq_alloc_info *info; 485 int idx, error; 486 uint16_t size; 487 488 sc = device_get_softc(dev); 489 490 if (sc->vtpci_nvqs != 0) 491 return (EALREADY); 492 if (nvqs <= 0) 493 return (EINVAL); 494 495 sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), 496 M_DEVBUF, M_NOWAIT | M_ZERO); 497 if (sc->vtpci_vqs == NULL) 498 return (ENOMEM); 499 500 for (idx = 0; idx < nvqs; idx++) { 501 vqx = &sc->vtpci_vqs[idx]; 502 info = &vq_info[idx]; 503 504 vtpci_select_virtqueue(sc, idx); 505 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 506 507 error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, 508 ~(vm_paddr_t)0, info, &vq); 509 if (error) { 510 device_printf(dev, 511 "cannot allocate virtqueue %d: %d\n", idx, error); 512 break; 513 } 514 515 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 516 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 517 518 vqx->vtv_vq = *info->vqai_vq = vq; 519 vqx->vtv_no_intr = info->vqai_intr == NULL; 520 521 sc->vtpci_nvqs++; 522 } 523 524 if (error) 525 vtpci_free_virtqueues(sc); 526 527 return (error); 528 } 529 530 static int 531 vtpci_setup_intr(device_t dev, enum intr_type type) 532 { 533 struct vtpci_softc *sc; 534 int attempt, error; 535 536 sc = device_get_softc(dev); 537 538 for (attempt = 0; attempt < 5; attempt++) { 539 /* 540 * Start with the most desirable interrupt configuration and 541 * fallback towards less desirable ones. 542 */ 543 switch (attempt) { 544 case 0: 545 error = vtpci_alloc_intr_msix_pervq(sc); 546 break; 547 case 1: 548 error = vtpci_alloc_intr_msix_shared(sc); 549 break; 550 case 2: 551 error = vtpci_alloc_intr_msi(sc); 552 break; 553 case 3: 554 error = vtpci_alloc_intr_legacy(sc); 555 break; 556 default: 557 device_printf(dev, 558 "exhausted all interrupt allocation attempts\n"); 559 return (ENXIO); 560 } 561 562 if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) 563 break; 564 565 vtpci_cleanup_setup_intr_attempt(sc); 566 } 567 568 if (bootverbose) { 569 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 570 device_printf(dev, "using legacy interrupt\n"); 571 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 572 device_printf(dev, "using MSI interrupt\n"); 573 else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) 574 device_printf(dev, "using shared MSIX interrupts\n"); 575 else 576 device_printf(dev, "using per VQ MSIX interrupts\n"); 577 } 578 579 return (0); 580 } 581 582 static void 583 vtpci_stop(device_t dev) 584 { 585 586 vtpci_reset(device_get_softc(dev)); 587 } 588 589 static int 590 vtpci_reinit(device_t dev, uint64_t features) 591 { 592 struct vtpci_softc *sc; 593 int idx, error; 594 595 sc = device_get_softc(dev); 596 597 /* 598 * Redrive the device initialization. This is a bit of an abuse of 599 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to 600 * play nice. 601 * 602 * We do not allow the host device to change from what was originally 603 * negotiated beyond what the guest driver changed. MSIX state should 604 * not change, number of virtqueues and their size remain the same, etc. 605 * This will need to be rethought when we want to support migration. 606 */ 607 608 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 609 vtpci_stop(dev); 610 611 /* 612 * Quickly drive the status through ACK and DRIVER. The device 613 * does not become usable again until vtpci_reinit_complete(). 614 */ 615 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 616 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 617 618 vtpci_negotiate_features(dev, features); 619 620 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 621 error = vtpci_reinit_virtqueue(sc, idx); 622 if (error) 623 return (error); 624 } 625 626 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 627 error = vtpci_set_host_msix_vectors(sc); 628 if (error) 629 return (error); 630 } 631 632 return (0); 633 } 634 635 static void 636 vtpci_reinit_complete(device_t dev) 637 { 638 639 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 640 } 641 642 static void 643 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 644 { 645 struct vtpci_softc *sc; 646 647 sc = device_get_softc(dev); 648 649 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 650 } 651 652 static uint8_t 653 vtpci_get_status(device_t dev) 654 { 655 struct vtpci_softc *sc; 656 657 sc = device_get_softc(dev); 658 659 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 660 } 661 662 static void 663 vtpci_set_status(device_t dev, uint8_t status) 664 { 665 struct vtpci_softc *sc; 666 667 sc = device_get_softc(dev); 668 669 if (status != VIRTIO_CONFIG_STATUS_RESET) 670 status |= vtpci_get_status(dev); 671 672 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 673 } 674 675 static void 676 vtpci_read_dev_config(device_t dev, bus_size_t offset, 677 void *dst, int length) 678 { 679 struct vtpci_softc *sc; 680 bus_size_t off; 681 uint8_t *d; 682 int size; 683 684 sc = device_get_softc(dev); 685 off = VIRTIO_PCI_CONFIG(sc) + offset; 686 687 for (d = dst; length > 0; d += size, off += size, length -= size) { 688 if (length >= 4) { 689 size = 4; 690 *(uint32_t *)d = vtpci_read_config_4(sc, off); 691 } else if (length >= 2) { 692 size = 2; 693 *(uint16_t *)d = vtpci_read_config_2(sc, off); 694 } else { 695 size = 1; 696 *d = vtpci_read_config_1(sc, off); 697 } 698 } 699 } 700 701 static void 702 vtpci_write_dev_config(device_t dev, bus_size_t offset, 703 void *src, int length) 704 { 705 struct vtpci_softc *sc; 706 bus_size_t off; 707 uint8_t *s; 708 int size; 709 710 sc = device_get_softc(dev); 711 off = VIRTIO_PCI_CONFIG(sc) + offset; 712 713 for (s = src; length > 0; s += size, off += size, length -= size) { 714 if (length >= 4) { 715 size = 4; 716 vtpci_write_config_4(sc, off, *(uint32_t *)s); 717 } else if (length >= 2) { 718 size = 2; 719 vtpci_write_config_2(sc, off, *(uint16_t *)s); 720 } else { 721 size = 1; 722 vtpci_write_config_1(sc, off, *s); 723 } 724 } 725 } 726 727 static void 728 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 729 uint64_t features) 730 { 731 device_t dev, child; 732 733 dev = sc->vtpci_dev; 734 child = sc->vtpci_child_dev; 735 736 if (device_is_attached(child) || bootverbose == 0) 737 return; 738 739 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 740 } 741 742 static void 743 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 744 { 745 device_t dev, child; 746 747 dev = sc->vtpci_dev; 748 child = sc->vtpci_child_dev; 749 750 if (child == NULL) 751 return; 752 753 if (device_get_state(child) != DS_NOTPRESENT) 754 return; 755 756 if (device_probe(child) != 0) 757 return; 758 759 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 760 if (device_attach(child) != 0) { 761 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 762 vtpci_reset(sc); 763 vtpci_release_child_resources(sc); 764 /* Reset status for future attempt. */ 765 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 766 } else { 767 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 768 VIRTIO_ATTACH_COMPLETED(child); 769 } 770 } 771 772 static int 773 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 774 { 775 device_t dev; 776 int nmsix, cnt, required; 777 778 dev = sc->vtpci_dev; 779 780 /* Allocate an additional vector for the config changes. */ 781 required = nvectors + 1; 782 783 nmsix = pci_msix_count(dev); 784 if (nmsix < required) 785 return (1); 786 787 cnt = required; 788 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 789 sc->vtpci_nmsix_resources = required; 790 return (0); 791 } 792 793 pci_release_msi(dev); 794 795 return (1); 796 } 797 798 static int 799 vtpci_alloc_msi(struct vtpci_softc *sc) 800 { 801 device_t dev; 802 int nmsi, cnt, required; 803 804 dev = sc->vtpci_dev; 805 required = 1; 806 807 nmsi = pci_msi_count(dev); 808 if (nmsi < required) 809 return (1); 810 811 cnt = required; 812 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) 813 return (0); 814 815 pci_release_msi(dev); 816 817 return (1); 818 } 819 820 static int 821 vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) 822 { 823 int i, nvectors, error; 824 825 if (vtpci_disable_msix != 0 || 826 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 827 return (ENOTSUP); 828 829 for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { 830 if (sc->vtpci_vqs[i].vtv_no_intr == 0) 831 nvectors++; 832 } 833 834 error = vtpci_alloc_msix(sc, nvectors); 835 if (error) 836 return (error); 837 838 sc->vtpci_flags |= VTPCI_FLAG_MSIX; 839 840 return (0); 841 } 842 843 static int 844 vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) 845 { 846 int error; 847 848 if (vtpci_disable_msix != 0 || 849 sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) 850 return (ENOTSUP); 851 852 error = vtpci_alloc_msix(sc, 1); 853 if (error) 854 return (error); 855 856 sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; 857 858 return (0); 859 } 860 861 static int 862 vtpci_alloc_intr_msi(struct vtpci_softc *sc) 863 { 864 int error; 865 866 /* Only BHyVe supports MSI. */ 867 if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) 868 return (ENOTSUP); 869 870 error = vtpci_alloc_msi(sc); 871 if (error) 872 return (error); 873 874 sc->vtpci_flags |= VTPCI_FLAG_MSI; 875 876 return (0); 877 } 878 879 static int 880 vtpci_alloc_intr_legacy(struct vtpci_softc *sc) 881 { 882 883 sc->vtpci_flags |= VTPCI_FLAG_LEGACY; 884 885 return (0); 886 } 887 888 static int 889 vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, 890 struct vtpci_interrupt *intr) 891 { 892 struct resource *irq; 893 894 irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); 895 if (irq == NULL) 896 return (ENXIO); 897 898 intr->vti_irq = irq; 899 intr->vti_rid = rid; 900 901 return (0); 902 } 903 904 static int 905 vtpci_alloc_intr_resources(struct vtpci_softc *sc) 906 { 907 struct vtpci_interrupt *intr; 908 int i, rid, flags, nvq_intrs, error; 909 910 rid = 0; 911 flags = RF_ACTIVE; 912 913 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 914 flags |= RF_SHAREABLE; 915 else 916 rid = 1; 917 918 /* 919 * For legacy and MSI interrupts, this single resource handles all 920 * interrupts. For MSIX, this resource is used for the configuration 921 * changed interrupt. 922 */ 923 intr = &sc->vtpci_device_interrupt; 924 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 925 if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) 926 return (error); 927 928 /* Subtract one for the configuration changed interrupt. */ 929 nvq_intrs = sc->vtpci_nmsix_resources - 1; 930 931 intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * 932 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO); 933 if (sc->vtpci_msix_vq_interrupts == NULL) 934 return (ENOMEM); 935 936 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) { 937 error = vtpci_alloc_interrupt(sc, rid, flags, intr); 938 if (error) 939 return (error); 940 } 941 942 return (0); 943 } 944 945 static int 946 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) 947 { 948 struct vtpci_interrupt *intr; 949 int error; 950 951 intr = &sc->vtpci_device_interrupt; 952 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, 953 vtpci_legacy_intr, sc, &intr->vti_handler); 954 955 return (error); 956 } 957 958 static int 959 vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 960 { 961 struct vtpci_virtqueue *vqx; 962 struct vtpci_interrupt *intr; 963 int i, error; 964 965 intr = sc->vtpci_msix_vq_interrupts; 966 967 for (i = 0; i < sc->vtpci_nvqs; i++) { 968 vqx = &sc->vtpci_vqs[i]; 969 970 if (vqx->vtv_no_intr) 971 continue; 972 973 error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, 974 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq, 975 &intr->vti_handler); 976 if (error) 977 return (error); 978 979 intr++; 980 } 981 982 return (0); 983 } 984 985 static int 986 vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) 987 { 988 device_t dev; 989 struct vtpci_interrupt *intr; 990 int error; 991 992 dev = sc->vtpci_dev; 993 intr = &sc->vtpci_device_interrupt; 994 995 error = bus_setup_intr(dev, intr->vti_irq, type, NULL, 996 vtpci_config_intr, sc, &intr->vti_handler); 997 if (error) 998 return (error); 999 1000 if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { 1001 intr = sc->vtpci_msix_vq_interrupts; 1002 error = bus_setup_intr(dev, intr->vti_irq, type, 1003 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, 1004 &intr->vti_handler); 1005 } else 1006 error = vtpci_setup_pervq_msix_interrupts(sc, type); 1007 1008 return (error ? error : vtpci_set_host_msix_vectors(sc)); 1009 } 1010 1011 static int 1012 vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) 1013 { 1014 int error; 1015 1016 type |= INTR_MPSAFE; 1017 KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, 1018 ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); 1019 1020 error = vtpci_alloc_intr_resources(sc); 1021 if (error) 1022 return (error); 1023 1024 if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) 1025 error = vtpci_setup_legacy_interrupt(sc, type); 1026 else if (sc->vtpci_flags & VTPCI_FLAG_MSI) 1027 error = vtpci_setup_msi_interrupt(sc, type); 1028 else 1029 error = vtpci_setup_msix_interrupts(sc, type); 1030 1031 return (error); 1032 } 1033 1034 static int 1035 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, 1036 struct vtpci_interrupt *intr) 1037 { 1038 device_t dev; 1039 uint16_t vector; 1040 1041 dev = sc->vtpci_dev; 1042 1043 if (intr != NULL) { 1044 /* Map from guest rid to host vector. */ 1045 vector = intr->vti_rid - 1; 1046 } else 1047 vector = VIRTIO_MSI_NO_VECTOR; 1048 1049 vtpci_write_config_2(sc, offset, vector); 1050 1051 /* Read vector to determine if the host had sufficient resources. */ 1052 if (vtpci_read_config_2(sc, offset) != vector) { 1053 device_printf(dev, 1054 "insufficient host resources for MSIX interrupts\n"); 1055 return (ENODEV); 1056 } 1057 1058 return (0); 1059 } 1060 1061 static int 1062 vtpci_set_host_msix_vectors(struct vtpci_softc *sc) 1063 { 1064 struct vtpci_interrupt *intr, *tintr; 1065 int idx, offset, error; 1066 1067 intr = &sc->vtpci_device_interrupt; 1068 offset = VIRTIO_MSI_CONFIG_VECTOR; 1069 1070 error = vtpci_register_msix_vector(sc, offset, intr); 1071 if (error) 1072 return (error); 1073 1074 intr = sc->vtpci_msix_vq_interrupts; 1075 offset = VIRTIO_MSI_QUEUE_VECTOR; 1076 1077 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1078 vtpci_select_virtqueue(sc, idx); 1079 1080 if (sc->vtpci_vqs[idx].vtv_no_intr) 1081 tintr = NULL; 1082 else 1083 tintr = intr; 1084 1085 error = vtpci_register_msix_vector(sc, offset, tintr); 1086 if (error) 1087 break; 1088 1089 /* 1090 * For shared MSIX, all the virtqueues share the first 1091 * interrupt. 1092 */ 1093 if (!sc->vtpci_vqs[idx].vtv_no_intr && 1094 (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) 1095 intr++; 1096 } 1097 1098 return (error); 1099 } 1100 1101 static int 1102 vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) 1103 { 1104 struct vtpci_virtqueue *vqx; 1105 struct virtqueue *vq; 1106 int error; 1107 uint16_t size; 1108 1109 vqx = &sc->vtpci_vqs[idx]; 1110 vq = vqx->vtv_vq; 1111 1112 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 1113 1114 vtpci_select_virtqueue(sc, idx); 1115 size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 1116 1117 error = virtqueue_reinit(vq, size); 1118 if (error) 1119 return (error); 1120 1121 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 1122 virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 1123 1124 return (0); 1125 } 1126 1127 static void 1128 vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) 1129 { 1130 device_t dev; 1131 1132 dev = sc->vtpci_dev; 1133 1134 if (intr->vti_handler != NULL) { 1135 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); 1136 intr->vti_handler = NULL; 1137 } 1138 1139 if (intr->vti_irq != NULL) { 1140 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, 1141 intr->vti_irq); 1142 intr->vti_irq = NULL; 1143 intr->vti_rid = -1; 1144 } 1145 } 1146 1147 static void 1148 vtpci_free_interrupts(struct vtpci_softc *sc) 1149 { 1150 struct vtpci_interrupt *intr; 1151 int i, nvq_intrs; 1152 1153 vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); 1154 1155 if (sc->vtpci_nmsix_resources != 0) { 1156 nvq_intrs = sc->vtpci_nmsix_resources - 1; 1157 sc->vtpci_nmsix_resources = 0; 1158 1159 intr = sc->vtpci_msix_vq_interrupts; 1160 if (intr != NULL) { 1161 for (i = 0; i < nvq_intrs; i++, intr++) 1162 vtpci_free_interrupt(sc, intr); 1163 1164 free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); 1165 sc->vtpci_msix_vq_interrupts = NULL; 1166 } 1167 } 1168 1169 if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) 1170 pci_release_msi(sc->vtpci_dev); 1171 1172 sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; 1173 } 1174 1175 static void 1176 vtpci_free_virtqueues(struct vtpci_softc *sc) 1177 { 1178 struct vtpci_virtqueue *vqx; 1179 int idx; 1180 1181 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1182 vqx = &sc->vtpci_vqs[idx]; 1183 1184 vtpci_select_virtqueue(sc, idx); 1185 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); 1186 1187 virtqueue_free(vqx->vtv_vq); 1188 vqx->vtv_vq = NULL; 1189 } 1190 1191 free(sc->vtpci_vqs, M_DEVBUF); 1192 sc->vtpci_vqs = NULL; 1193 sc->vtpci_nvqs = 0; 1194 } 1195 1196 static void 1197 vtpci_release_child_resources(struct vtpci_softc *sc) 1198 { 1199 1200 vtpci_free_interrupts(sc); 1201 vtpci_free_virtqueues(sc); 1202 } 1203 1204 static void 1205 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) 1206 { 1207 int idx; 1208 1209 if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { 1210 vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, 1211 VIRTIO_MSI_NO_VECTOR); 1212 1213 for (idx = 0; idx < sc->vtpci_nvqs; idx++) { 1214 vtpci_select_virtqueue(sc, idx); 1215 vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, 1216 VIRTIO_MSI_NO_VECTOR); 1217 } 1218 } 1219 1220 vtpci_free_interrupts(sc); 1221 } 1222 1223 static void 1224 vtpci_reset(struct vtpci_softc *sc) 1225 { 1226 1227 /* 1228 * Setting the status to RESET sets the host device to 1229 * the original, uninitialized state. 1230 */ 1231 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1232 } 1233 1234 static void 1235 vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) 1236 { 1237 1238 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); 1239 } 1240 1241 static void 1242 vtpci_legacy_intr(void *xsc) 1243 { 1244 struct vtpci_softc *sc; 1245 struct vtpci_virtqueue *vqx; 1246 int i; 1247 uint8_t isr; 1248 1249 sc = xsc; 1250 vqx = &sc->vtpci_vqs[0]; 1251 1252 /* Reading the ISR also clears it. */ 1253 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1254 1255 if (isr & VIRTIO_PCI_ISR_CONFIG) 1256 vtpci_config_intr(sc); 1257 1258 if (isr & VIRTIO_PCI_ISR_INTR) { 1259 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1260 if (vqx->vtv_no_intr == 0) 1261 virtqueue_intr(vqx->vtv_vq); 1262 } 1263 } 1264 } 1265 1266 static int 1267 vtpci_vq_shared_intr_filter(void *xsc) 1268 { 1269 struct vtpci_softc *sc; 1270 struct vtpci_virtqueue *vqx; 1271 int i, rc; 1272 1273 rc = 0; 1274 sc = xsc; 1275 vqx = &sc->vtpci_vqs[0]; 1276 1277 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1278 if (vqx->vtv_no_intr == 0) 1279 rc |= virtqueue_intr_filter(vqx->vtv_vq); 1280 } 1281 1282 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1283 } 1284 1285 static void 1286 vtpci_vq_shared_intr(void *xsc) 1287 { 1288 struct vtpci_softc *sc; 1289 struct vtpci_virtqueue *vqx; 1290 int i; 1291 1292 sc = xsc; 1293 vqx = &sc->vtpci_vqs[0]; 1294 1295 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { 1296 if (vqx->vtv_no_intr == 0) 1297 virtqueue_intr(vqx->vtv_vq); 1298 } 1299 } 1300 1301 static int 1302 vtpci_vq_intr_filter(void *xvq) 1303 { 1304 struct virtqueue *vq; 1305 int rc; 1306 1307 vq = xvq; 1308 rc = virtqueue_intr_filter(vq); 1309 1310 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY); 1311 } 1312 1313 static void 1314 vtpci_vq_intr(void *xvq) 1315 { 1316 struct virtqueue *vq; 1317 1318 vq = xvq; 1319 virtqueue_intr(vq); 1320 } 1321 1322 static void 1323 vtpci_config_intr(void *xsc) 1324 { 1325 struct vtpci_softc *sc; 1326 device_t child; 1327 1328 sc = xsc; 1329 child = sc->vtpci_child_dev; 1330 1331 if (child != NULL) 1332 VIRTIO_CONFIG_CHANGE(child); 1333 } 1334