1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/kernel.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/rman.h> 50 51 #include <machine/bus.h> 52 #include <machine/resource.h> 53 54 #include <dev/virtio/virtio.h> 55 #include <dev/virtio/virtqueue.h> 56 #include <dev/virtio/mmio/virtio_mmio.h> 57 58 #include "virtio_mmio_if.h" 59 #include "virtio_bus_if.h" 60 #include "virtio_if.h" 61 62 #define PAGE_SHIFT 12 63 64 struct vtmmio_virtqueue { 65 struct virtqueue *vtv_vq; 66 int vtv_no_intr; 67 }; 68 69 static int vtmmio_detach(device_t); 70 static int vtmmio_suspend(device_t); 71 static int vtmmio_resume(device_t); 72 static int vtmmio_shutdown(device_t); 73 static void vtmmio_driver_added(device_t, driver_t *); 74 static void vtmmio_child_detached(device_t, device_t); 75 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 76 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 77 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 78 static int vtmmio_with_feature(device_t, uint64_t); 79 static int vtmmio_alloc_virtqueues(device_t, int, int, 80 struct vq_alloc_info *); 81 static int vtmmio_setup_intr(device_t, enum intr_type); 82 static void vtmmio_stop(device_t); 83 static void vtmmio_poll(device_t); 84 static int vtmmio_reinit(device_t, uint64_t); 85 static void vtmmio_reinit_complete(device_t); 86 static void vtmmio_notify_virtqueue(device_t, uint16_t); 87 static uint8_t vtmmio_get_status(device_t); 88 static void vtmmio_set_status(device_t, uint8_t); 89 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 90 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 91 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 92 uint64_t); 93 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 94 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 95 static void vtmmio_free_interrupts(struct vtmmio_softc *); 96 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 97 static void vtmmio_release_child_resources(struct vtmmio_softc *); 98 static void vtmmio_reset(struct vtmmio_softc *); 99 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 100 static void vtmmio_vq_intr(void *); 101 102 /* 103 * I/O port read/write wrappers. 104 */ 105 #define vtmmio_write_config_1(sc, o, v) \ 106 do { \ 107 if (sc->platform != NULL) \ 108 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 109 bus_write_1((sc)->res[0], (o), (v)); \ 110 if (sc->platform != NULL) \ 111 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 112 } while (0) 113 #define vtmmio_write_config_2(sc, o, v) \ 114 do { \ 115 if (sc->platform != NULL) \ 116 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 117 bus_write_2((sc)->res[0], (o), (v)); \ 118 if (sc->platform != NULL) \ 119 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 120 } while (0) 121 #define vtmmio_write_config_4(sc, o, v) \ 122 do { \ 123 if (sc->platform != NULL) \ 124 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 125 bus_write_4((sc)->res[0], (o), (v)); \ 126 if (sc->platform != NULL) \ 127 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 128 } while (0) 129 130 #define vtmmio_read_config_1(sc, o) \ 131 bus_read_1((sc)->res[0], (o)) 132 #define vtmmio_read_config_2(sc, o) \ 133 bus_read_2((sc)->res[0], (o)) 134 #define vtmmio_read_config_4(sc, o) \ 135 bus_read_4((sc)->res[0], (o)) 136 137 static device_method_t vtmmio_methods[] = { 138 /* Device interface. */ 139 DEVMETHOD(device_attach, vtmmio_attach), 140 DEVMETHOD(device_detach, vtmmio_detach), 141 DEVMETHOD(device_suspend, vtmmio_suspend), 142 DEVMETHOD(device_resume, vtmmio_resume), 143 DEVMETHOD(device_shutdown, vtmmio_shutdown), 144 145 /* Bus interface. */ 146 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 147 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 148 DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), 149 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 150 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 151 152 /* VirtIO bus interface. */ 153 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 154 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 155 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 156 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 157 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 158 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 159 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 160 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 161 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 162 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 163 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 164 165 DEVMETHOD_END 166 }; 167 168 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, 169 sizeof(struct vtmmio_softc)); 170 171 MODULE_VERSION(virtio_mmio, 1); 172 173 static int 174 vtmmio_setup_intr(device_t dev, enum intr_type type) 175 { 176 struct vtmmio_softc *sc; 177 int rid; 178 int err; 179 180 sc = device_get_softc(dev); 181 182 if (sc->platform != NULL) { 183 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 184 vtmmio_vq_intr, sc); 185 if (err == 0) { 186 /* Okay we have backend-specific interrupts */ 187 return (0); 188 } 189 } 190 191 rid = 0; 192 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 193 RF_ACTIVE); 194 if (!sc->res[1]) { 195 device_printf(dev, "Can't allocate interrupt\n"); 196 return (ENXIO); 197 } 198 199 if (bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, 200 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 201 device_printf(dev, "Can't setup the interrupt\n"); 202 return (ENXIO); 203 } 204 205 return (0); 206 } 207 208 int 209 vtmmio_attach(device_t dev) 210 { 211 struct vtmmio_softc *sc; 212 device_t child; 213 int rid; 214 215 sc = device_get_softc(dev); 216 sc->dev = dev; 217 218 rid = 0; 219 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 220 RF_ACTIVE); 221 if (!sc->res[0]) { 222 device_printf(dev, "Cannot allocate memory window.\n"); 223 return (ENXIO); 224 } 225 226 vtmmio_reset(sc); 227 228 /* Tell the host we've noticed this device. */ 229 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 230 231 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 232 device_printf(dev, "Cannot create child device.\n"); 233 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 234 vtmmio_detach(dev); 235 return (ENOMEM); 236 } 237 238 sc->vtmmio_child_dev = child; 239 vtmmio_probe_and_attach_child(sc); 240 241 return (0); 242 } 243 244 static int 245 vtmmio_detach(device_t dev) 246 { 247 struct vtmmio_softc *sc; 248 device_t child; 249 int error; 250 251 sc = device_get_softc(dev); 252 253 if ((child = sc->vtmmio_child_dev) != NULL) { 254 error = device_delete_child(dev, child); 255 if (error) 256 return (error); 257 sc->vtmmio_child_dev = NULL; 258 } 259 260 vtmmio_reset(sc); 261 262 if (sc->res[0] != NULL) { 263 bus_release_resource(dev, SYS_RES_MEMORY, 0, 264 sc->res[0]); 265 sc->res[0] = NULL; 266 } 267 268 return (0); 269 } 270 271 static int 272 vtmmio_suspend(device_t dev) 273 { 274 275 return (bus_generic_suspend(dev)); 276 } 277 278 static int 279 vtmmio_resume(device_t dev) 280 { 281 282 return (bus_generic_resume(dev)); 283 } 284 285 static int 286 vtmmio_shutdown(device_t dev) 287 { 288 289 (void) bus_generic_shutdown(dev); 290 291 /* Forcibly stop the host device. */ 292 vtmmio_stop(dev); 293 294 return (0); 295 } 296 297 static void 298 vtmmio_driver_added(device_t dev, driver_t *driver) 299 { 300 struct vtmmio_softc *sc; 301 302 sc = device_get_softc(dev); 303 304 vtmmio_probe_and_attach_child(sc); 305 } 306 307 static void 308 vtmmio_child_detached(device_t dev, device_t child) 309 { 310 struct vtmmio_softc *sc; 311 312 sc = device_get_softc(dev); 313 314 vtmmio_reset(sc); 315 vtmmio_release_child_resources(sc); 316 } 317 318 static int 319 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 320 { 321 struct vtmmio_softc *sc; 322 323 sc = device_get_softc(dev); 324 325 if (sc->vtmmio_child_dev != child) 326 return (ENOENT); 327 328 switch (index) { 329 case VIRTIO_IVAR_DEVTYPE: 330 case VIRTIO_IVAR_SUBDEVICE: 331 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 332 break; 333 case VIRTIO_IVAR_VENDOR: 334 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 335 break; 336 case VIRTIO_IVAR_SUBVENDOR: 337 case VIRTIO_IVAR_DEVICE: 338 /* 339 * Dummy value for fields not present in this bus. Used by 340 * bus-agnostic virtio_child_pnpinfo_str. 341 */ 342 *result = 0; 343 break; 344 default: 345 return (ENOENT); 346 } 347 348 return (0); 349 } 350 351 static int 352 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 353 { 354 struct vtmmio_softc *sc; 355 356 sc = device_get_softc(dev); 357 358 if (sc->vtmmio_child_dev != child) 359 return (ENOENT); 360 361 switch (index) { 362 case VIRTIO_IVAR_FEATURE_DESC: 363 sc->vtmmio_child_feat_desc = (void *) value; 364 break; 365 default: 366 return (ENOENT); 367 } 368 369 return (0); 370 } 371 372 static uint64_t 373 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 374 { 375 struct vtmmio_softc *sc; 376 uint64_t host_features, features; 377 378 sc = device_get_softc(dev); 379 380 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 381 vtmmio_describe_features(sc, "host", host_features); 382 383 /* 384 * Limit negotiated features to what the driver, virtqueue, and 385 * host all support. 386 */ 387 features = host_features & child_features; 388 features = virtqueue_filter_features(features); 389 sc->vtmmio_features = features; 390 391 vtmmio_describe_features(sc, "negotiated", features); 392 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 393 394 return (features); 395 } 396 397 static int 398 vtmmio_with_feature(device_t dev, uint64_t feature) 399 { 400 struct vtmmio_softc *sc; 401 402 sc = device_get_softc(dev); 403 404 return ((sc->vtmmio_features & feature) != 0); 405 } 406 407 static int 408 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 409 struct vq_alloc_info *vq_info) 410 { 411 struct vtmmio_virtqueue *vqx; 412 struct vq_alloc_info *info; 413 struct vtmmio_softc *sc; 414 struct virtqueue *vq; 415 uint32_t size; 416 int idx, error; 417 418 sc = device_get_softc(dev); 419 420 if (sc->vtmmio_nvqs != 0) 421 return (EALREADY); 422 if (nvqs <= 0) 423 return (EINVAL); 424 425 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 426 M_DEVBUF, M_NOWAIT | M_ZERO); 427 if (sc->vtmmio_vqs == NULL) 428 return (ENOMEM); 429 430 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 431 (1 << PAGE_SHIFT)); 432 433 for (idx = 0; idx < nvqs; idx++) { 434 vqx = &sc->vtmmio_vqs[idx]; 435 info = &vq_info[idx]; 436 437 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 438 439 vtmmio_select_virtqueue(sc, idx); 440 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 441 442 error = virtqueue_alloc(dev, idx, size, 443 VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq); 444 if (error) { 445 device_printf(dev, 446 "cannot allocate virtqueue %d: %d\n", 447 idx, error); 448 break; 449 } 450 451 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 452 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 453 VIRTIO_MMIO_VRING_ALIGN); 454 #if 0 455 device_printf(dev, "virtqueue paddr 0x%08lx\n", 456 (uint64_t)virtqueue_paddr(vq)); 457 #endif 458 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 459 virtqueue_paddr(vq) >> PAGE_SHIFT); 460 461 vqx->vtv_vq = *info->vqai_vq = vq; 462 vqx->vtv_no_intr = info->vqai_intr == NULL; 463 464 sc->vtmmio_nvqs++; 465 } 466 467 if (error) 468 vtmmio_free_virtqueues(sc); 469 470 return (error); 471 } 472 473 static void 474 vtmmio_stop(device_t dev) 475 { 476 477 vtmmio_reset(device_get_softc(dev)); 478 } 479 480 static void 481 vtmmio_poll(device_t dev) 482 { 483 struct vtmmio_softc *sc; 484 485 sc = device_get_softc(dev); 486 487 if (sc->platform != NULL) 488 VIRTIO_MMIO_POLL(sc->platform); 489 } 490 491 static int 492 vtmmio_reinit(device_t dev, uint64_t features) 493 { 494 struct vtmmio_softc *sc; 495 int idx, error; 496 497 sc = device_get_softc(dev); 498 499 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 500 vtmmio_stop(dev); 501 502 /* 503 * Quickly drive the status through ACK and DRIVER. The device 504 * does not become usable again until vtmmio_reinit_complete(). 505 */ 506 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 507 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 508 509 vtmmio_negotiate_features(dev, features); 510 511 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 512 (1 << PAGE_SHIFT)); 513 514 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 515 error = vtmmio_reinit_virtqueue(sc, idx); 516 if (error) 517 return (error); 518 } 519 520 return (0); 521 } 522 523 static void 524 vtmmio_reinit_complete(device_t dev) 525 { 526 527 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 528 } 529 530 static void 531 vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 532 { 533 struct vtmmio_softc *sc; 534 535 sc = device_get_softc(dev); 536 537 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 538 } 539 540 static uint8_t 541 vtmmio_get_status(device_t dev) 542 { 543 struct vtmmio_softc *sc; 544 545 sc = device_get_softc(dev); 546 547 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 548 } 549 550 static void 551 vtmmio_set_status(device_t dev, uint8_t status) 552 { 553 struct vtmmio_softc *sc; 554 555 sc = device_get_softc(dev); 556 557 if (status != VIRTIO_CONFIG_STATUS_RESET) 558 status |= vtmmio_get_status(dev); 559 560 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 561 } 562 563 static void 564 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 565 void *dst, int length) 566 { 567 struct vtmmio_softc *sc; 568 bus_size_t off; 569 uint8_t *d; 570 int size; 571 572 sc = device_get_softc(dev); 573 off = VIRTIO_MMIO_CONFIG + offset; 574 575 for (d = dst; length > 0; d += size, off += size, length -= size) { 576 #ifdef ALLOW_WORD_ALIGNED_ACCESS 577 if (length >= 4) { 578 size = 4; 579 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 580 } else if (length >= 2) { 581 size = 2; 582 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 583 } else 584 #endif 585 { 586 size = 1; 587 *d = vtmmio_read_config_1(sc, off); 588 } 589 } 590 } 591 592 static void 593 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 594 void *src, int length) 595 { 596 struct vtmmio_softc *sc; 597 bus_size_t off; 598 uint8_t *s; 599 int size; 600 601 sc = device_get_softc(dev); 602 off = VIRTIO_MMIO_CONFIG + offset; 603 604 for (s = src; length > 0; s += size, off += size, length -= size) { 605 #ifdef ALLOW_WORD_ALIGNED_ACCESS 606 if (length >= 4) { 607 size = 4; 608 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 609 } else if (length >= 2) { 610 size = 2; 611 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 612 } else 613 #endif 614 { 615 size = 1; 616 vtmmio_write_config_1(sc, off, *s); 617 } 618 } 619 } 620 621 static void 622 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 623 uint64_t features) 624 { 625 device_t dev, child; 626 627 dev = sc->dev; 628 child = sc->vtmmio_child_dev; 629 630 if (device_is_attached(child) || bootverbose == 0) 631 return; 632 633 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 634 } 635 636 static void 637 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 638 { 639 device_t dev, child; 640 641 dev = sc->dev; 642 child = sc->vtmmio_child_dev; 643 644 if (child == NULL) 645 return; 646 647 if (device_get_state(child) != DS_NOTPRESENT) { 648 return; 649 } 650 651 if (device_probe(child) != 0) { 652 return; 653 } 654 655 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 656 if (device_attach(child) != 0) { 657 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 658 vtmmio_reset(sc); 659 vtmmio_release_child_resources(sc); 660 /* Reset status for future attempt. */ 661 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 662 } else { 663 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 664 VIRTIO_ATTACH_COMPLETED(child); 665 } 666 } 667 668 static int 669 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 670 { 671 struct vtmmio_virtqueue *vqx; 672 struct virtqueue *vq; 673 int error; 674 uint16_t size; 675 676 vqx = &sc->vtmmio_vqs[idx]; 677 vq = vqx->vtv_vq; 678 679 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 680 681 vtmmio_select_virtqueue(sc, idx); 682 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 683 684 error = virtqueue_reinit(vq, size); 685 if (error) 686 return (error); 687 688 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 689 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 690 VIRTIO_MMIO_VRING_ALIGN); 691 #if 0 692 device_printf(sc->dev, "virtqueue paddr 0x%08lx\n", 693 (uint64_t)virtqueue_paddr(vq)); 694 #endif 695 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 696 virtqueue_paddr(vq) >> PAGE_SHIFT); 697 698 return (0); 699 } 700 701 static void 702 vtmmio_free_interrupts(struct vtmmio_softc *sc) 703 { 704 705 if (sc->ih != NULL) 706 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 707 708 if (sc->res[1] != NULL) 709 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 710 } 711 712 static void 713 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 714 { 715 struct vtmmio_virtqueue *vqx; 716 int idx; 717 718 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 719 vqx = &sc->vtmmio_vqs[idx]; 720 721 vtmmio_select_virtqueue(sc, idx); 722 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 723 724 virtqueue_free(vqx->vtv_vq); 725 vqx->vtv_vq = NULL; 726 } 727 728 free(sc->vtmmio_vqs, M_DEVBUF); 729 sc->vtmmio_vqs = NULL; 730 sc->vtmmio_nvqs = 0; 731 } 732 733 static void 734 vtmmio_release_child_resources(struct vtmmio_softc *sc) 735 { 736 737 vtmmio_free_interrupts(sc); 738 vtmmio_free_virtqueues(sc); 739 } 740 741 static void 742 vtmmio_reset(struct vtmmio_softc *sc) 743 { 744 745 /* 746 * Setting the status to RESET sets the host device to 747 * the original, uninitialized state. 748 */ 749 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 750 } 751 752 static void 753 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 754 { 755 756 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 757 } 758 759 static void 760 vtmmio_vq_intr(void *arg) 761 { 762 struct vtmmio_virtqueue *vqx; 763 struct vtmmio_softc *sc; 764 struct virtqueue *vq; 765 uint32_t status; 766 int idx; 767 768 sc = arg; 769 770 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 771 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 772 773 /* The config changed */ 774 if (status & VIRTIO_MMIO_INT_CONFIG) 775 if (sc->vtmmio_child_dev != NULL) 776 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 777 778 /* Notify all virtqueues. */ 779 if (status & VIRTIO_MMIO_INT_VRING) { 780 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 781 vqx = &sc->vtmmio_vqs[idx]; 782 if (vqx->vtv_no_intr == 0) { 783 vq = vqx->vtv_vq; 784 virtqueue_intr(vq); 785 } 786 } 787 } 788 } 789