1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 #include <sys/cdefs.h> 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/bus.h> 44 #include <sys/kernel.h> 45 #include <sys/module.h> 46 #include <sys/malloc.h> 47 #include <sys/rman.h> 48 #include <sys/endian.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <dev/virtio/virtio.h> 54 #include <dev/virtio/virtqueue.h> 55 #include <dev/virtio/mmio/virtio_mmio.h> 56 57 #include "virtio_mmio_if.h" 58 #include "virtio_bus_if.h" 59 #include "virtio_if.h" 60 61 struct vtmmio_virtqueue { 62 struct virtqueue *vtv_vq; 63 int vtv_no_intr; 64 }; 65 66 static int vtmmio_detach(device_t); 67 static int vtmmio_suspend(device_t); 68 static int vtmmio_resume(device_t); 69 static int vtmmio_shutdown(device_t); 70 static void vtmmio_driver_added(device_t, driver_t *); 71 static void vtmmio_child_detached(device_t, device_t); 72 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 73 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 74 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 75 static int vtmmio_finalize_features(device_t); 76 static int vtmmio_with_feature(device_t, uint64_t); 77 static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, 78 struct virtqueue *vq, uint32_t size); 79 static int vtmmio_alloc_virtqueues(device_t, int, int, 80 struct vq_alloc_info *); 81 static int vtmmio_setup_intr(device_t, enum intr_type); 82 static void vtmmio_stop(device_t); 83 static void vtmmio_poll(device_t); 84 static int vtmmio_reinit(device_t, uint64_t); 85 static void vtmmio_reinit_complete(device_t); 86 static void vtmmio_notify_virtqueue(device_t, uint16_t, bus_size_t); 87 static int vtmmio_config_generation(device_t); 88 static uint8_t vtmmio_get_status(device_t); 89 static void vtmmio_set_status(device_t, uint8_t); 90 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 91 static uint64_t vtmmio_read_dev_config_8(struct vtmmio_softc *, bus_size_t); 92 static void vtmmio_write_dev_config(device_t, bus_size_t, const void *, int); 93 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 94 uint64_t); 95 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 96 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 97 static void vtmmio_free_interrupts(struct vtmmio_softc *); 98 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 99 static void vtmmio_release_child_resources(struct vtmmio_softc *); 100 static void vtmmio_reset(struct vtmmio_softc *); 101 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 102 static void vtmmio_vq_intr(void *); 103 104 /* 105 * I/O port read/write wrappers. 106 */ 107 #define vtmmio_write_config_1(sc, o, v) \ 108 do { \ 109 if (sc->platform != NULL) \ 110 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 111 bus_write_1((sc)->res[0], (o), (v)); \ 112 if (sc->platform != NULL) \ 113 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 114 } while (0) 115 #define vtmmio_write_config_2(sc, o, v) \ 116 do { \ 117 if (sc->platform != NULL) \ 118 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 119 bus_write_2((sc)->res[0], (o), (v)); \ 120 if (sc->platform != NULL) \ 121 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 122 } while (0) 123 #define vtmmio_write_config_4(sc, o, v) \ 124 do { \ 125 if (sc->platform != NULL) \ 126 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 127 bus_write_4((sc)->res[0], (o), (v)); \ 128 if (sc->platform != NULL) \ 129 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 130 } while (0) 131 132 #define vtmmio_read_config_1(sc, o) \ 133 bus_read_1((sc)->res[0], (o)) 134 #define vtmmio_read_config_2(sc, o) \ 135 bus_read_2((sc)->res[0], (o)) 136 #define vtmmio_read_config_4(sc, o) \ 137 bus_read_4((sc)->res[0], (o)) 138 139 static device_method_t vtmmio_methods[] = { 140 /* Device interface. */ 141 DEVMETHOD(device_attach, vtmmio_attach), 142 DEVMETHOD(device_detach, vtmmio_detach), 143 DEVMETHOD(device_suspend, vtmmio_suspend), 144 DEVMETHOD(device_resume, vtmmio_resume), 145 DEVMETHOD(device_shutdown, vtmmio_shutdown), 146 147 /* Bus interface. */ 148 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 149 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 150 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo), 151 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 152 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 153 154 /* VirtIO bus interface. */ 155 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 156 DEVMETHOD(virtio_bus_finalize_features, vtmmio_finalize_features), 157 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 158 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 159 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 160 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 161 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 162 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 163 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 164 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 165 DEVMETHOD(virtio_bus_config_generation, vtmmio_config_generation), 166 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 167 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 168 169 DEVMETHOD_END 170 }; 171 172 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, 173 sizeof(struct vtmmio_softc)); 174 175 MODULE_VERSION(virtio_mmio, 1); 176 177 int 178 vtmmio_probe(device_t dev) 179 { 180 struct vtmmio_softc *sc; 181 int rid; 182 uint32_t magic, version; 183 184 sc = device_get_softc(dev); 185 186 rid = 0; 187 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 188 RF_ACTIVE); 189 if (sc->res[0] == NULL) { 190 device_printf(dev, "Cannot allocate memory window.\n"); 191 return (ENXIO); 192 } 193 194 magic = vtmmio_read_config_4(sc, VIRTIO_MMIO_MAGIC_VALUE); 195 if (magic != VIRTIO_MMIO_MAGIC_VIRT) { 196 device_printf(dev, "Bad magic value %#x\n", magic); 197 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 198 return (ENXIO); 199 } 200 201 version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 202 if (version < 1 || version > 2) { 203 device_printf(dev, "Unsupported version: %#x\n", version); 204 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 205 return (ENXIO); 206 } 207 208 if (vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID) == 0) { 209 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 210 return (ENXIO); 211 } 212 213 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->res[0]); 214 215 device_set_desc(dev, "VirtIO MMIO adapter"); 216 return (BUS_PROBE_DEFAULT); 217 } 218 219 static int 220 vtmmio_setup_intr(device_t dev, enum intr_type type) 221 { 222 struct vtmmio_softc *sc; 223 int rid; 224 int err; 225 226 sc = device_get_softc(dev); 227 228 if (sc->platform != NULL) { 229 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 230 vtmmio_vq_intr, sc); 231 if (err == 0) { 232 /* Okay we have backend-specific interrupts */ 233 return (0); 234 } 235 } 236 237 rid = 0; 238 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 239 RF_ACTIVE); 240 if (!sc->res[1]) { 241 device_printf(dev, "Can't allocate interrupt\n"); 242 return (ENXIO); 243 } 244 245 if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE, 246 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 247 device_printf(dev, "Can't setup the interrupt\n"); 248 return (ENXIO); 249 } 250 251 return (0); 252 } 253 254 int 255 vtmmio_attach(device_t dev) 256 { 257 struct vtmmio_softc *sc; 258 device_t child; 259 int rid; 260 261 sc = device_get_softc(dev); 262 sc->dev = dev; 263 264 rid = 0; 265 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 266 RF_ACTIVE); 267 if (sc->res[0] == NULL) { 268 device_printf(dev, "Cannot allocate memory window.\n"); 269 return (ENXIO); 270 } 271 272 sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 273 274 vtmmio_reset(sc); 275 276 /* Tell the host we've noticed this device. */ 277 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 278 279 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 280 device_printf(dev, "Cannot create child device.\n"); 281 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 282 vtmmio_detach(dev); 283 return (ENOMEM); 284 } 285 286 sc->vtmmio_child_dev = child; 287 vtmmio_probe_and_attach_child(sc); 288 289 return (0); 290 } 291 292 static int 293 vtmmio_detach(device_t dev) 294 { 295 struct vtmmio_softc *sc; 296 device_t child; 297 int error; 298 299 sc = device_get_softc(dev); 300 301 if ((child = sc->vtmmio_child_dev) != NULL) { 302 error = device_delete_child(dev, child); 303 if (error) 304 return (error); 305 sc->vtmmio_child_dev = NULL; 306 } 307 308 vtmmio_reset(sc); 309 310 if (sc->res[0] != NULL) { 311 bus_release_resource(dev, SYS_RES_MEMORY, 0, 312 sc->res[0]); 313 sc->res[0] = NULL; 314 } 315 316 return (0); 317 } 318 319 static int 320 vtmmio_suspend(device_t dev) 321 { 322 323 return (bus_generic_suspend(dev)); 324 } 325 326 static int 327 vtmmio_resume(device_t dev) 328 { 329 330 return (bus_generic_resume(dev)); 331 } 332 333 static int 334 vtmmio_shutdown(device_t dev) 335 { 336 337 (void) bus_generic_shutdown(dev); 338 339 /* Forcibly stop the host device. */ 340 vtmmio_stop(dev); 341 342 return (0); 343 } 344 345 static void 346 vtmmio_driver_added(device_t dev, driver_t *driver) 347 { 348 struct vtmmio_softc *sc; 349 350 sc = device_get_softc(dev); 351 352 vtmmio_probe_and_attach_child(sc); 353 } 354 355 static void 356 vtmmio_child_detached(device_t dev, device_t child) 357 { 358 struct vtmmio_softc *sc; 359 360 sc = device_get_softc(dev); 361 362 vtmmio_reset(sc); 363 vtmmio_release_child_resources(sc); 364 } 365 366 static int 367 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 368 { 369 struct vtmmio_softc *sc; 370 371 sc = device_get_softc(dev); 372 373 if (sc->vtmmio_child_dev != child) 374 return (ENOENT); 375 376 switch (index) { 377 case VIRTIO_IVAR_DEVTYPE: 378 case VIRTIO_IVAR_SUBDEVICE: 379 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 380 break; 381 case VIRTIO_IVAR_VENDOR: 382 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 383 break; 384 case VIRTIO_IVAR_SUBVENDOR: 385 case VIRTIO_IVAR_DEVICE: 386 /* 387 * Dummy value for fields not present in this bus. Used by 388 * bus-agnostic virtio_child_pnpinfo. 389 */ 390 *result = 0; 391 break; 392 case VIRTIO_IVAR_MODERN: 393 /* 394 * There are several modern (aka MMIO v2) spec compliance 395 * issues with this driver, but keep the status quo. 396 */ 397 *result = sc->vtmmio_version > 1; 398 break; 399 default: 400 return (ENOENT); 401 } 402 403 return (0); 404 } 405 406 static int 407 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 408 { 409 struct vtmmio_softc *sc; 410 411 sc = device_get_softc(dev); 412 413 if (sc->vtmmio_child_dev != child) 414 return (ENOENT); 415 416 switch (index) { 417 case VIRTIO_IVAR_FEATURE_DESC: 418 sc->vtmmio_child_feat_desc = (void *) value; 419 break; 420 default: 421 return (ENOENT); 422 } 423 424 return (0); 425 } 426 427 static uint64_t 428 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 429 { 430 struct vtmmio_softc *sc; 431 uint64_t host_features, features; 432 433 sc = device_get_softc(dev); 434 435 if (sc->vtmmio_version > 1) { 436 child_features |= VIRTIO_F_VERSION_1; 437 } 438 439 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); 440 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 441 host_features <<= 32; 442 443 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 444 host_features |= vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 445 446 vtmmio_describe_features(sc, "host", host_features); 447 448 /* 449 * Limit negotiated features to what the driver, virtqueue, and 450 * host all support. 451 */ 452 features = host_features & child_features; 453 features = virtio_filter_transport_features(features); 454 sc->vtmmio_features = features; 455 456 vtmmio_describe_features(sc, "negotiated", features); 457 458 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 1); 459 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features >> 32); 460 461 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES_SEL, 0); 462 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 463 464 return (features); 465 } 466 467 static int 468 vtmmio_finalize_features(device_t dev) 469 { 470 struct vtmmio_softc *sc; 471 uint8_t status; 472 473 sc = device_get_softc(dev); 474 475 if (sc->vtmmio_version > 1) { 476 /* 477 * Must re-read the status after setting it to verify the 478 * negotiated features were accepted by the device. 479 */ 480 vtmmio_set_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); 481 482 status = vtmmio_get_status(dev); 483 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) { 484 device_printf(dev, "desired features were not accepted\n"); 485 return (ENOTSUP); 486 } 487 } 488 489 return (0); 490 } 491 492 static int 493 vtmmio_with_feature(device_t dev, uint64_t feature) 494 { 495 struct vtmmio_softc *sc; 496 497 sc = device_get_softc(dev); 498 499 return ((sc->vtmmio_features & feature) != 0); 500 } 501 502 static void 503 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, 504 uint32_t size) 505 { 506 vm_paddr_t paddr; 507 508 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 509 510 if (sc->vtmmio_version == 1) { 511 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 512 VIRTIO_MMIO_VRING_ALIGN); 513 paddr = virtqueue_paddr(vq); 514 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 515 paddr >> PAGE_SHIFT); 516 } else { 517 paddr = virtqueue_desc_paddr(vq); 518 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW, 519 paddr); 520 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH, 521 ((uint64_t)paddr) >> 32); 522 523 paddr = virtqueue_avail_paddr(vq); 524 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 525 paddr); 526 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 527 ((uint64_t)paddr) >> 32); 528 529 paddr = virtqueue_used_paddr(vq); 530 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW, 531 paddr); 532 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH, 533 ((uint64_t)paddr) >> 32); 534 535 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1); 536 } 537 } 538 539 static int 540 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 541 struct vq_alloc_info *vq_info) 542 { 543 struct vtmmio_virtqueue *vqx; 544 struct vq_alloc_info *info; 545 struct vtmmio_softc *sc; 546 struct virtqueue *vq; 547 uint32_t size; 548 int idx, error; 549 550 sc = device_get_softc(dev); 551 552 if (sc->vtmmio_nvqs != 0) 553 return (EALREADY); 554 if (nvqs <= 0) 555 return (EINVAL); 556 557 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 558 M_DEVBUF, M_NOWAIT | M_ZERO); 559 if (sc->vtmmio_vqs == NULL) 560 return (ENOMEM); 561 562 if (sc->vtmmio_version == 1) { 563 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 564 (1 << PAGE_SHIFT)); 565 } 566 567 for (idx = 0; idx < nvqs; idx++) { 568 vqx = &sc->vtmmio_vqs[idx]; 569 info = &vq_info[idx]; 570 571 vtmmio_select_virtqueue(sc, idx); 572 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 573 574 error = virtqueue_alloc(dev, idx, size, 575 VIRTIO_MMIO_QUEUE_NOTIFY, VIRTIO_MMIO_VRING_ALIGN, 576 ~(vm_paddr_t)0, info, &vq); 577 if (error) { 578 device_printf(dev, 579 "cannot allocate virtqueue %d: %d\n", 580 idx, error); 581 break; 582 } 583 584 vtmmio_set_virtqueue(sc, vq, size); 585 586 vqx->vtv_vq = *info->vqai_vq = vq; 587 vqx->vtv_no_intr = info->vqai_intr == NULL; 588 589 sc->vtmmio_nvqs++; 590 } 591 592 if (error) 593 vtmmio_free_virtqueues(sc); 594 595 return (error); 596 } 597 598 static void 599 vtmmio_stop(device_t dev) 600 { 601 602 vtmmio_reset(device_get_softc(dev)); 603 } 604 605 static void 606 vtmmio_poll(device_t dev) 607 { 608 struct vtmmio_softc *sc; 609 610 sc = device_get_softc(dev); 611 612 if (sc->platform != NULL) 613 VIRTIO_MMIO_POLL(sc->platform); 614 } 615 616 static int 617 vtmmio_reinit(device_t dev, uint64_t features) 618 { 619 struct vtmmio_softc *sc; 620 int idx, error; 621 622 sc = device_get_softc(dev); 623 624 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 625 vtmmio_stop(dev); 626 627 /* 628 * Quickly drive the status through ACK and DRIVER. The device 629 * does not become usable again until vtmmio_reinit_complete(). 630 */ 631 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 632 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 633 634 /* 635 * TODO: Check that features are not added as to what was 636 * originally negotiated. 637 */ 638 vtmmio_negotiate_features(dev, features); 639 error = vtmmio_finalize_features(dev); 640 if (error) { 641 device_printf(dev, "cannot finalize features during reinit\n"); 642 return (error); 643 } 644 645 if (sc->vtmmio_version == 1) { 646 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 647 (1 << PAGE_SHIFT)); 648 } 649 650 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 651 error = vtmmio_reinit_virtqueue(sc, idx); 652 if (error) 653 return (error); 654 } 655 656 return (0); 657 } 658 659 static void 660 vtmmio_reinit_complete(device_t dev) 661 { 662 663 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 664 } 665 666 static void 667 vtmmio_notify_virtqueue(device_t dev, uint16_t queue, bus_size_t offset) 668 { 669 struct vtmmio_softc *sc; 670 671 sc = device_get_softc(dev); 672 MPASS(offset == VIRTIO_MMIO_QUEUE_NOTIFY); 673 674 vtmmio_write_config_4(sc, offset, queue); 675 } 676 677 static int 678 vtmmio_config_generation(device_t dev) 679 { 680 struct vtmmio_softc *sc; 681 uint32_t gen; 682 683 sc = device_get_softc(dev); 684 685 if (sc->vtmmio_version > 1) 686 gen = vtmmio_read_config_4(sc, VIRTIO_MMIO_CONFIG_GENERATION); 687 else 688 gen = 0; 689 690 return (gen); 691 } 692 693 static uint8_t 694 vtmmio_get_status(device_t dev) 695 { 696 struct vtmmio_softc *sc; 697 698 sc = device_get_softc(dev); 699 700 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 701 } 702 703 static void 704 vtmmio_set_status(device_t dev, uint8_t status) 705 { 706 struct vtmmio_softc *sc; 707 708 sc = device_get_softc(dev); 709 710 if (status != VIRTIO_CONFIG_STATUS_RESET) 711 status |= vtmmio_get_status(dev); 712 713 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 714 } 715 716 static void 717 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 718 void *dst, int length) 719 { 720 struct vtmmio_softc *sc; 721 bus_size_t off; 722 uint8_t *d; 723 int size; 724 725 sc = device_get_softc(dev); 726 off = VIRTIO_MMIO_CONFIG + offset; 727 728 /* 729 * The non-legacy MMIO specification adds the following restriction: 730 * 731 * 4.2.2.2: For the device-specific configuration space, the driver 732 * MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide 733 * and aligned accesses for 16 bit wide fields and 32 bit wide and 734 * aligned accesses for 32 and 64 bit wide fields. 735 * 736 * The endianness also varies between non-legacy and legacy: 737 * 738 * 2.4: Note: The device configuration space uses the little-endian 739 * format for multi-byte fields. 740 * 741 * 2.4.3: Note that for legacy interfaces, device configuration space 742 * is generally the guest’s native endian, rather than PCI’s 743 * little-endian. The correct endian-ness is documented for each 744 * device. 745 */ 746 if (sc->vtmmio_version > 1) { 747 switch (length) { 748 case 1: 749 *(uint8_t *)dst = vtmmio_read_config_1(sc, off); 750 break; 751 case 2: 752 *(uint16_t *)dst = 753 le16toh(vtmmio_read_config_2(sc, off)); 754 break; 755 case 4: 756 *(uint32_t *)dst = 757 le32toh(vtmmio_read_config_4(sc, off)); 758 break; 759 case 8: 760 *(uint64_t *)dst = vtmmio_read_dev_config_8(sc, off); 761 break; 762 default: 763 panic("%s: invalid length %d\n", __func__, length); 764 } 765 766 return; 767 } 768 769 for (d = dst; length > 0; d += size, off += size, length -= size) { 770 #ifdef ALLOW_WORD_ALIGNED_ACCESS 771 if (length >= 4) { 772 size = 4; 773 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 774 } else if (length >= 2) { 775 size = 2; 776 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 777 } else 778 #endif 779 { 780 size = 1; 781 *d = vtmmio_read_config_1(sc, off); 782 } 783 } 784 } 785 786 static uint64_t 787 vtmmio_read_dev_config_8(struct vtmmio_softc *sc, bus_size_t off) 788 { 789 device_t dev; 790 int gen; 791 uint32_t val0, val1; 792 793 dev = sc->dev; 794 795 do { 796 gen = vtmmio_config_generation(dev); 797 val0 = le32toh(vtmmio_read_config_4(sc, off)); 798 val1 = le32toh(vtmmio_read_config_4(sc, off + 4)); 799 } while (gen != vtmmio_config_generation(dev)); 800 801 return (((uint64_t) val1 << 32) | val0); 802 } 803 804 static void 805 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 806 const void *src, int length) 807 { 808 struct vtmmio_softc *sc; 809 bus_size_t off; 810 const uint8_t *s; 811 int size; 812 813 sc = device_get_softc(dev); 814 off = VIRTIO_MMIO_CONFIG + offset; 815 816 /* 817 * The non-legacy MMIO specification adds size and alignment 818 * restrctions. It also changes the endianness from native-endian to 819 * little-endian. See vtmmio_read_dev_config. 820 */ 821 if (sc->vtmmio_version > 1) { 822 switch (length) { 823 case 1: 824 vtmmio_write_config_1(sc, off, *(const uint8_t *)src); 825 break; 826 case 2: 827 vtmmio_write_config_2(sc, off, 828 htole16(*(const uint16_t *)src)); 829 break; 830 case 4: 831 vtmmio_write_config_4(sc, off, 832 htole32(*(const uint32_t *)src)); 833 break; 834 case 8: 835 vtmmio_write_config_4(sc, off, 836 htole32(*(const uint64_t *)src)); 837 vtmmio_write_config_4(sc, off + 4, 838 htole32((*(const uint64_t *)src) >> 32)); 839 break; 840 default: 841 panic("%s: invalid length %d\n", __func__, length); 842 } 843 844 return; 845 } 846 847 for (s = src; length > 0; s += size, off += size, length -= size) { 848 #ifdef ALLOW_WORD_ALIGNED_ACCESS 849 if (length >= 4) { 850 size = 4; 851 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 852 } else if (length >= 2) { 853 size = 2; 854 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 855 } else 856 #endif 857 { 858 size = 1; 859 vtmmio_write_config_1(sc, off, *s); 860 } 861 } 862 } 863 864 static void 865 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 866 uint64_t features) 867 { 868 device_t dev, child; 869 870 dev = sc->dev; 871 child = sc->vtmmio_child_dev; 872 873 if (device_is_attached(child) || bootverbose == 0) 874 return; 875 876 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 877 } 878 879 static void 880 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 881 { 882 device_t dev, child; 883 884 dev = sc->dev; 885 child = sc->vtmmio_child_dev; 886 887 if (child == NULL) 888 return; 889 890 if (device_get_state(child) != DS_NOTPRESENT) { 891 return; 892 } 893 894 if (device_probe(child) != 0) { 895 return; 896 } 897 898 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 899 if (device_attach(child) != 0) { 900 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 901 vtmmio_reset(sc); 902 vtmmio_release_child_resources(sc); 903 /* Reset status for future attempt. */ 904 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 905 } else { 906 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 907 VIRTIO_ATTACH_COMPLETED(child); 908 } 909 } 910 911 static int 912 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 913 { 914 struct vtmmio_virtqueue *vqx; 915 struct virtqueue *vq; 916 int error; 917 uint16_t size; 918 919 vqx = &sc->vtmmio_vqs[idx]; 920 vq = vqx->vtv_vq; 921 922 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 923 924 vtmmio_select_virtqueue(sc, idx); 925 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 926 927 error = virtqueue_reinit(vq, size); 928 if (error) 929 return (error); 930 931 vtmmio_set_virtqueue(sc, vq, size); 932 933 return (0); 934 } 935 936 static void 937 vtmmio_free_interrupts(struct vtmmio_softc *sc) 938 { 939 940 if (sc->ih != NULL) 941 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 942 943 if (sc->res[1] != NULL) 944 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 945 } 946 947 static void 948 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 949 { 950 struct vtmmio_virtqueue *vqx; 951 int idx; 952 953 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 954 vqx = &sc->vtmmio_vqs[idx]; 955 956 vtmmio_select_virtqueue(sc, idx); 957 if (sc->vtmmio_version > 1) { 958 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0); 959 vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_READY); 960 } else 961 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 962 963 virtqueue_free(vqx->vtv_vq); 964 vqx->vtv_vq = NULL; 965 } 966 967 free(sc->vtmmio_vqs, M_DEVBUF); 968 sc->vtmmio_vqs = NULL; 969 sc->vtmmio_nvqs = 0; 970 } 971 972 static void 973 vtmmio_release_child_resources(struct vtmmio_softc *sc) 974 { 975 976 vtmmio_free_interrupts(sc); 977 vtmmio_free_virtqueues(sc); 978 } 979 980 static void 981 vtmmio_reset(struct vtmmio_softc *sc) 982 { 983 984 /* 985 * Setting the status to RESET sets the host device to 986 * the original, uninitialized state. 987 */ 988 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 989 } 990 991 static void 992 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 993 { 994 995 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 996 } 997 998 static void 999 vtmmio_vq_intr(void *arg) 1000 { 1001 struct vtmmio_virtqueue *vqx; 1002 struct vtmmio_softc *sc; 1003 struct virtqueue *vq; 1004 uint32_t status; 1005 int idx; 1006 1007 sc = arg; 1008 1009 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 1010 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 1011 1012 /* The config changed */ 1013 if (status & VIRTIO_MMIO_INT_CONFIG) 1014 if (sc->vtmmio_child_dev != NULL) 1015 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 1016 1017 /* Notify all virtqueues. */ 1018 if (status & VIRTIO_MMIO_INT_VRING) { 1019 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 1020 vqx = &sc->vtmmio_vqs[idx]; 1021 if (vqx->vtv_no_intr == 0) { 1022 vq = vqx->vtv_vq; 1023 virtqueue_intr(vq); 1024 } 1025 } 1026 } 1027 } 1028