1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by SRI International and the University of 7 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 8 * ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by Andrew Turner 11 * under sponsorship from the FreeBSD Foundation. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * VirtIO MMIO interface. 37 * This driver is heavily based on VirtIO PCI interface driver. 38 */ 39 40 #include <sys/cdefs.h> 41 __FBSDID("$FreeBSD$"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/kernel.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/rman.h> 50 #include <sys/endian.h> 51 52 #include <machine/bus.h> 53 #include <machine/resource.h> 54 55 #include <dev/virtio/virtio.h> 56 #include <dev/virtio/virtqueue.h> 57 #include <dev/virtio/mmio/virtio_mmio.h> 58 59 #include "virtio_mmio_if.h" 60 #include "virtio_bus_if.h" 61 #include "virtio_if.h" 62 63 #define PAGE_SHIFT 12 64 65 struct vtmmio_virtqueue { 66 struct virtqueue *vtv_vq; 67 int vtv_no_intr; 68 }; 69 70 static int vtmmio_detach(device_t); 71 static int vtmmio_suspend(device_t); 72 static int vtmmio_resume(device_t); 73 static int vtmmio_shutdown(device_t); 74 static void vtmmio_driver_added(device_t, driver_t *); 75 static void vtmmio_child_detached(device_t, device_t); 76 static int vtmmio_read_ivar(device_t, device_t, int, uintptr_t *); 77 static int vtmmio_write_ivar(device_t, device_t, int, uintptr_t); 78 static uint64_t vtmmio_negotiate_features(device_t, uint64_t); 79 static int vtmmio_with_feature(device_t, uint64_t); 80 static void vtmmio_set_virtqueue(struct vtmmio_softc *sc, 81 struct virtqueue *vq, uint32_t size); 82 static int vtmmio_alloc_virtqueues(device_t, int, int, 83 struct vq_alloc_info *); 84 static int vtmmio_setup_intr(device_t, enum intr_type); 85 static void vtmmio_stop(device_t); 86 static void vtmmio_poll(device_t); 87 static int vtmmio_reinit(device_t, uint64_t); 88 static void vtmmio_reinit_complete(device_t); 89 static void vtmmio_notify_virtqueue(device_t, uint16_t); 90 static uint8_t vtmmio_get_status(device_t); 91 static void vtmmio_set_status(device_t, uint8_t); 92 static void vtmmio_read_dev_config(device_t, bus_size_t, void *, int); 93 static void vtmmio_write_dev_config(device_t, bus_size_t, void *, int); 94 static void vtmmio_describe_features(struct vtmmio_softc *, const char *, 95 uint64_t); 96 static void vtmmio_probe_and_attach_child(struct vtmmio_softc *); 97 static int vtmmio_reinit_virtqueue(struct vtmmio_softc *, int); 98 static void vtmmio_free_interrupts(struct vtmmio_softc *); 99 static void vtmmio_free_virtqueues(struct vtmmio_softc *); 100 static void vtmmio_release_child_resources(struct vtmmio_softc *); 101 static void vtmmio_reset(struct vtmmio_softc *); 102 static void vtmmio_select_virtqueue(struct vtmmio_softc *, int); 103 static void vtmmio_vq_intr(void *); 104 105 /* 106 * I/O port read/write wrappers. 107 */ 108 #define vtmmio_write_config_1(sc, o, v) \ 109 do { \ 110 if (sc->platform != NULL) \ 111 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 112 bus_write_1((sc)->res[0], (o), (v)); \ 113 if (sc->platform != NULL) \ 114 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 115 } while (0) 116 #define vtmmio_write_config_2(sc, o, v) \ 117 do { \ 118 if (sc->platform != NULL) \ 119 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 120 bus_write_2((sc)->res[0], (o), (v)); \ 121 if (sc->platform != NULL) \ 122 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 123 } while (0) 124 #define vtmmio_write_config_4(sc, o, v) \ 125 do { \ 126 if (sc->platform != NULL) \ 127 VIRTIO_MMIO_PREWRITE(sc->platform, (o), (v)); \ 128 bus_write_4((sc)->res[0], (o), (v)); \ 129 if (sc->platform != NULL) \ 130 VIRTIO_MMIO_NOTE(sc->platform, (o), (v)); \ 131 } while (0) 132 133 #define vtmmio_read_config_1(sc, o) \ 134 bus_read_1((sc)->res[0], (o)) 135 #define vtmmio_read_config_2(sc, o) \ 136 bus_read_2((sc)->res[0], (o)) 137 #define vtmmio_read_config_4(sc, o) \ 138 bus_read_4((sc)->res[0], (o)) 139 140 static device_method_t vtmmio_methods[] = { 141 /* Device interface. */ 142 DEVMETHOD(device_attach, vtmmio_attach), 143 DEVMETHOD(device_detach, vtmmio_detach), 144 DEVMETHOD(device_suspend, vtmmio_suspend), 145 DEVMETHOD(device_resume, vtmmio_resume), 146 DEVMETHOD(device_shutdown, vtmmio_shutdown), 147 148 /* Bus interface. */ 149 DEVMETHOD(bus_driver_added, vtmmio_driver_added), 150 DEVMETHOD(bus_child_detached, vtmmio_child_detached), 151 DEVMETHOD(bus_child_pnpinfo_str, virtio_child_pnpinfo_str), 152 DEVMETHOD(bus_read_ivar, vtmmio_read_ivar), 153 DEVMETHOD(bus_write_ivar, vtmmio_write_ivar), 154 155 /* VirtIO bus interface. */ 156 DEVMETHOD(virtio_bus_negotiate_features, vtmmio_negotiate_features), 157 DEVMETHOD(virtio_bus_with_feature, vtmmio_with_feature), 158 DEVMETHOD(virtio_bus_alloc_virtqueues, vtmmio_alloc_virtqueues), 159 DEVMETHOD(virtio_bus_setup_intr, vtmmio_setup_intr), 160 DEVMETHOD(virtio_bus_stop, vtmmio_stop), 161 DEVMETHOD(virtio_bus_poll, vtmmio_poll), 162 DEVMETHOD(virtio_bus_reinit, vtmmio_reinit), 163 DEVMETHOD(virtio_bus_reinit_complete, vtmmio_reinit_complete), 164 DEVMETHOD(virtio_bus_notify_vq, vtmmio_notify_virtqueue), 165 DEVMETHOD(virtio_bus_read_device_config, vtmmio_read_dev_config), 166 DEVMETHOD(virtio_bus_write_device_config, vtmmio_write_dev_config), 167 168 DEVMETHOD_END 169 }; 170 171 DEFINE_CLASS_0(virtio_mmio, vtmmio_driver, vtmmio_methods, 172 sizeof(struct vtmmio_softc)); 173 174 MODULE_VERSION(virtio_mmio, 1); 175 176 static int 177 vtmmio_setup_intr(device_t dev, enum intr_type type) 178 { 179 struct vtmmio_softc *sc; 180 int rid; 181 int err; 182 183 sc = device_get_softc(dev); 184 185 if (sc->platform != NULL) { 186 err = VIRTIO_MMIO_SETUP_INTR(sc->platform, sc->dev, 187 vtmmio_vq_intr, sc); 188 if (err == 0) { 189 /* Okay we have backend-specific interrupts */ 190 return (0); 191 } 192 } 193 194 rid = 0; 195 sc->res[1] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 196 RF_ACTIVE); 197 if (!sc->res[1]) { 198 device_printf(dev, "Can't allocate interrupt\n"); 199 return (ENXIO); 200 } 201 202 if (bus_setup_intr(dev, sc->res[1], type | INTR_MPSAFE, 203 NULL, vtmmio_vq_intr, sc, &sc->ih)) { 204 device_printf(dev, "Can't setup the interrupt\n"); 205 return (ENXIO); 206 } 207 208 return (0); 209 } 210 211 int 212 vtmmio_attach(device_t dev) 213 { 214 struct vtmmio_softc *sc; 215 device_t child; 216 int rid; 217 218 sc = device_get_softc(dev); 219 sc->dev = dev; 220 221 rid = 0; 222 sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 223 RF_ACTIVE); 224 if (!sc->res[0]) { 225 device_printf(dev, "Cannot allocate memory window.\n"); 226 return (ENXIO); 227 } 228 229 sc->vtmmio_version = vtmmio_read_config_4(sc, VIRTIO_MMIO_VERSION); 230 if (sc->vtmmio_version < 1 || sc->vtmmio_version > 2) { 231 device_printf(dev, "Unsupported version: %x\n", 232 sc->vtmmio_version); 233 bus_release_resource(dev, SYS_RES_MEMORY, 0, 234 sc->res[0]); 235 sc->res[0] = NULL; 236 return (ENXIO); 237 } 238 239 vtmmio_reset(sc); 240 241 /* Tell the host we've noticed this device. */ 242 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 243 244 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 245 device_printf(dev, "Cannot create child device.\n"); 246 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 247 vtmmio_detach(dev); 248 return (ENOMEM); 249 } 250 251 sc->vtmmio_child_dev = child; 252 vtmmio_probe_and_attach_child(sc); 253 254 return (0); 255 } 256 257 static int 258 vtmmio_detach(device_t dev) 259 { 260 struct vtmmio_softc *sc; 261 device_t child; 262 int error; 263 264 sc = device_get_softc(dev); 265 266 if ((child = sc->vtmmio_child_dev) != NULL) { 267 error = device_delete_child(dev, child); 268 if (error) 269 return (error); 270 sc->vtmmio_child_dev = NULL; 271 } 272 273 vtmmio_reset(sc); 274 275 if (sc->res[0] != NULL) { 276 bus_release_resource(dev, SYS_RES_MEMORY, 0, 277 sc->res[0]); 278 sc->res[0] = NULL; 279 } 280 281 return (0); 282 } 283 284 static int 285 vtmmio_suspend(device_t dev) 286 { 287 288 return (bus_generic_suspend(dev)); 289 } 290 291 static int 292 vtmmio_resume(device_t dev) 293 { 294 295 return (bus_generic_resume(dev)); 296 } 297 298 static int 299 vtmmio_shutdown(device_t dev) 300 { 301 302 (void) bus_generic_shutdown(dev); 303 304 /* Forcibly stop the host device. */ 305 vtmmio_stop(dev); 306 307 return (0); 308 } 309 310 static void 311 vtmmio_driver_added(device_t dev, driver_t *driver) 312 { 313 struct vtmmio_softc *sc; 314 315 sc = device_get_softc(dev); 316 317 vtmmio_probe_and_attach_child(sc); 318 } 319 320 static void 321 vtmmio_child_detached(device_t dev, device_t child) 322 { 323 struct vtmmio_softc *sc; 324 325 sc = device_get_softc(dev); 326 327 vtmmio_reset(sc); 328 vtmmio_release_child_resources(sc); 329 } 330 331 static int 332 vtmmio_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 333 { 334 struct vtmmio_softc *sc; 335 336 sc = device_get_softc(dev); 337 338 if (sc->vtmmio_child_dev != child) 339 return (ENOENT); 340 341 switch (index) { 342 case VIRTIO_IVAR_DEVTYPE: 343 case VIRTIO_IVAR_SUBDEVICE: 344 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_DEVICE_ID); 345 break; 346 case VIRTIO_IVAR_VENDOR: 347 *result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID); 348 break; 349 case VIRTIO_IVAR_SUBVENDOR: 350 case VIRTIO_IVAR_DEVICE: 351 /* 352 * Dummy value for fields not present in this bus. Used by 353 * bus-agnostic virtio_child_pnpinfo_str. 354 */ 355 *result = 0; 356 break; 357 default: 358 return (ENOENT); 359 } 360 361 return (0); 362 } 363 364 static int 365 vtmmio_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 366 { 367 struct vtmmio_softc *sc; 368 369 sc = device_get_softc(dev); 370 371 if (sc->vtmmio_child_dev != child) 372 return (ENOENT); 373 374 switch (index) { 375 case VIRTIO_IVAR_FEATURE_DESC: 376 sc->vtmmio_child_feat_desc = (void *) value; 377 break; 378 default: 379 return (ENOENT); 380 } 381 382 return (0); 383 } 384 385 static uint64_t 386 vtmmio_negotiate_features(device_t dev, uint64_t child_features) 387 { 388 struct vtmmio_softc *sc; 389 uint64_t host_features, features; 390 391 sc = device_get_softc(dev); 392 393 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); 394 host_features = vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 395 host_features <<= 32; 396 397 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 398 host_features |= vtmmio_read_config_4(sc, VIRTIO_MMIO_HOST_FEATURES); 399 400 vtmmio_describe_features(sc, "host", host_features); 401 402 /* 403 * Limit negotiated features to what the driver, virtqueue, and 404 * host all support. 405 */ 406 features = host_features & child_features; 407 features = virtqueue_filter_features(features); 408 sc->vtmmio_features = features; 409 410 vtmmio_describe_features(sc, "negotiated", features); 411 412 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 1); 413 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features >> 32); 414 415 vtmmio_write_config_4(sc, VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 416 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_FEATURES, features); 417 418 return (features); 419 } 420 421 static int 422 vtmmio_with_feature(device_t dev, uint64_t feature) 423 { 424 struct vtmmio_softc *sc; 425 426 sc = device_get_softc(dev); 427 428 return ((sc->vtmmio_features & feature) != 0); 429 } 430 431 static void 432 vtmmio_set_virtqueue(struct vtmmio_softc *sc, struct virtqueue *vq, 433 uint32_t size) 434 { 435 vm_paddr_t paddr; 436 437 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NUM, size); 438 #if 0 439 device_printf(dev, "virtqueue paddr 0x%08lx\n", 440 (uint64_t)paddr); 441 #endif 442 if (sc->vtmmio_version == 1) { 443 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_ALIGN, 444 VIRTIO_MMIO_VRING_ALIGN); 445 paddr = virtqueue_paddr(vq); 446 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 447 paddr >> PAGE_SHIFT); 448 } else { 449 paddr = virtqueue_desc_paddr(vq); 450 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_LOW, 451 paddr); 452 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_DESC_HIGH, 453 ((uint64_t)paddr) >> 32); 454 455 paddr = virtqueue_avail_paddr(vq); 456 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 457 paddr); 458 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 459 ((uint64_t)paddr) >> 32); 460 461 paddr = virtqueue_used_paddr(vq); 462 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_LOW, 463 paddr); 464 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_USED_HIGH, 465 ((uint64_t)paddr) >> 32); 466 467 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 1); 468 } 469 } 470 471 static int 472 vtmmio_alloc_virtqueues(device_t dev, int flags, int nvqs, 473 struct vq_alloc_info *vq_info) 474 { 475 struct vtmmio_virtqueue *vqx; 476 struct vq_alloc_info *info; 477 struct vtmmio_softc *sc; 478 struct virtqueue *vq; 479 uint32_t size; 480 int idx, error; 481 482 sc = device_get_softc(dev); 483 484 if (sc->vtmmio_nvqs != 0) 485 return (EALREADY); 486 if (nvqs <= 0) 487 return (EINVAL); 488 489 sc->vtmmio_vqs = malloc(nvqs * sizeof(struct vtmmio_virtqueue), 490 M_DEVBUF, M_NOWAIT | M_ZERO); 491 if (sc->vtmmio_vqs == NULL) 492 return (ENOMEM); 493 494 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 495 (1 << PAGE_SHIFT)); 496 497 for (idx = 0; idx < nvqs; idx++) { 498 vqx = &sc->vtmmio_vqs[idx]; 499 info = &vq_info[idx]; 500 501 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 502 503 vtmmio_select_virtqueue(sc, idx); 504 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 505 506 error = virtqueue_alloc(dev, idx, size, 507 VIRTIO_MMIO_VRING_ALIGN, ~(vm_paddr_t)0, info, &vq); 508 if (error) { 509 device_printf(dev, 510 "cannot allocate virtqueue %d: %d\n", 511 idx, error); 512 break; 513 } 514 515 vtmmio_set_virtqueue(sc, vq, size); 516 517 vqx->vtv_vq = *info->vqai_vq = vq; 518 vqx->vtv_no_intr = info->vqai_intr == NULL; 519 520 sc->vtmmio_nvqs++; 521 } 522 523 if (error) 524 vtmmio_free_virtqueues(sc); 525 526 return (error); 527 } 528 529 static void 530 vtmmio_stop(device_t dev) 531 { 532 533 vtmmio_reset(device_get_softc(dev)); 534 } 535 536 static void 537 vtmmio_poll(device_t dev) 538 { 539 struct vtmmio_softc *sc; 540 541 sc = device_get_softc(dev); 542 543 if (sc->platform != NULL) 544 VIRTIO_MMIO_POLL(sc->platform); 545 } 546 547 static int 548 vtmmio_reinit(device_t dev, uint64_t features) 549 { 550 struct vtmmio_softc *sc; 551 int idx, error; 552 553 sc = device_get_softc(dev); 554 555 if (vtmmio_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 556 vtmmio_stop(dev); 557 558 /* 559 * Quickly drive the status through ACK and DRIVER. The device 560 * does not become usable again until vtmmio_reinit_complete(). 561 */ 562 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 563 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 564 565 vtmmio_negotiate_features(dev, features); 566 567 vtmmio_write_config_4(sc, VIRTIO_MMIO_GUEST_PAGE_SIZE, 568 (1 << PAGE_SHIFT)); 569 570 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 571 error = vtmmio_reinit_virtqueue(sc, idx); 572 if (error) 573 return (error); 574 } 575 576 return (0); 577 } 578 579 static void 580 vtmmio_reinit_complete(device_t dev) 581 { 582 583 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 584 } 585 586 static void 587 vtmmio_notify_virtqueue(device_t dev, uint16_t queue) 588 { 589 struct vtmmio_softc *sc; 590 591 sc = device_get_softc(dev); 592 593 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_NOTIFY, queue); 594 } 595 596 static uint8_t 597 vtmmio_get_status(device_t dev) 598 { 599 struct vtmmio_softc *sc; 600 601 sc = device_get_softc(dev); 602 603 return (vtmmio_read_config_4(sc, VIRTIO_MMIO_STATUS)); 604 } 605 606 static void 607 vtmmio_set_status(device_t dev, uint8_t status) 608 { 609 struct vtmmio_softc *sc; 610 611 sc = device_get_softc(dev); 612 613 if (status != VIRTIO_CONFIG_STATUS_RESET) 614 status |= vtmmio_get_status(dev); 615 616 vtmmio_write_config_4(sc, VIRTIO_MMIO_STATUS, status); 617 } 618 619 static void 620 vtmmio_read_dev_config(device_t dev, bus_size_t offset, 621 void *dst, int length) 622 { 623 struct vtmmio_softc *sc; 624 bus_size_t off; 625 uint8_t *d; 626 int size; 627 uint64_t low32, high32; 628 629 sc = device_get_softc(dev); 630 off = VIRTIO_MMIO_CONFIG + offset; 631 632 /* 633 * The non-legacy MMIO specification adds the following restriction: 634 * 635 * 4.2.2.2: For the device-specific configuration space, the driver 636 * MUST use 8 bit wide accesses for 8 bit wide fields, 16 bit wide 637 * and aligned accesses for 16 bit wide fields and 32 bit wide and 638 * aligned accesses for 32 and 64 bit wide fields. 639 * 640 * The endianness also varies between non-legacy and legacy: 641 * 642 * 2.4: Note: The device configuration space uses the little-endian 643 * format for multi-byte fields. 644 * 645 * 2.4.3: Note that for legacy interfaces, device configuration space 646 * is generally the guest’s native endian, rather than PCI’s 647 * little-endian. The correct endian-ness is documented for each 648 * device. 649 */ 650 if (sc->vtmmio_version > 1) { 651 switch (length) { 652 case 1: 653 *(uint8_t *)dst = vtmmio_read_config_1(sc, off); 654 break; 655 case 2: 656 *(uint16_t *)dst = 657 le16toh(vtmmio_read_config_2(sc, off)); 658 break; 659 case 4: 660 *(uint32_t *)dst = 661 le32toh(vtmmio_read_config_4(sc, off)); 662 break; 663 case 8: 664 low32 = le32toh(vtmmio_read_config_4(sc, off)); 665 high32 = le32toh(vtmmio_read_config_4(sc, off + 4)); 666 *(uint64_t *)dst = (high32 << 32) | low32; 667 break; 668 default: 669 panic("%s: invalid length %d\n", __func__, length); 670 } 671 672 return; 673 } 674 675 for (d = dst; length > 0; d += size, off += size, length -= size) { 676 #ifdef ALLOW_WORD_ALIGNED_ACCESS 677 if (length >= 4) { 678 size = 4; 679 *(uint32_t *)d = vtmmio_read_config_4(sc, off); 680 } else if (length >= 2) { 681 size = 2; 682 *(uint16_t *)d = vtmmio_read_config_2(sc, off); 683 } else 684 #endif 685 { 686 size = 1; 687 *d = vtmmio_read_config_1(sc, off); 688 } 689 } 690 } 691 692 static void 693 vtmmio_write_dev_config(device_t dev, bus_size_t offset, 694 void *src, int length) 695 { 696 struct vtmmio_softc *sc; 697 bus_size_t off; 698 uint8_t *s; 699 int size; 700 701 sc = device_get_softc(dev); 702 off = VIRTIO_MMIO_CONFIG + offset; 703 704 /* 705 * The non-legacy MMIO specification adds size and alignment 706 * restrctions. It also changes the endianness from native-endian to 707 * little-endian. See vtmmio_read_dev_config. 708 */ 709 if (sc->vtmmio_version > 1) { 710 switch (length) { 711 case 1: 712 vtmmio_write_config_1(sc, off, *(uint8_t *)src); 713 break; 714 case 2: 715 vtmmio_write_config_2(sc, off, 716 htole16(*(uint16_t *)src)); 717 break; 718 case 4: 719 vtmmio_write_config_4(sc, off, 720 htole32(*(uint32_t *)src)); 721 break; 722 case 8: 723 vtmmio_write_config_4(sc, off, 724 htole32(*(uint64_t *)src)); 725 vtmmio_write_config_4(sc, off + 4, 726 htole32((*(uint64_t *)src) >> 32)); 727 break; 728 default: 729 panic("%s: invalid length %d\n", __func__, length); 730 } 731 732 return; 733 } 734 735 for (s = src; length > 0; s += size, off += size, length -= size) { 736 #ifdef ALLOW_WORD_ALIGNED_ACCESS 737 if (length >= 4) { 738 size = 4; 739 vtmmio_write_config_4(sc, off, *(uint32_t *)s); 740 } else if (length >= 2) { 741 size = 2; 742 vtmmio_write_config_2(sc, off, *(uint16_t *)s); 743 } else 744 #endif 745 { 746 size = 1; 747 vtmmio_write_config_1(sc, off, *s); 748 } 749 } 750 } 751 752 static void 753 vtmmio_describe_features(struct vtmmio_softc *sc, const char *msg, 754 uint64_t features) 755 { 756 device_t dev, child; 757 758 dev = sc->dev; 759 child = sc->vtmmio_child_dev; 760 761 if (device_is_attached(child) || bootverbose == 0) 762 return; 763 764 virtio_describe(dev, msg, features, sc->vtmmio_child_feat_desc); 765 } 766 767 static void 768 vtmmio_probe_and_attach_child(struct vtmmio_softc *sc) 769 { 770 device_t dev, child; 771 772 dev = sc->dev; 773 child = sc->vtmmio_child_dev; 774 775 if (child == NULL) 776 return; 777 778 if (device_get_state(child) != DS_NOTPRESENT) { 779 return; 780 } 781 782 if (device_probe(child) != 0) { 783 return; 784 } 785 786 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 787 if (device_attach(child) != 0) { 788 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 789 vtmmio_reset(sc); 790 vtmmio_release_child_resources(sc); 791 /* Reset status for future attempt. */ 792 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 793 } else { 794 vtmmio_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 795 VIRTIO_ATTACH_COMPLETED(child); 796 } 797 } 798 799 static int 800 vtmmio_reinit_virtqueue(struct vtmmio_softc *sc, int idx) 801 { 802 struct vtmmio_virtqueue *vqx; 803 struct virtqueue *vq; 804 int error; 805 uint16_t size; 806 807 vqx = &sc->vtmmio_vqs[idx]; 808 vq = vqx->vtv_vq; 809 810 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); 811 812 vtmmio_select_virtqueue(sc, idx); 813 size = vtmmio_read_config_4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX); 814 815 error = virtqueue_reinit(vq, size); 816 if (error) 817 return (error); 818 819 vtmmio_set_virtqueue(sc, vq, size); 820 821 return (0); 822 } 823 824 static void 825 vtmmio_free_interrupts(struct vtmmio_softc *sc) 826 { 827 828 if (sc->ih != NULL) 829 bus_teardown_intr(sc->dev, sc->res[1], sc->ih); 830 831 if (sc->res[1] != NULL) 832 bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->res[1]); 833 } 834 835 static void 836 vtmmio_free_virtqueues(struct vtmmio_softc *sc) 837 { 838 struct vtmmio_virtqueue *vqx; 839 int idx; 840 841 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 842 vqx = &sc->vtmmio_vqs[idx]; 843 844 vtmmio_select_virtqueue(sc, idx); 845 if (sc->vtmmio_version == 1) 846 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_PFN, 0); 847 else 848 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_READY, 0); 849 850 virtqueue_free(vqx->vtv_vq); 851 vqx->vtv_vq = NULL; 852 } 853 854 free(sc->vtmmio_vqs, M_DEVBUF); 855 sc->vtmmio_vqs = NULL; 856 sc->vtmmio_nvqs = 0; 857 } 858 859 static void 860 vtmmio_release_child_resources(struct vtmmio_softc *sc) 861 { 862 863 vtmmio_free_interrupts(sc); 864 vtmmio_free_virtqueues(sc); 865 } 866 867 static void 868 vtmmio_reset(struct vtmmio_softc *sc) 869 { 870 871 /* 872 * Setting the status to RESET sets the host device to 873 * the original, uninitialized state. 874 */ 875 vtmmio_set_status(sc->dev, VIRTIO_CONFIG_STATUS_RESET); 876 } 877 878 static void 879 vtmmio_select_virtqueue(struct vtmmio_softc *sc, int idx) 880 { 881 882 vtmmio_write_config_4(sc, VIRTIO_MMIO_QUEUE_SEL, idx); 883 } 884 885 static void 886 vtmmio_vq_intr(void *arg) 887 { 888 struct vtmmio_virtqueue *vqx; 889 struct vtmmio_softc *sc; 890 struct virtqueue *vq; 891 uint32_t status; 892 int idx; 893 894 sc = arg; 895 896 status = vtmmio_read_config_4(sc, VIRTIO_MMIO_INTERRUPT_STATUS); 897 vtmmio_write_config_4(sc, VIRTIO_MMIO_INTERRUPT_ACK, status); 898 899 /* The config changed */ 900 if (status & VIRTIO_MMIO_INT_CONFIG) 901 if (sc->vtmmio_child_dev != NULL) 902 VIRTIO_CONFIG_CHANGE(sc->vtmmio_child_dev); 903 904 /* Notify all virtqueues. */ 905 if (status & VIRTIO_MMIO_INT_VRING) { 906 for (idx = 0; idx < sc->vtmmio_nvqs; idx++) { 907 vqx = &sc->vtmmio_vqs[idx]; 908 if (vqx->vtv_no_intr == 0) { 909 vq = vqx->vtv_vq; 910 virtqueue_intr(vq); 911 } 912 } 913 } 914 } 915