1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/bus.h> 33 #include <sys/kernel.h> 34 #include <sys/module.h> 35 #include <sys/malloc.h> 36 #include <sys/pcpu.h> 37 #include <sys/rman.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 41 #include <vm/vm.h> 42 #include <vm/pmap.h> 43 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 47 #include <machine/resource.h> 48 #include <machine/vmm.h> 49 #include <machine/pmap.h> 50 #include <machine/vmparam.h> 51 #include <machine/pci_cfgreg.h> 52 53 #include "ivhd_if.h" 54 #include "pcib_if.h" 55 56 #include "io/iommu.h" 57 #include "amdvi_priv.h" 58 59 SYSCTL_DECL(_hw_vmm); 60 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 61 NULL); 62 63 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 64 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 65 66 /* Print RID or device ID in PCI string format. */ 67 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 68 69 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count); 70 static void amdvi_print_dev_cap(struct amdvi_softc *softc); 71 72 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 73 74 extern device_t *ivhd_devs; 75 76 extern int ivhd_count; 77 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 78 0, NULL); 79 80 static int amdvi_enable_user = 0; 81 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 82 &amdvi_enable_user, 0, NULL); 83 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 84 85 #ifdef AMDVI_ATS_ENABLE 86 /* XXX: ATS is not tested. */ 87 static int amdvi_enable_iotlb = 1; 88 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 89 &amdvi_enable_iotlb, 0, NULL); 90 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 91 #endif 92 93 static int amdvi_host_ptp = 1; /* Use page tables for host. */ 94 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 95 &amdvi_host_ptp, 0, NULL); 96 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 97 98 /* Page table level used <= supported by h/w[v1=7]. */ 99 int amdvi_ptp_level = 4; 100 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 101 &amdvi_ptp_level, 0, NULL); 102 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 103 104 /* Disable fault event reporting. */ 105 static int amdvi_disable_io_fault = 0; 106 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 107 &amdvi_disable_io_fault, 0, NULL); 108 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 109 110 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 111 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 112 &amdvi_dom_id, 0, NULL); 113 /* 114 * Device table entry. 115 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 116 * = 256 * 2 * PAGE_SIZE. 117 */ 118 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 119 CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 120 CTASSERT(sizeof(amdvi_dte) == 0x200000); 121 122 static SLIST_HEAD (, amdvi_domain) dom_head; 123 124 static inline uint32_t 125 amdvi_pci_read(struct amdvi_softc *softc, int off) 126 { 127 128 return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid), 129 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 130 off, 4)); 131 } 132 133 #ifdef AMDVI_ATS_ENABLE 134 /* XXX: Should be in pci.c */ 135 /* 136 * Check if device has ATS capability and its enabled. 137 * If ATS is absent or disabled, return (-1), otherwise ATS 138 * queue length. 139 */ 140 static int 141 amdvi_find_ats_qlen(uint16_t devid) 142 { 143 device_t dev; 144 uint32_t off, cap; 145 int qlen = -1; 146 147 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 148 PCI_RID2FUNC(devid)); 149 150 if (!dev) { 151 return (-1); 152 } 153 #define PCIM_ATS_EN BIT(31) 154 155 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 156 cap = pci_read_config(dev, off + 4, 4); 157 qlen = (cap & 0x1F); 158 qlen = qlen ? qlen : 32; 159 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 160 RID2PCI_STR(devid), 161 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 162 qlen); 163 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 164 } 165 166 return (qlen); 167 } 168 169 /* 170 * Check if an endpoint device support device IOTLB or ATS. 171 */ 172 static inline bool 173 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 174 { 175 struct ivhd_dev_cfg *cfg; 176 int qlen, i; 177 bool pci_ats, ivhd_ats; 178 179 qlen = amdvi_find_ats_qlen(devid); 180 if (qlen < 0) 181 return (false); 182 183 KASSERT(softc, ("softc is NULL")); 184 cfg = softc->dev_cfg; 185 186 ivhd_ats = false; 187 for (i = 0; i < softc->dev_cfg_cnt; i++) { 188 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 189 ivhd_ats = cfg->enable_ats; 190 break; 191 } 192 cfg++; 193 } 194 195 pci_ats = (qlen < 0) ? false : true; 196 if (pci_ats != ivhd_ats) 197 device_printf(softc->dev, 198 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 199 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 200 201 /* Ignore IVRS setting and respect PCI setting. */ 202 return (pci_ats); 203 } 204 #endif 205 206 /* Enable IOTLB support for IOMMU if its supported. */ 207 static inline void 208 amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 209 { 210 #ifndef AMDVI_ATS_ENABLE 211 softc->iotlb = false; 212 #else 213 bool supported; 214 215 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 216 217 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 218 if (!supported) 219 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 220 221 if (supported && !amdvi_enable_iotlb) { 222 device_printf(softc->dev, "IOTLB disabled by user.\n"); 223 supported = false; 224 } 225 } else 226 supported = false; 227 228 softc->iotlb = supported; 229 230 #endif 231 } 232 233 static int 234 amdvi_init_cmd(struct amdvi_softc *softc) 235 { 236 struct amdvi_ctrl *ctrl = softc->ctrl; 237 238 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 239 softc->cmd_max = 1 << ctrl->cmd.len; 240 241 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 242 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 243 244 if ((uintptr_t)softc->cmd & PAGE_MASK) 245 panic("AMDVi: Command buffer not aligned on page boundary."); 246 247 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 248 /* 249 * XXX: Reset the h/w pointers in case IOMMU is restarting, 250 * h/w doesn't clear these pointers based on empirical data. 251 */ 252 ctrl->cmd_tail = 0; 253 ctrl->cmd_head = 0; 254 255 return (0); 256 } 257 258 /* 259 * Note: Update tail pointer after we have written the command since tail 260 * pointer update cause h/w to execute new commands, see section 3.3 261 * of AMD IOMMU spec ver 2.0. 262 */ 263 /* Get the command tail pointer w/o updating it. */ 264 static struct amdvi_cmd * 265 amdvi_get_cmd_tail(struct amdvi_softc *softc) 266 { 267 struct amdvi_ctrl *ctrl; 268 struct amdvi_cmd *tail; 269 270 KASSERT(softc, ("softc is NULL")); 271 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 272 273 ctrl = softc->ctrl; 274 KASSERT(ctrl != NULL, ("ctrl is NULL")); 275 276 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 277 ctrl->cmd_tail); 278 279 return (tail); 280 } 281 282 /* 283 * Update the command tail pointer which will start command execution. 284 */ 285 static void 286 amdvi_update_cmd_tail(struct amdvi_softc *softc) 287 { 288 struct amdvi_ctrl *ctrl; 289 int size; 290 291 size = sizeof(struct amdvi_cmd); 292 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 293 294 ctrl = softc->ctrl; 295 KASSERT(ctrl != NULL, ("ctrl is NULL")); 296 297 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 298 softc->total_cmd++; 299 300 #ifdef AMDVI_DEBUG_CMD 301 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 302 ctrl->cmd_tail, 303 ctrl->cmd_head); 304 #endif 305 306 } 307 308 /* 309 * Various commands supported by IOMMU. 310 */ 311 312 /* Completion wait command. */ 313 static void 314 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 315 { 316 struct amdvi_cmd *cmd; 317 uint64_t pa; 318 319 cmd = amdvi_get_cmd_tail(softc); 320 KASSERT(cmd != NULL, ("Cmd is NULL")); 321 322 pa = vtophys(&softc->cmp_data); 323 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 324 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; 325 cmd->word1 = (pa >> 32) & 0xFFFFF; 326 cmd->addr = data; 327 328 amdvi_update_cmd_tail(softc); 329 } 330 331 /* Invalidate device table entry. */ 332 static void 333 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 334 { 335 struct amdvi_cmd *cmd; 336 337 cmd = amdvi_get_cmd_tail(softc); 338 KASSERT(cmd != NULL, ("Cmd is NULL")); 339 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 340 cmd->word0 = devid; 341 amdvi_update_cmd_tail(softc); 342 #ifdef AMDVI_DEBUG_CMD 343 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 344 #endif 345 } 346 347 /* Invalidate IOMMU page, use for invalidation of domain. */ 348 static void 349 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 350 uint64_t addr, bool guest_nested, 351 bool pde, bool page) 352 { 353 struct amdvi_cmd *cmd; 354 355 cmd = amdvi_get_cmd_tail(softc); 356 KASSERT(cmd != NULL, ("Cmd is NULL")); 357 358 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 359 cmd->word1 = domain_id; 360 /* 361 * Invalidate all addresses for this domain. 362 */ 363 cmd->addr = addr; 364 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 365 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 366 367 amdvi_update_cmd_tail(softc); 368 } 369 370 #ifdef AMDVI_ATS_ENABLE 371 /* Invalidate device IOTLB. */ 372 static void 373 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 374 { 375 struct amdvi_cmd *cmd; 376 int qlen; 377 378 if (!softc->iotlb) 379 return; 380 381 qlen = amdvi_find_ats_qlen(devid); 382 if (qlen < 0) { 383 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 384 qlen, RID2PCI_STR(devid)); 385 } 386 cmd = amdvi_get_cmd_tail(softc); 387 KASSERT(cmd != NULL, ("Cmd is NULL")); 388 389 #ifdef AMDVI_DEBUG_CMD 390 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 391 " Qlen:%d\n", devid, qlen); 392 #endif 393 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 394 cmd->word0 = devid; 395 cmd->word1 = qlen; 396 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 397 AMDVI_INVD_IOTLB_S; 398 amdvi_update_cmd_tail(softc); 399 } 400 #endif 401 402 #ifdef notyet /* For Interrupt Remap. */ 403 static void 404 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 405 uint16_t devid) 406 { 407 struct amdvi_cmd *cmd; 408 409 cmd = amdvi_get_cmd_tail(softc); 410 KASSERT(cmd != NULL, ("Cmd is NULL")); 411 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 412 cmd->word0 = devid; 413 amdvi_update_cmd_tail(softc); 414 #ifdef AMDVI_DEBUG_CMD 415 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 416 #endif 417 } 418 #endif 419 420 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 421 static void 422 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 423 { 424 struct amdvi_cmd *cmd __diagused; 425 426 cmd = amdvi_get_cmd_tail(softc); 427 KASSERT(cmd != NULL, ("Cmd is NULL")); 428 429 /* 430 * See section 3.3.3 of IOMMU spec rev 2.0, software note 431 * for invalidating domain. 432 */ 433 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 434 false, true, true); 435 436 #ifdef AMDVI_DEBUG_CMD 437 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 438 439 #endif 440 } 441 442 static bool 443 amdvi_cmp_wait(struct amdvi_softc *softc) 444 { 445 #ifdef AMDVI_DEBUG_CMD 446 struct amdvi_ctrl *ctrl = softc->ctrl; 447 #endif 448 const uint64_t VERIFY = 0xA5A5; 449 volatile uint64_t *read; 450 int i; 451 bool status; 452 453 read = &softc->cmp_data; 454 *read = 0; 455 amdvi_cmd_cmp(softc, VERIFY); 456 /* Wait for h/w to update completion data. */ 457 for (i = 0; i < 100 && (*read != VERIFY); i++) { 458 DELAY(1000); /* 1 ms */ 459 } 460 status = (VERIFY == softc->cmp_data) ? true : false; 461 462 #ifdef AMDVI_DEBUG_CMD 463 if (status) 464 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 465 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 466 ctrl->cmd_head, loop); 467 #endif 468 return (status); 469 } 470 471 static void 472 amdvi_wait(struct amdvi_softc *softc) 473 { 474 struct amdvi_ctrl *ctrl; 475 int i; 476 477 KASSERT(softc, ("softc is NULL")); 478 479 ctrl = softc->ctrl; 480 KASSERT(ctrl != NULL, ("ctrl is NULL")); 481 /* Don't wait if h/w is not enabled. */ 482 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 483 return; 484 485 for (i = 0; i < 10; i++) { 486 if (amdvi_cmp_wait(softc)) 487 return; 488 } 489 490 device_printf(softc->dev, "Error: completion failed" 491 " tail:0x%x, head:0x%x.\n", 492 ctrl->cmd_tail, ctrl->cmd_head); 493 /* Dump the last command. */ 494 amdvi_dump_cmds(softc, 1); 495 } 496 497 static void 498 amdvi_dump_cmds(struct amdvi_softc *softc, int count) 499 { 500 struct amdvi_ctrl *ctrl; 501 struct amdvi_cmd *cmd; 502 int off, i; 503 504 ctrl = softc->ctrl; 505 device_printf(softc->dev, "Dump last %d command(s):\n", count); 506 /* 507 * If h/w is stuck in completion, it is the previous command, 508 * start dumping from previous command onward. 509 */ 510 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 511 softc->cmd_max); 512 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { 513 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 514 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 515 " 0x%x 0x%lx\n", i, off, cmd->opcode, 516 cmd->word0, cmd->word1, cmd->addr); 517 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max); 518 } 519 } 520 521 static int 522 amdvi_init_event(struct amdvi_softc *softc) 523 { 524 struct amdvi_ctrl *ctrl; 525 526 ctrl = softc->ctrl; 527 ctrl->event.len = 8; 528 softc->event_max = 1 << ctrl->event.len; 529 softc->event = malloc(sizeof(struct amdvi_event) * 530 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 531 if ((uintptr_t)softc->event & PAGE_MASK) { 532 device_printf(softc->dev, "Event buffer not aligned on page."); 533 return (false); 534 } 535 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 536 537 /* Reset the pointers. */ 538 ctrl->evt_head = 0; 539 ctrl->evt_tail = 0; 540 541 return (0); 542 } 543 544 static inline void 545 amdvi_decode_evt_flag(uint16_t flag) 546 { 547 548 flag &= AMDVI_EVENT_FLAG_MASK; 549 printf(" 0x%b]\n", flag, 550 "\020" 551 "\001GN" 552 "\002NX" 553 "\003US" 554 "\004I" 555 "\005PR" 556 "\006RW" 557 "\007PE" 558 "\010RZ" 559 "\011TR" 560 ); 561 } 562 563 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 564 static inline void 565 amdvi_decode_evt_flag_type(uint8_t type) 566 { 567 568 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 569 case 0: 570 printf("RSVD\n"); 571 break; 572 case 1: 573 printf("Master Abort\n"); 574 break; 575 case 2: 576 printf("Target Abort\n"); 577 break; 578 case 3: 579 printf("Data Err\n"); 580 break; 581 default: 582 break; 583 } 584 } 585 586 static void 587 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 588 uint16_t flag) 589 { 590 591 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 592 " Addr:0x%lx", 593 devid, domid, addr); 594 amdvi_decode_evt_flag(flag); 595 } 596 597 static void 598 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 599 uint16_t flag) 600 { 601 602 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 603 " Addr:0x%lx", 604 devid, domid, addr); 605 amdvi_decode_evt_flag(flag); 606 } 607 608 static void 609 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 610 uint64_t addr, uint16_t flag) 611 { 612 613 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 614 " Addr:0x%lx", devid, domid, addr); 615 amdvi_decode_evt_flag(flag); 616 amdvi_decode_evt_flag_type(flag); 617 } 618 619 static void 620 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 621 uint16_t flag) 622 { 623 624 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 625 " Addr:0x%lx", devid, domid, addr); 626 amdvi_decode_evt_flag(flag); 627 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 628 } 629 630 static void 631 amdvi_decode_evt(struct amdvi_event *evt) 632 { 633 struct amdvi_cmd *cmd; 634 635 switch (evt->opcode) { 636 case AMDVI_EVENT_INVALID_DTE: 637 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 638 evt->addr, evt->flag); 639 break; 640 641 case AMDVI_EVENT_PFAULT: 642 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 643 evt->addr, evt->flag); 644 break; 645 646 case AMDVI_EVENT_DTE_HW_ERROR: 647 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 648 evt->addr, evt->flag); 649 break; 650 651 case AMDVI_EVENT_PAGE_HW_ERROR: 652 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 653 evt->addr, evt->flag); 654 break; 655 656 case AMDVI_EVENT_ILLEGAL_CMD: 657 /* FALL THROUGH */ 658 case AMDVI_EVENT_CMD_HW_ERROR: 659 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 660 "ILLEGAL CMD" : "CMD HW ERR"); 661 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 662 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 663 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 664 break; 665 666 case AMDVI_EVENT_IOTLB_TIMEOUT: 667 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 668 evt->devid, evt->addr); 669 break; 670 671 case AMDVI_EVENT_INVALID_DTE_REQ: 672 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 673 evt->devid, evt->addr, evt->flag >> 9, 674 (evt->flag >> 8) & 1); 675 break; 676 677 case AMDVI_EVENT_INVALID_PPR_REQ: 678 case AMDVI_EVENT_COUNTER_ZERO: 679 printf("AMD-Vi: v2 events.\n"); 680 break; 681 682 default: 683 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 684 } 685 } 686 687 static void 688 amdvi_print_events(struct amdvi_softc *softc) 689 { 690 struct amdvi_ctrl *ctrl; 691 struct amdvi_event *event; 692 int i, size; 693 694 ctrl = softc->ctrl; 695 size = sizeof(struct amdvi_event); 696 for (i = 0; i < softc->event_max; i++) { 697 event = &softc->event[ctrl->evt_head / size]; 698 if (!event->opcode) 699 break; 700 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 701 i, ctrl->evt_head, ctrl->evt_tail); 702 amdvi_decode_evt(event); 703 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 704 softc->event_max); 705 } 706 } 707 708 static int 709 amdvi_init_dte(struct amdvi_softc *softc) 710 { 711 struct amdvi_ctrl *ctrl; 712 713 ctrl = softc->ctrl; 714 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 715 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 716 717 return (0); 718 } 719 720 /* 721 * Not all capabilities of IOMMU are available in ACPI IVHD flag 722 * or EFR entry, read directly from device. 723 */ 724 static int 725 amdvi_print_pci_cap(device_t dev) 726 { 727 struct amdvi_softc *softc; 728 uint32_t off, cap; 729 730 softc = device_get_softc(dev); 731 off = softc->cap_off; 732 733 /* 734 * Section 3.7.1 of IOMMU sepc rev 2.0. 735 * Read capability from device. 736 */ 737 cap = amdvi_pci_read(softc, off); 738 739 /* Make sure capability type[18:16] is 3. */ 740 KASSERT((((cap >> 16) & 0x7) == 0x3), 741 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 742 743 softc->pci_cap = cap >> 24; 744 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 745 cap, off, softc->pci_cap, 746 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 747 748 return (0); 749 } 750 751 static void 752 amdvi_event_intr(void *arg) 753 { 754 struct amdvi_softc *softc; 755 struct amdvi_ctrl *ctrl; 756 757 softc = (struct amdvi_softc *)arg; 758 ctrl = softc->ctrl; 759 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 760 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 761 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 762 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 763 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 764 765 amdvi_print_events(softc); 766 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 767 } 768 769 static void 770 amdvi_free_evt_intr_res(device_t dev) 771 { 772 773 struct amdvi_softc *softc; 774 device_t mmio_dev; 775 776 softc = device_get_softc(dev); 777 mmio_dev = softc->pci_dev; 778 779 IVHD_TEARDOWN_INTR(mmio_dev); 780 } 781 782 static bool 783 amdvi_alloc_intr_resources(struct amdvi_softc *softc) 784 { 785 struct amdvi_ctrl *ctrl; 786 device_t dev, mmio_dev; 787 int err; 788 789 dev = softc->dev; 790 mmio_dev = softc->pci_dev; 791 792 /* Clear interrupt status bits. */ 793 ctrl = softc->ctrl; 794 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 795 796 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault"); 797 if (err) 798 device_printf(dev, "Interrupt setup failed on %s\n", 799 device_get_nameunit(mmio_dev)); 800 return (err); 801 } 802 803 static void 804 amdvi_print_dev_cap(struct amdvi_softc *softc) 805 { 806 struct ivhd_dev_cfg *cfg; 807 int i; 808 809 cfg = softc->dev_cfg; 810 for (i = 0; i < softc->dev_cfg_cnt; i++) { 811 device_printf(softc->dev, "device [0x%x - 0x%x] " 812 "config:%b%s\n", cfg->start_id, cfg->end_id, 813 cfg->data, 814 "\020\001INIT\002ExtInt\003NMI" 815 "\007LINT0\010LINT1", 816 cfg->enable_ats ? "ATS enabled" : ""); 817 cfg++; 818 } 819 } 820 821 static int 822 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 823 { 824 struct amdvi_softc *softc; 825 int result, type, error = 0; 826 827 softc = (struct amdvi_softc *)arg1; 828 type = arg2; 829 830 switch (type) { 831 case 0: 832 result = softc->ctrl->cmd_head; 833 error = sysctl_handle_int(oidp, &result, 0, 834 req); 835 break; 836 case 1: 837 result = softc->ctrl->cmd_tail; 838 error = sysctl_handle_int(oidp, &result, 0, 839 req); 840 break; 841 case 2: 842 result = softc->ctrl->evt_head; 843 error = sysctl_handle_int(oidp, &result, 0, 844 req); 845 break; 846 case 3: 847 result = softc->ctrl->evt_tail; 848 error = sysctl_handle_int(oidp, &result, 0, 849 req); 850 break; 851 852 default: 853 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 854 } 855 856 return (error); 857 } 858 859 static void 860 amdvi_add_sysctl(struct amdvi_softc *softc) 861 { 862 struct sysctl_oid_list *child; 863 struct sysctl_ctx_list *ctx; 864 device_t dev; 865 866 dev = softc->dev; 867 ctx = device_get_sysctl_ctx(dev); 868 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 869 870 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 871 &softc->event_intr_cnt, "Event interrupt count"); 872 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 873 &softc->total_cmd, "Command submitted count"); 874 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 875 &softc->pci_rid, 0, "IOMMU RID"); 876 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 877 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 878 amdvi_handle_sysctl, "IU", "Command head"); 879 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 880 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 881 amdvi_handle_sysctl, "IU", "Command tail"); 882 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 883 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2, 884 amdvi_handle_sysctl, "IU", "Command head"); 885 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 886 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3, 887 amdvi_handle_sysctl, "IU", "Command tail"); 888 } 889 890 int 891 amdvi_setup_hw(struct amdvi_softc *softc) 892 { 893 device_t dev; 894 int status; 895 896 dev = softc->dev; 897 898 amdvi_hw_enable_iotlb(softc); 899 900 amdvi_print_dev_cap(softc); 901 902 if ((status = amdvi_print_pci_cap(dev)) != 0) { 903 device_printf(dev, "PCI capability.\n"); 904 return (status); 905 } 906 if ((status = amdvi_init_cmd(softc)) != 0) { 907 device_printf(dev, "Couldn't configure command buffer.\n"); 908 return (status); 909 } 910 if ((status = amdvi_init_event(softc)) != 0) { 911 device_printf(dev, "Couldn't configure event buffer.\n"); 912 return (status); 913 } 914 if ((status = amdvi_init_dte(softc)) != 0) { 915 device_printf(dev, "Couldn't configure device table.\n"); 916 return (status); 917 } 918 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 919 return (status); 920 } 921 amdvi_add_sysctl(softc); 922 return (0); 923 } 924 925 int 926 amdvi_teardown_hw(struct amdvi_softc *softc) 927 { 928 device_t dev; 929 930 dev = softc->dev; 931 932 /* 933 * Called after disable, h/w is stopped by now, free all the resources. 934 */ 935 amdvi_free_evt_intr_res(dev); 936 937 if (softc->cmd) 938 free(softc->cmd, M_AMDVI); 939 940 if (softc->event) 941 free(softc->event, M_AMDVI); 942 943 return (0); 944 } 945 946 /*********** bhyve interfaces *********************/ 947 static int 948 amdvi_init(void) 949 { 950 if (!ivhd_count) { 951 return (EIO); 952 } 953 if (!amdvi_enable_user && ivhd_count) { 954 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 955 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 956 ivhd_count); 957 return (EINVAL); 958 } 959 return (0); 960 } 961 962 static void 963 amdvi_cleanup(void) 964 { 965 /* Nothing. */ 966 } 967 968 static uint16_t 969 amdvi_domainId(void) 970 { 971 972 /* 973 * If we hit maximum domain limit, rollover leaving host 974 * domain(0). 975 * XXX: make sure that this domain is not used. 976 */ 977 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 978 amdvi_dom_id = 1; 979 980 return ((uint16_t)amdvi_dom_id++); 981 } 982 983 static void 984 amdvi_do_inv_domain(uint16_t domain_id, bool create) 985 { 986 struct amdvi_softc *softc; 987 int i; 988 989 for (i = 0; i < ivhd_count; i++) { 990 softc = device_get_softc(ivhd_devs[i]); 991 KASSERT(softc, ("softc is NULL")); 992 /* 993 * If not present pages are cached, invalidate page after 994 * creating domain. 995 */ 996 #if 0 997 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 998 continue; 999 #endif 1000 amdvi_inv_domain(softc, domain_id); 1001 amdvi_wait(softc); 1002 } 1003 } 1004 1005 static void * 1006 amdvi_create_domain(vm_paddr_t maxaddr) 1007 { 1008 struct amdvi_domain *dom; 1009 1010 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1011 dom->id = amdvi_domainId(); 1012 //dom->maxaddr = maxaddr; 1013 #ifdef AMDVI_DEBUG_CMD 1014 printf("Created domain #%d\n", dom->id); 1015 #endif 1016 /* 1017 * Host domain(#0) don't create translation table. 1018 */ 1019 if (dom->id || amdvi_host_ptp) 1020 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1021 1022 dom->ptp_level = amdvi_ptp_level; 1023 1024 amdvi_do_inv_domain(dom->id, true); 1025 SLIST_INSERT_HEAD(&dom_head, dom, next); 1026 1027 return (dom); 1028 } 1029 1030 static void 1031 amdvi_free_ptp(uint64_t *ptp, int level) 1032 { 1033 int i; 1034 1035 if (level < 1) 1036 return; 1037 1038 for (i = 0; i < NPTEPG ; i++) { 1039 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1040 continue; 1041 /* XXX: Add super-page or PTE mapping > 4KB. */ 1042 #ifdef notyet 1043 /* Super-page mapping. */ 1044 if (AMDVI_PD_SUPER(ptp[i])) 1045 continue; 1046 #endif 1047 1048 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1049 & AMDVI_PT_MASK), level - 1); 1050 } 1051 1052 free(ptp, M_AMDVI); 1053 } 1054 1055 static void 1056 amdvi_destroy_domain(void *arg) 1057 { 1058 struct amdvi_domain *domain; 1059 1060 domain = (struct amdvi_domain *)arg; 1061 KASSERT(domain, ("domain is NULL")); 1062 #ifdef AMDVI_DEBUG_CMD 1063 printf("Destroying domain %d\n", domain->id); 1064 #endif 1065 if (domain->ptp) 1066 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1067 1068 amdvi_do_inv_domain(domain->id, false); 1069 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1070 free(domain, M_AMDVI); 1071 } 1072 1073 static uint64_t 1074 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1075 vm_paddr_t hpa, uint64_t pg_size, bool create) 1076 { 1077 uint64_t *page, pa; 1078 int shift, index; 1079 const int PT_SHIFT = 9; 1080 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1081 1082 if (!pg_size) 1083 return (0); 1084 1085 if (hpa & (pg_size - 1)) { 1086 printf("HPA is not size aligned.\n"); 1087 return (0); 1088 } 1089 if (gpa & (pg_size - 1)) { 1090 printf("HPA is not size aligned.\n"); 1091 return (0); 1092 } 1093 shift = PML4SHIFT; 1094 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1095 index = (gpa >> shift) & PT_INDEX_MASK; 1096 1097 if ((pt[index] == 0) && create) { 1098 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1099 pa = vtophys(page); 1100 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1101 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1102 } 1103 #ifdef AMDVI_DEBUG_PTE 1104 if ((gpa % 0x1000000) == 0) 1105 printf("[level%d, shift = %d]PTE:0x%lx\n", 1106 level, shift, pt[index]); 1107 #endif 1108 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1109 pa = PTE2PA(pt[index]); 1110 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1111 shift -= PT_SHIFT; 1112 level--; 1113 } 1114 1115 /* Leaf entry. */ 1116 index = (gpa >> shift) & PT_INDEX_MASK; 1117 1118 if (create) { 1119 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1120 } else 1121 pt[index] = 0; 1122 1123 #ifdef AMDVI_DEBUG_PTE 1124 if ((gpa % 0x1000000) == 0) 1125 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1126 level, shift, pt[index]); 1127 #endif 1128 return (1ULL << shift); 1129 } 1130 1131 static uint64_t 1132 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1133 vm_paddr_t hpa, uint64_t size, bool create) 1134 { 1135 uint64_t mapped, *ptp, len; 1136 int level; 1137 1138 KASSERT(domain, ("domain is NULL")); 1139 level = domain->ptp_level; 1140 KASSERT(level, ("Page table level is 0")); 1141 1142 ptp = domain->ptp; 1143 KASSERT(ptp, ("PTP is NULL")); 1144 mapped = 0; 1145 while (mapped < size) { 1146 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1147 PAGE_SIZE, create); 1148 if (!len) { 1149 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1150 hpa, gpa); 1151 return (0); 1152 } 1153 mapped += len; 1154 } 1155 1156 return (mapped); 1157 } 1158 1159 static uint64_t 1160 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1161 uint64_t len) 1162 { 1163 struct amdvi_domain *domain; 1164 1165 domain = (struct amdvi_domain *)arg; 1166 1167 if (domain->id && !domain->ptp) { 1168 printf("ptp is NULL"); 1169 return (-1); 1170 } 1171 1172 /* 1173 * If host domain is created w/o page table, skip IOMMU page 1174 * table set-up. 1175 */ 1176 if (domain->ptp) 1177 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1178 else 1179 return (len); 1180 } 1181 1182 static uint64_t 1183 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1184 { 1185 struct amdvi_domain *domain; 1186 1187 domain = (struct amdvi_domain *)arg; 1188 /* 1189 * If host domain is created w/o page table, skip IOMMU page 1190 * table set-up. 1191 */ 1192 if (domain->ptp) 1193 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1194 return 1195 (len); 1196 } 1197 1198 static struct amdvi_softc * 1199 amdvi_find_iommu(uint16_t devid) 1200 { 1201 struct amdvi_softc *softc; 1202 int i, j; 1203 1204 for (i = 0; i < ivhd_count; i++) { 1205 softc = device_get_softc(ivhd_devs[i]); 1206 for (j = 0; j < softc->dev_cfg_cnt; j++) 1207 if ((devid >= softc->dev_cfg[j].start_id) && 1208 (devid <= softc->dev_cfg[j].end_id)) 1209 return (softc); 1210 } 1211 1212 return (NULL); 1213 } 1214 1215 /* 1216 * Set-up device table entry. 1217 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1218 * be set concurrently, e.g. read and write bits. 1219 */ 1220 static void 1221 amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc, 1222 uint16_t devid, bool enable) 1223 { 1224 struct amdvi_dte* temp; 1225 1226 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1227 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1228 1229 temp = &amdvi_dte[devid]; 1230 1231 #ifdef AMDVI_ATS_ENABLE 1232 /* If IOMMU and device support IOTLB, enable it. */ 1233 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1234 temp->iotlb_enable = 1; 1235 #endif 1236 1237 /* Avoid duplicate I/O faults. */ 1238 temp->sup_second_io_fault = 1; 1239 temp->sup_all_io_fault = amdvi_disable_io_fault; 1240 1241 temp->dt_valid = 1; 1242 temp->domain_id = domain->id; 1243 1244 if (enable) { 1245 if (domain->ptp) { 1246 temp->pt_base = vtophys(domain->ptp) >> 12; 1247 temp->pt_level = amdvi_ptp_level; 1248 } 1249 /* 1250 * XXX: Page table valid[TV] bit must be set even if host domain 1251 * page tables are not enabled. 1252 */ 1253 temp->pt_valid = 1; 1254 temp->read_allow = 1; 1255 temp->write_allow = 1; 1256 } 1257 } 1258 1259 static void 1260 amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid) 1261 { 1262 KASSERT(softc, ("softc is NULL")); 1263 1264 amdvi_cmd_inv_dte(softc, devid); 1265 #ifdef AMDVI_ATS_ENABLE 1266 if (amdvi_dev_support_iotlb(softc, devid)) 1267 amdvi_cmd_inv_iotlb(softc, devid); 1268 #endif 1269 amdvi_wait(softc); 1270 } 1271 1272 static void 1273 amdvi_add_device(void *arg, uint16_t devid) 1274 { 1275 struct amdvi_domain *domain; 1276 struct amdvi_softc *softc; 1277 1278 domain = (struct amdvi_domain *)arg; 1279 KASSERT(domain != NULL, ("domain is NULL")); 1280 #ifdef AMDVI_DEBUG_CMD 1281 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1282 RID2PCI_STR(devid), domain->id); 1283 #endif 1284 softc = amdvi_find_iommu(devid); 1285 if (softc == NULL) 1286 return; 1287 amdvi_set_dte(domain, softc, devid, true); 1288 amdvi_inv_device(softc, devid); 1289 } 1290 1291 static void 1292 amdvi_remove_device(void *arg, uint16_t devid) 1293 { 1294 struct amdvi_domain *domain; 1295 struct amdvi_softc *softc; 1296 1297 domain = (struct amdvi_domain *)arg; 1298 #ifdef AMDVI_DEBUG_CMD 1299 printf("Remove device(0x%x) from domain:%d\n", 1300 devid, domain->id); 1301 #endif 1302 softc = amdvi_find_iommu(devid); 1303 if (softc == NULL) 1304 return; 1305 amdvi_set_dte(domain, softc, devid, false); 1306 amdvi_inv_device(softc, devid); 1307 } 1308 1309 static void 1310 amdvi_enable(void) 1311 { 1312 struct amdvi_ctrl *ctrl; 1313 struct amdvi_softc *softc; 1314 uint64_t val; 1315 int i; 1316 1317 for (i = 0; i < ivhd_count; i++) { 1318 softc = device_get_softc(ivhd_devs[i]); 1319 KASSERT(softc, ("softc is NULL\n")); 1320 ctrl = softc->ctrl; 1321 KASSERT(ctrl, ("ctrl is NULL\n")); 1322 1323 val = ( AMDVI_CTRL_EN | 1324 AMDVI_CTRL_CMD | 1325 AMDVI_CTRL_ELOG | 1326 AMDVI_CTRL_ELOGINT | 1327 AMDVI_CTRL_INV_TO_1S); 1328 1329 if (softc->ivhd_flag & IVHD_FLAG_COH) 1330 val |= AMDVI_CTRL_COH; 1331 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1332 val |= AMDVI_CTRL_HTT; 1333 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1334 val |= AMDVI_CTRL_RPPW; 1335 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1336 val |= AMDVI_CTRL_PPW; 1337 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1338 val |= AMDVI_CTRL_ISOC; 1339 1340 ctrl->control = val; 1341 } 1342 } 1343 1344 static void 1345 amdvi_disable(void) 1346 { 1347 struct amdvi_ctrl *ctrl; 1348 struct amdvi_softc *softc; 1349 int i; 1350 1351 for (i = 0; i < ivhd_count; i++) { 1352 softc = device_get_softc(ivhd_devs[i]); 1353 KASSERT(softc, ("softc is NULL\n")); 1354 ctrl = softc->ctrl; 1355 KASSERT(ctrl, ("ctrl is NULL\n")); 1356 1357 ctrl->control = 0; 1358 } 1359 } 1360 1361 static void 1362 amdvi_invalidate_tlb(void *arg) 1363 { 1364 struct amdvi_domain *domain; 1365 1366 domain = (struct amdvi_domain *)arg; 1367 KASSERT(domain, ("domain is NULL")); 1368 amdvi_do_inv_domain(domain->id, false); 1369 } 1370 1371 const struct iommu_ops iommu_ops_amd = { 1372 .init = amdvi_init, 1373 .cleanup = amdvi_cleanup, 1374 .enable = amdvi_enable, 1375 .disable = amdvi_disable, 1376 .create_domain = amdvi_create_domain, 1377 .destroy_domain = amdvi_destroy_domain, 1378 .create_mapping = amdvi_create_mapping, 1379 .remove_mapping = amdvi_remove_mapping, 1380 .add_device = amdvi_add_device, 1381 .remove_device = amdvi_remove_device, 1382 .invalidate_tlb = amdvi_invalidate_tlb 1383 }; 1384