1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/malloc.h> 38 #include <sys/pcpu.h> 39 #include <sys/rman.h> 40 #include <sys/smp.h> 41 #include <sys/sysctl.h> 42 43 #include <vm/vm.h> 44 #include <vm/pmap.h> 45 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pcireg.h> 48 49 #include <machine/resource.h> 50 #include <machine/vmm.h> 51 #include <machine/pmap.h> 52 #include <machine/vmparam.h> 53 #include <machine/pci_cfgreg.h> 54 55 #include "pcib_if.h" 56 57 #include "io/iommu.h" 58 #include "amdvi_priv.h" 59 60 SYSCTL_DECL(_hw_vmm); 61 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW, NULL, NULL); 62 63 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 64 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 65 66 /* Print RID or device ID in PCI string format. */ 67 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 68 69 static void amdvi_dump_cmds(struct amdvi_softc *softc); 70 static void amdvi_print_dev_cap(struct amdvi_softc *softc); 71 72 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 73 74 extern device_t *ivhd_devs; 75 76 extern int ivhd_count; 77 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 78 0, NULL); 79 80 static int amdvi_enable_user = 0; 81 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 82 &amdvi_enable_user, 0, NULL); 83 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 84 85 #ifdef AMDVI_ATS_ENABLE 86 /* XXX: ATS is not tested. */ 87 static int amdvi_enable_iotlb = 1; 88 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 89 &amdvi_enable_iotlb, 0, NULL); 90 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 91 #endif 92 93 static int amdvi_host_ptp = 1; /* Use page tables for host. */ 94 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 95 &amdvi_host_ptp, 0, NULL); 96 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 97 98 /* Page table level used <= supported by h/w[v1=7]. */ 99 static int amdvi_ptp_level = 4; 100 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 101 &amdvi_ptp_level, 0, NULL); 102 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 103 104 /* Disable fault event reporting. */ 105 static int amdvi_disable_io_fault = 0; 106 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 107 &amdvi_disable_io_fault, 0, NULL); 108 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 109 110 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 111 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 112 &amdvi_dom_id, 0, NULL); 113 /* 114 * Device table entry. 115 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 116 * = 256 * 2 * PAGE_SIZE. 117 */ 118 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 119 CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 120 CTASSERT(sizeof(amdvi_dte) == 0x200000); 121 122 static SLIST_HEAD (, amdvi_domain) dom_head; 123 124 static inline uint32_t 125 amdvi_pci_read(struct amdvi_softc *softc, int off) 126 { 127 128 return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid), 129 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 130 off, 4)); 131 } 132 133 #ifdef AMDVI_ATS_ENABLE 134 /* XXX: Should be in pci.c */ 135 /* 136 * Check if device has ATS capability and its enabled. 137 * If ATS is absent or disabled, return (-1), otherwise ATS 138 * queue length. 139 */ 140 static int 141 amdvi_find_ats_qlen(uint16_t devid) 142 { 143 device_t dev; 144 uint32_t off, cap; 145 int qlen = -1; 146 147 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 148 PCI_RID2FUNC(devid)); 149 150 if (!dev) { 151 return (-1); 152 } 153 #define PCIM_ATS_EN BIT(31) 154 155 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 156 cap = pci_read_config(dev, off + 4, 4); 157 qlen = (cap & 0x1F); 158 qlen = qlen ? qlen : 32; 159 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 160 RID2PCI_STR(devid), 161 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 162 qlen); 163 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 164 } 165 166 return (qlen); 167 } 168 169 /* 170 * Check if an endpoint device support device IOTLB or ATS. 171 */ 172 static inline bool 173 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 174 { 175 struct ivhd_dev_cfg *cfg; 176 int qlen, i; 177 bool pci_ats, ivhd_ats; 178 179 qlen = amdvi_find_ats_qlen(devid); 180 if (qlen < 0) 181 return (false); 182 183 KASSERT(softc, ("softc is NULL")); 184 cfg = softc->dev_cfg; 185 186 ivhd_ats = false; 187 for (i = 0; i < softc->dev_cfg_cnt; i++) { 188 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 189 ivhd_ats = cfg->enable_ats; 190 break; 191 } 192 cfg++; 193 } 194 195 pci_ats = (qlen < 0) ? false : true; 196 if (pci_ats != ivhd_ats) 197 device_printf(softc->dev, 198 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 199 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 200 201 /* Ignore IVRS setting and respect PCI setting. */ 202 return (pci_ats); 203 } 204 #endif 205 206 /* Enable IOTLB support for IOMMU if its supported. */ 207 static inline void 208 amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 209 { 210 #ifndef AMDVI_ATS_ENABLE 211 softc->iotlb = false; 212 #else 213 bool supported; 214 215 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 216 217 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 218 if (!supported) 219 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 220 221 if (supported && !amdvi_enable_iotlb) { 222 device_printf(softc->dev, "IOTLB disabled by user.\n"); 223 supported = false; 224 } 225 } else 226 supported = false; 227 228 softc->iotlb = supported; 229 230 #endif 231 } 232 233 static int 234 amdvi_init_cmd(struct amdvi_softc *softc) 235 { 236 struct amdvi_ctrl *ctrl = softc->ctrl; 237 238 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 239 softc->cmd_max = 1 << ctrl->cmd.len; 240 241 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 242 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 243 244 if ((uintptr_t)softc->cmd & PAGE_MASK) 245 panic("AMDVi: Command buffer not aligned on page boundary."); 246 247 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 248 /* 249 * XXX: Reset the h/w pointers in case IOMMU is restarting, 250 * h/w doesn't clear these pointers based on empirical data. 251 */ 252 ctrl->cmd_tail = 0; 253 ctrl->cmd_head = 0; 254 255 return (0); 256 } 257 258 /* 259 * Note: Update tail pointer after we have written the command since tail 260 * pointer update cause h/w to execute new commands, see section 3.3 261 * of AMD IOMMU spec ver 2.0. 262 */ 263 /* Get the command tail pointer w/o updating it. */ 264 static struct amdvi_cmd * 265 amdvi_get_cmd_tail(struct amdvi_softc *softc) 266 { 267 struct amdvi_ctrl *ctrl; 268 struct amdvi_cmd *tail; 269 270 KASSERT(softc, ("softc is NULL")); 271 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 272 273 ctrl = softc->ctrl; 274 KASSERT(ctrl != NULL, ("ctrl is NULL")); 275 276 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 277 ctrl->cmd_tail); 278 279 return (tail); 280 } 281 282 /* 283 * Update the command tail pointer which will start command execution. 284 */ 285 static void 286 amdvi_update_cmd_tail(struct amdvi_softc *softc) 287 { 288 struct amdvi_ctrl *ctrl; 289 int size; 290 291 size = sizeof(struct amdvi_cmd); 292 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 293 294 ctrl = softc->ctrl; 295 KASSERT(ctrl != NULL, ("ctrl is NULL")); 296 297 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 298 softc->total_cmd++; 299 300 #ifdef AMDVI_DEBUG_CMD 301 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 302 ctrl->cmd_tail, 303 ctrl->cmd_head); 304 #endif 305 306 } 307 308 /* 309 * Various commands supported by IOMMU. 310 */ 311 312 /* Completion wait command. */ 313 static void 314 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 315 { 316 struct amdvi_cmd *cmd; 317 uint64_t pa; 318 319 cmd = amdvi_get_cmd_tail(softc); 320 KASSERT(cmd != NULL, ("Cmd is NULL")); 321 322 pa = vtophys(&softc->cmp_data); 323 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 324 cmd->word0 = (pa & 0xFFFFFFF8) | 325 (AMDVI_CMP_WAIT_STORE); 326 //(AMDVI_CMP_WAIT_FLUSH | AMDVI_CMP_WAIT_STORE); 327 cmd->word1 = (pa >> 32) & 0xFFFFF; 328 cmd->addr = data; 329 330 amdvi_update_cmd_tail(softc); 331 } 332 333 /* Invalidate device table entry. */ 334 static void 335 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 336 { 337 struct amdvi_cmd *cmd; 338 339 cmd = amdvi_get_cmd_tail(softc); 340 KASSERT(cmd != NULL, ("Cmd is NULL")); 341 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 342 cmd->word0 = devid; 343 amdvi_update_cmd_tail(softc); 344 #ifdef AMDVI_DEBUG_CMD 345 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 346 #endif 347 } 348 349 /* Invalidate IOMMU page, use for invalidation of domain. */ 350 static void 351 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 352 uint64_t addr, bool guest_nested, 353 bool pde, bool page) 354 { 355 struct amdvi_cmd *cmd; 356 357 cmd = amdvi_get_cmd_tail(softc); 358 KASSERT(cmd != NULL, ("Cmd is NULL")); 359 360 361 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 362 cmd->word1 = domain_id; 363 /* 364 * Invalidate all addresses for this domain. 365 */ 366 cmd->addr = addr; 367 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 368 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 369 370 amdvi_update_cmd_tail(softc); 371 } 372 373 #ifdef AMDVI_ATS_ENABLE 374 /* Invalidate device IOTLB. */ 375 static void 376 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 377 { 378 struct amdvi_cmd *cmd; 379 int qlen; 380 381 if (!softc->iotlb) 382 return; 383 384 qlen = amdvi_find_ats_qlen(devid); 385 if (qlen < 0) { 386 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 387 qlen, RID2PCI_STR(devid)); 388 } 389 cmd = amdvi_get_cmd_tail(softc); 390 KASSERT(cmd != NULL, ("Cmd is NULL")); 391 392 #ifdef AMDVI_DEBUG_CMD 393 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 394 " Qlen:%d\n", devid, qlen); 395 #endif 396 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 397 cmd->word0 = devid; 398 cmd->word1 = qlen; 399 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 400 AMDVI_INVD_IOTLB_S; 401 amdvi_update_cmd_tail(softc); 402 } 403 #endif 404 405 #ifdef notyet /* For Interrupt Remap. */ 406 static void 407 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 408 uint16_t devid) 409 { 410 struct amdvi_cmd *cmd; 411 412 cmd = amdvi_get_cmd_tail(softc); 413 KASSERT(cmd != NULL, ("Cmd is NULL")); 414 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 415 cmd->word0 = devid; 416 amdvi_update_cmd_tail(softc); 417 #ifdef AMDVI_DEBUG_CMD 418 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 419 #endif 420 } 421 #endif 422 423 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 424 static void 425 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 426 { 427 struct amdvi_cmd *cmd; 428 429 cmd = amdvi_get_cmd_tail(softc); 430 KASSERT(cmd != NULL, ("Cmd is NULL")); 431 432 /* 433 * See section 3.3.3 of IOMMU spec rev 2.0, software note 434 * for invalidating domain. 435 */ 436 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 437 false, true, true); 438 439 #ifdef AMDVI_DEBUG_CMD 440 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 441 442 #endif 443 } 444 445 static bool 446 amdvi_cmp_wait(struct amdvi_softc *softc) 447 { 448 struct amdvi_ctrl *ctrl; 449 const uint64_t VERIFY = 0xA5A5; 450 volatile uint64_t *read; 451 int i; 452 bool status; 453 454 ctrl = softc->ctrl; 455 read = &softc->cmp_data; 456 *read = 0; 457 amdvi_cmd_cmp(softc, VERIFY); 458 /* Wait for h/w to update completion data. */ 459 for (i = 0; i < 100 && (*read != VERIFY); i++) { 460 DELAY(1000); /* 1 ms */ 461 } 462 status = (VERIFY == softc->cmp_data) ? true : false; 463 464 #ifdef AMDVI_DEBUG_CMD 465 if (status) 466 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 467 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 468 ctrl->cmd_head, loop); 469 #endif 470 return (status); 471 } 472 473 static void 474 amdvi_wait(struct amdvi_softc *softc) 475 { 476 struct amdvi_ctrl *ctrl; 477 int i; 478 479 KASSERT(softc, ("softc is NULL")); 480 481 ctrl = softc->ctrl; 482 KASSERT(ctrl != NULL, ("ctrl is NULL")); 483 /* Don't wait if h/w is not enabled. */ 484 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 485 return; 486 487 for (i = 0; i < 10; i++) { 488 if (amdvi_cmp_wait(softc)) 489 return; 490 } 491 492 device_printf(softc->dev, "Error: completion failed" 493 " tail:0x%x, head:0x%x.\n", 494 ctrl->cmd_tail, ctrl->cmd_head); 495 amdvi_dump_cmds(softc); 496 } 497 498 static void 499 amdvi_dump_cmds(struct amdvi_softc *softc) 500 { 501 struct amdvi_ctrl *ctrl; 502 struct amdvi_cmd *cmd; 503 int off, i; 504 505 ctrl = softc->ctrl; 506 device_printf(softc->dev, "Dump all the commands:\n"); 507 /* 508 * If h/w is stuck in completion, it is the previous command, 509 * start dumping from previous command onward. 510 */ 511 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 512 softc->cmd_max); 513 for (i = 0; off != ctrl->cmd_tail && 514 i < softc->cmd_max; i++) { 515 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 516 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 517 " 0x%x 0x%lx\n", i, off, cmd->opcode, 518 cmd->word0, cmd->word1, cmd->addr); 519 off = (off + sizeof(struct amdvi_cmd)) % 520 (softc->cmd_max * sizeof(struct amdvi_cmd)); 521 } 522 } 523 524 static int 525 amdvi_init_event(struct amdvi_softc *softc) 526 { 527 struct amdvi_ctrl *ctrl; 528 529 ctrl = softc->ctrl; 530 ctrl->event.len = 8; 531 softc->event_max = 1 << ctrl->event.len; 532 softc->event = malloc(sizeof(struct amdvi_event) * 533 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 534 if ((uintptr_t)softc->event & PAGE_MASK) { 535 device_printf(softc->dev, "Event buffer not aligned on page."); 536 return (false); 537 } 538 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 539 540 /* Reset the pointers. */ 541 ctrl->evt_head = 0; 542 ctrl->evt_tail = 0; 543 544 return (0); 545 } 546 547 static inline void 548 amdvi_decode_evt_flag(uint16_t flag) 549 { 550 551 flag &= AMDVI_EVENT_FLAG_MASK; 552 printf(" 0x%b]\n", flag, 553 "\020" 554 "\001GN" 555 "\002NX" 556 "\003US" 557 "\004I" 558 "\005PR" 559 "\006RW" 560 "\007PE" 561 "\010RZ" 562 "\011TR" 563 ); 564 } 565 566 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 567 static inline void 568 amdvi_decode_evt_flag_type(uint8_t type) 569 { 570 571 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 572 case 0: 573 printf("RSVD\n"); 574 break; 575 case 1: 576 printf("Master Abort\n"); 577 break; 578 case 2: 579 printf("Target Abort\n"); 580 break; 581 case 3: 582 printf("Data Err\n"); 583 break; 584 default: 585 break; 586 } 587 } 588 589 static void 590 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 591 uint16_t flag) 592 { 593 594 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 595 " Addr:0x%lx", 596 devid, domid, addr); 597 amdvi_decode_evt_flag(flag); 598 } 599 600 static void 601 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 602 uint16_t flag) 603 { 604 605 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 606 " Addr:0x%lx", 607 devid, domid, addr); 608 amdvi_decode_evt_flag(flag); 609 } 610 611 static void 612 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 613 uint64_t addr, uint16_t flag) 614 { 615 616 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 617 " Addr:0x%lx", devid, domid, addr); 618 amdvi_decode_evt_flag(flag); 619 amdvi_decode_evt_flag_type(flag); 620 } 621 622 static void 623 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 624 uint16_t flag) 625 { 626 627 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 628 " Addr:0x%lx", devid, domid, addr); 629 amdvi_decode_evt_flag(flag); 630 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 631 } 632 633 static void 634 amdvi_decode_evt(struct amdvi_event *evt) 635 { 636 struct amdvi_cmd *cmd; 637 638 switch (evt->opcode) { 639 case AMDVI_EVENT_INVALID_DTE: 640 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 641 evt->addr, evt->flag); 642 break; 643 644 case AMDVI_EVENT_PFAULT: 645 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 646 evt->addr, evt->flag); 647 break; 648 649 case AMDVI_EVENT_DTE_HW_ERROR: 650 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 651 evt->addr, evt->flag); 652 break; 653 654 case AMDVI_EVENT_PAGE_HW_ERROR: 655 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 656 evt->addr, evt->flag); 657 break; 658 659 case AMDVI_EVENT_ILLEGAL_CMD: 660 /* FALL THROUGH */ 661 case AMDVI_EVENT_CMD_HW_ERROR: 662 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 663 "ILLEGAL CMD" : "CMD HW ERR"); 664 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 665 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 666 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 667 break; 668 669 case AMDVI_EVENT_IOTLB_TIMEOUT: 670 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 671 evt->devid, evt->addr); 672 break; 673 674 case AMDVI_EVENT_INVALID_DTE_REQ: 675 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 676 evt->devid, evt->addr, evt->flag >> 9, 677 (evt->flag >> 8) & 1); 678 break; 679 680 case AMDVI_EVENT_INVALID_PPR_REQ: 681 case AMDVI_EVENT_COUNTER_ZERO: 682 printf("AMD-Vi: v2 events.\n"); 683 break; 684 685 default: 686 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 687 } 688 } 689 690 static void 691 amdvi_print_events(struct amdvi_softc *softc) 692 { 693 struct amdvi_ctrl *ctrl; 694 struct amdvi_event *event; 695 int i, size; 696 697 ctrl = softc->ctrl; 698 size = sizeof(struct amdvi_event); 699 for (i = 0; i < softc->event_max; i++) { 700 event = &softc->event[ctrl->evt_head / size]; 701 if (!event->opcode) 702 break; 703 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 704 i, ctrl->evt_head, ctrl->evt_tail); 705 amdvi_decode_evt(event); 706 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 707 softc->event_max); 708 } 709 } 710 711 static int 712 amdvi_init_dte(struct amdvi_softc *softc) 713 { 714 struct amdvi_ctrl *ctrl; 715 716 ctrl = softc->ctrl; 717 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 718 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 719 720 return (0); 721 } 722 723 /* 724 * Not all capabilities of IOMMU are available in ACPI IVHD flag 725 * or EFR entry, read directly from device. 726 */ 727 static int 728 amdvi_print_pci_cap(device_t dev) 729 { 730 struct amdvi_softc *softc; 731 uint32_t off, cap; 732 733 734 softc = device_get_softc(dev); 735 off = softc->cap_off; 736 737 /* 738 * Section 3.7.1 of IOMMU sepc rev 2.0. 739 * Read capability from device. 740 */ 741 cap = amdvi_pci_read(softc, off); 742 743 /* Make sure capability type[18:16] is 3. */ 744 KASSERT((((cap >> 16) & 0x7) == 0x3), 745 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 746 747 softc->pci_cap = cap >> 24; 748 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 749 cap, off, softc->pci_cap, 750 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 751 752 return (0); 753 } 754 755 static void 756 amdvi_event_intr(void *arg) 757 { 758 struct amdvi_softc *softc; 759 struct amdvi_ctrl *ctrl; 760 761 softc = (struct amdvi_softc *)arg; 762 ctrl = softc->ctrl; 763 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 764 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 765 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 766 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 767 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 768 769 amdvi_print_events(softc); 770 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 771 } 772 773 static void 774 amdvi_free_evt_intr_res(device_t dev) 775 { 776 777 struct amdvi_softc *softc; 778 779 softc = device_get_softc(dev); 780 if (softc->event_tag != NULL) { 781 bus_teardown_intr(dev, softc->event_res, softc->event_tag); 782 } 783 if (softc->event_res != NULL) { 784 bus_release_resource(dev, SYS_RES_IRQ, softc->event_rid, 785 softc->event_res); 786 } 787 bus_delete_resource(dev, SYS_RES_IRQ, softc->event_rid); 788 PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)), 789 dev, 1, &softc->event_irq); 790 } 791 792 static bool 793 amdvi_alloc_intr_resources(struct amdvi_softc *softc) 794 { 795 struct amdvi_ctrl *ctrl; 796 device_t dev, pcib; 797 device_t mmio_dev; 798 uint64_t msi_addr; 799 uint32_t msi_data; 800 int err; 801 802 dev = softc->dev; 803 pcib = device_get_parent(device_get_parent(dev)); 804 mmio_dev = pci_find_bsf(PCI_RID2BUS(softc->pci_rid), 805 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid)); 806 if (device_is_attached(mmio_dev)) { 807 device_printf(dev, 808 "warning: IOMMU device is claimed by another driver %s\n", 809 device_get_driver(mmio_dev)->name); 810 } 811 812 softc->event_irq = -1; 813 softc->event_rid = 0; 814 815 /* 816 * Section 3.7.1 of IOMMU rev 2.0. With MSI, there is only one 817 * interrupt. XXX: Enable MSI/X support. 818 */ 819 err = PCIB_ALLOC_MSI(pcib, dev, 1, 1, &softc->event_irq); 820 if (err) { 821 device_printf(dev, 822 "Couldn't find event MSI IRQ resource.\n"); 823 return (ENOENT); 824 } 825 826 err = bus_set_resource(dev, SYS_RES_IRQ, softc->event_rid, 827 softc->event_irq, 1); 828 if (err) { 829 device_printf(dev, "Couldn't set event MSI resource.\n"); 830 return (ENXIO); 831 } 832 833 softc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 834 &softc->event_rid, RF_ACTIVE); 835 if (!softc->event_res) { 836 device_printf(dev, 837 "Unable to allocate event INTR resource.\n"); 838 return (ENOMEM); 839 } 840 841 if (bus_setup_intr(dev, softc->event_res, 842 INTR_TYPE_MISC | INTR_MPSAFE, NULL, amdvi_event_intr, 843 softc, &softc->event_tag)) { 844 device_printf(dev, "Fail to setup event intr\n"); 845 bus_release_resource(softc->dev, SYS_RES_IRQ, 846 softc->event_rid, softc->event_res); 847 softc->event_res = NULL; 848 return (ENXIO); 849 } 850 851 bus_describe_intr(dev, softc->event_res, softc->event_tag, 852 "fault"); 853 854 err = PCIB_MAP_MSI(pcib, dev, softc->event_irq, &msi_addr, 855 &msi_data); 856 if (err) { 857 device_printf(dev, 858 "Event interrupt config failed, err=%d.\n", 859 err); 860 amdvi_free_evt_intr_res(softc->dev); 861 return (err); 862 } 863 864 /* Clear interrupt status bits. */ 865 ctrl = softc->ctrl; 866 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 867 868 /* Now enable MSI interrupt. */ 869 pci_enable_msi(mmio_dev, msi_addr, msi_data); 870 return (0); 871 } 872 873 874 static void 875 amdvi_print_dev_cap(struct amdvi_softc *softc) 876 { 877 struct ivhd_dev_cfg *cfg; 878 int i; 879 880 cfg = softc->dev_cfg; 881 for (i = 0; i < softc->dev_cfg_cnt; i++) { 882 device_printf(softc->dev, "device [0x%x - 0x%x]" 883 "config:%b%s\n", cfg->start_id, cfg->end_id, 884 cfg->data, 885 "\020\001INIT\002ExtInt\003NMI" 886 "\007LINT0\008LINT1", 887 cfg->enable_ats ? "ATS enabled" : ""); 888 cfg++; 889 } 890 } 891 892 static int 893 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 894 { 895 struct amdvi_softc *softc; 896 int result, type, error = 0; 897 898 softc = (struct amdvi_softc *)arg1; 899 type = arg2; 900 901 switch (type) { 902 case 0: 903 result = softc->ctrl->cmd_head; 904 error = sysctl_handle_int(oidp, &result, 0, 905 req); 906 break; 907 case 1: 908 result = softc->ctrl->cmd_tail; 909 error = sysctl_handle_int(oidp, &result, 0, 910 req); 911 break; 912 case 2: 913 result = softc->ctrl->evt_head; 914 error = sysctl_handle_int(oidp, &result, 0, 915 req); 916 break; 917 case 3: 918 result = softc->ctrl->evt_tail; 919 error = sysctl_handle_int(oidp, &result, 0, 920 req); 921 break; 922 923 default: 924 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 925 } 926 927 return (error); 928 } 929 930 static void 931 amdvi_add_sysctl(struct amdvi_softc *softc) 932 { 933 struct sysctl_oid_list *child; 934 struct sysctl_ctx_list *ctx; 935 device_t dev; 936 937 dev = softc->dev; 938 ctx = device_get_sysctl_ctx(dev); 939 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 940 941 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 942 &softc->event_intr_cnt, "Event interrupt count"); 943 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 944 &softc->total_cmd, "Command submitted count"); 945 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 946 &softc->pci_rid, 0, "IOMMU RID"); 947 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "start_dev_rid", CTLFLAG_RD, 948 &softc->start_dev_rid, 0, "Start of device under this IOMMU"); 949 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "end_dev_rid", CTLFLAG_RD, 950 &softc->end_dev_rid, 0, "End of device under this IOMMU"); 951 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 952 CTLTYPE_UINT | CTLFLAG_RD, softc, 0, 953 amdvi_handle_sysctl, "IU", "Command head"); 954 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 955 CTLTYPE_UINT | CTLFLAG_RD, softc, 1, 956 amdvi_handle_sysctl, "IU", "Command tail"); 957 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 958 CTLTYPE_UINT | CTLFLAG_RD, softc, 2, 959 amdvi_handle_sysctl, "IU", "Command head"); 960 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 961 CTLTYPE_UINT | CTLFLAG_RD, softc, 3, 962 amdvi_handle_sysctl, "IU", "Command tail"); 963 } 964 965 int 966 amdvi_setup_hw(struct amdvi_softc *softc) 967 { 968 device_t dev; 969 int status; 970 971 dev = softc->dev; 972 973 amdvi_hw_enable_iotlb(softc); 974 975 amdvi_print_dev_cap(softc); 976 977 if ((status = amdvi_print_pci_cap(dev)) != 0) { 978 device_printf(dev, "PCI capability.\n"); 979 return (status); 980 } 981 if ((status = amdvi_init_cmd(softc)) != 0) { 982 device_printf(dev, "Couldn't configure command buffer.\n"); 983 return (status); 984 } 985 if ((status = amdvi_init_event(softc)) != 0) { 986 device_printf(dev, "Couldn't configure event buffer.\n"); 987 return (status); 988 } 989 if ((status = amdvi_init_dte(softc)) != 0) { 990 device_printf(dev, "Couldn't configure device table.\n"); 991 return (status); 992 } 993 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 994 return (status); 995 } 996 amdvi_add_sysctl(softc); 997 return (0); 998 } 999 1000 int 1001 amdvi_teardown_hw(struct amdvi_softc *softc) 1002 { 1003 device_t dev; 1004 1005 dev = softc->dev; 1006 1007 /* 1008 * Called after disable, h/w is stopped by now, free all the resources. 1009 */ 1010 amdvi_free_evt_intr_res(dev); 1011 1012 if (softc->cmd) 1013 free(softc->cmd, M_AMDVI); 1014 1015 if (softc->event) 1016 free(softc->event, M_AMDVI); 1017 1018 return (0); 1019 } 1020 1021 /*********** bhyve interfaces *********************/ 1022 static int 1023 amdvi_init(void) 1024 { 1025 if (!ivhd_count) { 1026 return (EIO); 1027 } 1028 if (!amdvi_enable_user && ivhd_count) { 1029 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 1030 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 1031 ivhd_count); 1032 return (EINVAL); 1033 } 1034 return (0); 1035 } 1036 1037 static void 1038 amdvi_cleanup(void) 1039 { 1040 /* Nothing. */ 1041 } 1042 1043 static uint16_t 1044 amdvi_domainId(void) 1045 { 1046 1047 /* 1048 * If we hit maximum domain limit, rollover leaving host 1049 * domain(0). 1050 * XXX: make sure that this domain is not used. 1051 */ 1052 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 1053 amdvi_dom_id = 1; 1054 1055 return ((uint16_t)amdvi_dom_id++); 1056 } 1057 1058 static void 1059 amdvi_do_inv_domain(uint16_t domain_id, bool create) 1060 { 1061 struct amdvi_softc *softc; 1062 int i; 1063 1064 for (i = 0; i < ivhd_count; i++) { 1065 softc = device_get_softc(ivhd_devs[i]); 1066 KASSERT(softc, ("softc is NULL")); 1067 /* 1068 * If not present pages are cached, invalidate page after 1069 * creating domain. 1070 */ 1071 #if 0 1072 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 1073 continue; 1074 #endif 1075 amdvi_inv_domain(softc, domain_id); 1076 amdvi_wait(softc); 1077 } 1078 } 1079 1080 static void * 1081 amdvi_create_domain(vm_paddr_t maxaddr) 1082 { 1083 struct amdvi_domain *dom; 1084 1085 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1086 dom->id = amdvi_domainId(); 1087 //dom->maxaddr = maxaddr; 1088 #ifdef AMDVI_DEBUG_CMD 1089 printf("Created domain #%d\n", dom->id); 1090 #endif 1091 /* 1092 * Host domain(#0) don't create translation table. 1093 */ 1094 if (dom->id || amdvi_host_ptp) 1095 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1096 1097 dom->ptp_level = amdvi_ptp_level; 1098 1099 amdvi_do_inv_domain(dom->id, true); 1100 SLIST_INSERT_HEAD(&dom_head, dom, next); 1101 1102 return (dom); 1103 } 1104 1105 static void 1106 amdvi_free_ptp(uint64_t *ptp, int level) 1107 { 1108 int i; 1109 1110 if (level < 1) 1111 return; 1112 1113 for (i = 0; i < NPTEPG ; i++) { 1114 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1115 continue; 1116 /* XXX: Add super-page or PTE mapping > 4KB. */ 1117 #ifdef notyet 1118 /* Super-page mapping. */ 1119 if (AMDVI_PD_SUPER(ptp[i])) 1120 continue; 1121 #endif 1122 1123 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1124 & AMDVI_PT_MASK), level - 1); 1125 1126 } 1127 1128 free(ptp, M_AMDVI); 1129 } 1130 1131 static void 1132 amdvi_destroy_domain(void *arg) 1133 { 1134 struct amdvi_domain *domain; 1135 1136 domain = (struct amdvi_domain *)arg; 1137 KASSERT(domain, ("domain is NULL")); 1138 #ifdef AMDVI_DEBUG_CMD 1139 printf("Destroying domain %d\n", domain->id); 1140 #endif 1141 if (domain->ptp) 1142 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1143 1144 amdvi_do_inv_domain(domain->id, false); 1145 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1146 free(domain, M_AMDVI); 1147 } 1148 1149 static uint64_t 1150 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1151 vm_paddr_t hpa, uint64_t pg_size, bool create) 1152 { 1153 uint64_t *page, pa; 1154 int shift, index; 1155 const int PT_SHIFT = 9; 1156 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1157 1158 if (!pg_size) 1159 return (0); 1160 1161 if (hpa & (pg_size - 1)) { 1162 printf("HPA is not size aligned.\n"); 1163 return (0); 1164 } 1165 if (gpa & (pg_size - 1)) { 1166 printf("HPA is not size aligned.\n"); 1167 return (0); 1168 } 1169 shift = PML4SHIFT; 1170 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1171 index = (gpa >> shift) & PT_INDEX_MASK; 1172 1173 if ((pt[index] == 0) && create) { 1174 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1175 pa = vtophys(page); 1176 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1177 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1178 } 1179 #ifdef AMDVI_DEBUG_PTE 1180 if ((gpa % 0x1000000) == 0) 1181 printf("[level%d, shift = %d]PTE:0x%lx\n", 1182 level, shift, pt[index]); 1183 #endif 1184 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1185 pa = PTE2PA(pt[index]); 1186 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1187 shift -= PT_SHIFT; 1188 level--; 1189 } 1190 1191 /* Leaf entry. */ 1192 index = (gpa >> shift) & PT_INDEX_MASK; 1193 1194 if (create) { 1195 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1196 } else 1197 pt[index] = 0; 1198 1199 #ifdef AMDVI_DEBUG_PTE 1200 if ((gpa % 0x1000000) == 0) 1201 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1202 level, shift, pt[index]); 1203 #endif 1204 return (1ULL << shift); 1205 } 1206 1207 static uint64_t 1208 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1209 vm_paddr_t hpa, uint64_t size, bool create) 1210 { 1211 uint64_t mapped, *ptp, len; 1212 int level; 1213 1214 KASSERT(domain, ("domain is NULL")); 1215 level = domain->ptp_level; 1216 KASSERT(level, ("Page table level is 0")); 1217 1218 ptp = domain->ptp; 1219 KASSERT(ptp, ("PTP is NULL")); 1220 mapped = 0; 1221 while (mapped < size) { 1222 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1223 PAGE_SIZE, create); 1224 if (!len) { 1225 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1226 hpa, gpa); 1227 return (0); 1228 } 1229 mapped += len; 1230 } 1231 1232 return (mapped); 1233 } 1234 1235 static uint64_t 1236 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1237 uint64_t len) 1238 { 1239 struct amdvi_domain *domain; 1240 1241 domain = (struct amdvi_domain *)arg; 1242 1243 if (domain->id && !domain->ptp) { 1244 printf("ptp is NULL"); 1245 return (-1); 1246 } 1247 1248 /* 1249 * If host domain is created w/o page table, skip IOMMU page 1250 * table set-up. 1251 */ 1252 if (domain->ptp) 1253 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1254 else 1255 return (len); 1256 } 1257 1258 static uint64_t 1259 amdvi_destroy_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1260 { 1261 struct amdvi_domain *domain; 1262 1263 domain = (struct amdvi_domain *)arg; 1264 /* 1265 * If host domain is created w/o page table, skip IOMMU page 1266 * table set-up. 1267 */ 1268 if (domain->ptp) 1269 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1270 return 1271 (len); 1272 } 1273 1274 static struct amdvi_softc * 1275 amdvi_find_iommu(uint16_t devid) 1276 { 1277 struct amdvi_softc *softc; 1278 int i; 1279 1280 for (i = 0; i < ivhd_count; i++) { 1281 softc = device_get_softc(ivhd_devs[i]); 1282 if ((devid >= softc->start_dev_rid) && 1283 (devid <= softc->end_dev_rid)) 1284 return (softc); 1285 } 1286 1287 /* 1288 * XXX: BIOS bug, device not in IVRS table, assume its from first IOMMU. 1289 */ 1290 printf("BIOS bug device(%d.%d.%d) doesn't have IVHD entry.\n", 1291 RID2PCI_STR(devid)); 1292 1293 return (device_get_softc(ivhd_devs[0])); 1294 } 1295 1296 /* 1297 * Set-up device table entry. 1298 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1299 * be set concurrently, e.g. read and write bits. 1300 */ 1301 static void 1302 amdvi_set_dte(struct amdvi_domain *domain, uint16_t devid, bool enable) 1303 { 1304 struct amdvi_softc *softc; 1305 struct amdvi_dte* temp; 1306 1307 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1308 1309 softc = amdvi_find_iommu(devid); 1310 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1311 1312 temp = &amdvi_dte[devid]; 1313 1314 #ifdef AMDVI_ATS_ENABLE 1315 /* If IOMMU and device support IOTLB, enable it. */ 1316 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1317 temp->iotlb_enable = 1; 1318 #endif 1319 1320 /* Avoid duplicate I/O faults. */ 1321 temp->sup_second_io_fault = 1; 1322 temp->sup_all_io_fault = amdvi_disable_io_fault; 1323 1324 temp->dt_valid = 1; 1325 temp->domain_id = domain->id; 1326 1327 if (enable) { 1328 if (domain->ptp) { 1329 temp->pt_base = vtophys(domain->ptp) >> 12; 1330 temp->pt_level = amdvi_ptp_level; 1331 } 1332 /* 1333 * XXX: Page table valid[TV] bit must be set even if host domain 1334 * page tables are not enabled. 1335 */ 1336 temp->pt_valid = 1; 1337 temp->read_allow = 1; 1338 temp->write_allow = 1; 1339 } 1340 } 1341 1342 static void 1343 amdvi_inv_device(uint16_t devid) 1344 { 1345 struct amdvi_softc *softc; 1346 1347 softc = amdvi_find_iommu(devid); 1348 KASSERT(softc, ("softc is NULL")); 1349 1350 amdvi_cmd_inv_dte(softc, devid); 1351 #ifdef AMDVI_ATS_ENABLE 1352 if (amdvi_dev_support_iotlb(softc, devid)) 1353 amdvi_cmd_inv_iotlb(softc, devid); 1354 #endif 1355 amdvi_wait(softc); 1356 } 1357 1358 static void 1359 amdvi_add_device(void *arg, uint16_t devid) 1360 { 1361 struct amdvi_domain *domain; 1362 1363 domain = (struct amdvi_domain *)arg; 1364 KASSERT(domain != NULL, ("domain is NULL")); 1365 #ifdef AMDVI_DEBUG_CMD 1366 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1367 RID2PCI_STR(devid), domain->id); 1368 #endif 1369 amdvi_set_dte(domain, devid, true); 1370 amdvi_inv_device(devid); 1371 } 1372 1373 static void 1374 amdvi_remove_device(void *arg, uint16_t devid) 1375 { 1376 struct amdvi_domain *domain; 1377 1378 domain = (struct amdvi_domain *)arg; 1379 #ifdef AMDVI_DEBUG_CMD 1380 printf("Remove device(0x%x) from domain:%d\n", 1381 devid, domain->id); 1382 #endif 1383 amdvi_set_dte(domain, devid, false); 1384 amdvi_inv_device(devid); 1385 } 1386 1387 static void 1388 amdvi_enable(void) 1389 { 1390 struct amdvi_ctrl *ctrl; 1391 struct amdvi_softc *softc; 1392 uint64_t val; 1393 int i; 1394 1395 for (i = 0; i < ivhd_count; i++) { 1396 softc = device_get_softc(ivhd_devs[i]); 1397 KASSERT(softc, ("softc is NULL\n")); 1398 ctrl = softc->ctrl; 1399 KASSERT(ctrl, ("ctrl is NULL\n")); 1400 1401 val = ( AMDVI_CTRL_EN | 1402 AMDVI_CTRL_CMD | 1403 AMDVI_CTRL_ELOG | 1404 AMDVI_CTRL_ELOGINT | 1405 AMDVI_CTRL_INV_TO_1S); 1406 1407 if (softc->ivhd_flag & IVHD_FLAG_COH) 1408 val |= AMDVI_CTRL_COH; 1409 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1410 val |= AMDVI_CTRL_HTT; 1411 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1412 val |= AMDVI_CTRL_RPPW; 1413 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1414 val |= AMDVI_CTRL_PPW; 1415 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1416 val |= AMDVI_CTRL_ISOC; 1417 1418 ctrl->control = val; 1419 } 1420 } 1421 1422 static void 1423 amdvi_disable(void) 1424 { 1425 struct amdvi_ctrl *ctrl; 1426 struct amdvi_softc *softc; 1427 int i; 1428 1429 for (i = 0; i < ivhd_count; i++) { 1430 softc = device_get_softc(ivhd_devs[i]); 1431 KASSERT(softc, ("softc is NULL\n")); 1432 ctrl = softc->ctrl; 1433 KASSERT(ctrl, ("ctrl is NULL\n")); 1434 1435 ctrl->control = 0; 1436 } 1437 } 1438 1439 static void 1440 amdvi_inv_tlb(void *arg) 1441 { 1442 struct amdvi_domain *domain; 1443 1444 domain = (struct amdvi_domain *)arg; 1445 KASSERT(domain, ("domain is NULL")); 1446 amdvi_do_inv_domain(domain->id, false); 1447 } 1448 1449 struct iommu_ops iommu_ops_amd = { 1450 amdvi_init, 1451 amdvi_cleanup, 1452 amdvi_enable, 1453 amdvi_disable, 1454 amdvi_create_domain, 1455 amdvi_destroy_domain, 1456 amdvi_create_mapping, 1457 amdvi_destroy_mapping, 1458 amdvi_add_device, 1459 amdvi_remove_device, 1460 amdvi_inv_tlb 1461 }; 1462