1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/malloc.h> 38 #include <sys/pcpu.h> 39 #include <sys/rman.h> 40 #include <sys/smp.h> 41 #include <sys/sysctl.h> 42 43 #include <vm/vm.h> 44 #include <vm/pmap.h> 45 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pcireg.h> 48 49 #include <machine/resource.h> 50 #include <machine/vmm.h> 51 #include <machine/pmap.h> 52 #include <machine/vmparam.h> 53 #include <machine/pci_cfgreg.h> 54 55 #include "pcib_if.h" 56 57 #include "io/iommu.h" 58 #include "amdvi_priv.h" 59 60 SYSCTL_DECL(_hw_vmm); 61 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 62 NULL); 63 64 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 65 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 66 67 /* Print RID or device ID in PCI string format. */ 68 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 69 70 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count); 71 static void amdvi_print_dev_cap(struct amdvi_softc *softc); 72 73 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 74 75 extern device_t *ivhd_devs; 76 77 extern int ivhd_count; 78 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 79 0, NULL); 80 81 static int amdvi_enable_user = 0; 82 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 83 &amdvi_enable_user, 0, NULL); 84 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 85 86 #ifdef AMDVI_ATS_ENABLE 87 /* XXX: ATS is not tested. */ 88 static int amdvi_enable_iotlb = 1; 89 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 90 &amdvi_enable_iotlb, 0, NULL); 91 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 92 #endif 93 94 static int amdvi_host_ptp = 1; /* Use page tables for host. */ 95 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 96 &amdvi_host_ptp, 0, NULL); 97 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 98 99 /* Page table level used <= supported by h/w[v1=7]. */ 100 int amdvi_ptp_level = 4; 101 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 102 &amdvi_ptp_level, 0, NULL); 103 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 104 105 /* Disable fault event reporting. */ 106 static int amdvi_disable_io_fault = 0; 107 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 108 &amdvi_disable_io_fault, 0, NULL); 109 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 110 111 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 112 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 113 &amdvi_dom_id, 0, NULL); 114 /* 115 * Device table entry. 116 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 117 * = 256 * 2 * PAGE_SIZE. 118 */ 119 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 120 CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 121 CTASSERT(sizeof(amdvi_dte) == 0x200000); 122 123 static SLIST_HEAD (, amdvi_domain) dom_head; 124 125 static inline uint32_t 126 amdvi_pci_read(struct amdvi_softc *softc, int off) 127 { 128 129 return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid), 130 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 131 off, 4)); 132 } 133 134 #ifdef AMDVI_ATS_ENABLE 135 /* XXX: Should be in pci.c */ 136 /* 137 * Check if device has ATS capability and its enabled. 138 * If ATS is absent or disabled, return (-1), otherwise ATS 139 * queue length. 140 */ 141 static int 142 amdvi_find_ats_qlen(uint16_t devid) 143 { 144 device_t dev; 145 uint32_t off, cap; 146 int qlen = -1; 147 148 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 149 PCI_RID2FUNC(devid)); 150 151 if (!dev) { 152 return (-1); 153 } 154 #define PCIM_ATS_EN BIT(31) 155 156 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 157 cap = pci_read_config(dev, off + 4, 4); 158 qlen = (cap & 0x1F); 159 qlen = qlen ? qlen : 32; 160 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 161 RID2PCI_STR(devid), 162 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 163 qlen); 164 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 165 } 166 167 return (qlen); 168 } 169 170 /* 171 * Check if an endpoint device support device IOTLB or ATS. 172 */ 173 static inline bool 174 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 175 { 176 struct ivhd_dev_cfg *cfg; 177 int qlen, i; 178 bool pci_ats, ivhd_ats; 179 180 qlen = amdvi_find_ats_qlen(devid); 181 if (qlen < 0) 182 return (false); 183 184 KASSERT(softc, ("softc is NULL")); 185 cfg = softc->dev_cfg; 186 187 ivhd_ats = false; 188 for (i = 0; i < softc->dev_cfg_cnt; i++) { 189 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 190 ivhd_ats = cfg->enable_ats; 191 break; 192 } 193 cfg++; 194 } 195 196 pci_ats = (qlen < 0) ? false : true; 197 if (pci_ats != ivhd_ats) 198 device_printf(softc->dev, 199 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 200 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 201 202 /* Ignore IVRS setting and respect PCI setting. */ 203 return (pci_ats); 204 } 205 #endif 206 207 /* Enable IOTLB support for IOMMU if its supported. */ 208 static inline void 209 amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 210 { 211 #ifndef AMDVI_ATS_ENABLE 212 softc->iotlb = false; 213 #else 214 bool supported; 215 216 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 217 218 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 219 if (!supported) 220 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 221 222 if (supported && !amdvi_enable_iotlb) { 223 device_printf(softc->dev, "IOTLB disabled by user.\n"); 224 supported = false; 225 } 226 } else 227 supported = false; 228 229 softc->iotlb = supported; 230 231 #endif 232 } 233 234 static int 235 amdvi_init_cmd(struct amdvi_softc *softc) 236 { 237 struct amdvi_ctrl *ctrl = softc->ctrl; 238 239 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 240 softc->cmd_max = 1 << ctrl->cmd.len; 241 242 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 243 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 244 245 if ((uintptr_t)softc->cmd & PAGE_MASK) 246 panic("AMDVi: Command buffer not aligned on page boundary."); 247 248 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 249 /* 250 * XXX: Reset the h/w pointers in case IOMMU is restarting, 251 * h/w doesn't clear these pointers based on empirical data. 252 */ 253 ctrl->cmd_tail = 0; 254 ctrl->cmd_head = 0; 255 256 return (0); 257 } 258 259 /* 260 * Note: Update tail pointer after we have written the command since tail 261 * pointer update cause h/w to execute new commands, see section 3.3 262 * of AMD IOMMU spec ver 2.0. 263 */ 264 /* Get the command tail pointer w/o updating it. */ 265 static struct amdvi_cmd * 266 amdvi_get_cmd_tail(struct amdvi_softc *softc) 267 { 268 struct amdvi_ctrl *ctrl; 269 struct amdvi_cmd *tail; 270 271 KASSERT(softc, ("softc is NULL")); 272 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 273 274 ctrl = softc->ctrl; 275 KASSERT(ctrl != NULL, ("ctrl is NULL")); 276 277 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 278 ctrl->cmd_tail); 279 280 return (tail); 281 } 282 283 /* 284 * Update the command tail pointer which will start command execution. 285 */ 286 static void 287 amdvi_update_cmd_tail(struct amdvi_softc *softc) 288 { 289 struct amdvi_ctrl *ctrl; 290 int size; 291 292 size = sizeof(struct amdvi_cmd); 293 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 294 295 ctrl = softc->ctrl; 296 KASSERT(ctrl != NULL, ("ctrl is NULL")); 297 298 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 299 softc->total_cmd++; 300 301 #ifdef AMDVI_DEBUG_CMD 302 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 303 ctrl->cmd_tail, 304 ctrl->cmd_head); 305 #endif 306 307 } 308 309 /* 310 * Various commands supported by IOMMU. 311 */ 312 313 /* Completion wait command. */ 314 static void 315 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 316 { 317 struct amdvi_cmd *cmd; 318 uint64_t pa; 319 320 cmd = amdvi_get_cmd_tail(softc); 321 KASSERT(cmd != NULL, ("Cmd is NULL")); 322 323 pa = vtophys(&softc->cmp_data); 324 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 325 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; 326 cmd->word1 = (pa >> 32) & 0xFFFFF; 327 cmd->addr = data; 328 329 amdvi_update_cmd_tail(softc); 330 } 331 332 /* Invalidate device table entry. */ 333 static void 334 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 335 { 336 struct amdvi_cmd *cmd; 337 338 cmd = amdvi_get_cmd_tail(softc); 339 KASSERT(cmd != NULL, ("Cmd is NULL")); 340 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 341 cmd->word0 = devid; 342 amdvi_update_cmd_tail(softc); 343 #ifdef AMDVI_DEBUG_CMD 344 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 345 #endif 346 } 347 348 /* Invalidate IOMMU page, use for invalidation of domain. */ 349 static void 350 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 351 uint64_t addr, bool guest_nested, 352 bool pde, bool page) 353 { 354 struct amdvi_cmd *cmd; 355 356 cmd = amdvi_get_cmd_tail(softc); 357 KASSERT(cmd != NULL, ("Cmd is NULL")); 358 359 360 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 361 cmd->word1 = domain_id; 362 /* 363 * Invalidate all addresses for this domain. 364 */ 365 cmd->addr = addr; 366 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 367 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 368 369 amdvi_update_cmd_tail(softc); 370 } 371 372 #ifdef AMDVI_ATS_ENABLE 373 /* Invalidate device IOTLB. */ 374 static void 375 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 376 { 377 struct amdvi_cmd *cmd; 378 int qlen; 379 380 if (!softc->iotlb) 381 return; 382 383 qlen = amdvi_find_ats_qlen(devid); 384 if (qlen < 0) { 385 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 386 qlen, RID2PCI_STR(devid)); 387 } 388 cmd = amdvi_get_cmd_tail(softc); 389 KASSERT(cmd != NULL, ("Cmd is NULL")); 390 391 #ifdef AMDVI_DEBUG_CMD 392 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 393 " Qlen:%d\n", devid, qlen); 394 #endif 395 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 396 cmd->word0 = devid; 397 cmd->word1 = qlen; 398 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 399 AMDVI_INVD_IOTLB_S; 400 amdvi_update_cmd_tail(softc); 401 } 402 #endif 403 404 #ifdef notyet /* For Interrupt Remap. */ 405 static void 406 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 407 uint16_t devid) 408 { 409 struct amdvi_cmd *cmd; 410 411 cmd = amdvi_get_cmd_tail(softc); 412 KASSERT(cmd != NULL, ("Cmd is NULL")); 413 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 414 cmd->word0 = devid; 415 amdvi_update_cmd_tail(softc); 416 #ifdef AMDVI_DEBUG_CMD 417 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 418 #endif 419 } 420 #endif 421 422 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 423 static void 424 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 425 { 426 struct amdvi_cmd *cmd; 427 428 cmd = amdvi_get_cmd_tail(softc); 429 KASSERT(cmd != NULL, ("Cmd is NULL")); 430 431 /* 432 * See section 3.3.3 of IOMMU spec rev 2.0, software note 433 * for invalidating domain. 434 */ 435 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 436 false, true, true); 437 438 #ifdef AMDVI_DEBUG_CMD 439 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 440 441 #endif 442 } 443 444 static bool 445 amdvi_cmp_wait(struct amdvi_softc *softc) 446 { 447 struct amdvi_ctrl *ctrl; 448 const uint64_t VERIFY = 0xA5A5; 449 volatile uint64_t *read; 450 int i; 451 bool status; 452 453 ctrl = softc->ctrl; 454 read = &softc->cmp_data; 455 *read = 0; 456 amdvi_cmd_cmp(softc, VERIFY); 457 /* Wait for h/w to update completion data. */ 458 for (i = 0; i < 100 && (*read != VERIFY); i++) { 459 DELAY(1000); /* 1 ms */ 460 } 461 status = (VERIFY == softc->cmp_data) ? true : false; 462 463 #ifdef AMDVI_DEBUG_CMD 464 if (status) 465 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 466 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 467 ctrl->cmd_head, loop); 468 #endif 469 return (status); 470 } 471 472 static void 473 amdvi_wait(struct amdvi_softc *softc) 474 { 475 struct amdvi_ctrl *ctrl; 476 int i; 477 478 KASSERT(softc, ("softc is NULL")); 479 480 ctrl = softc->ctrl; 481 KASSERT(ctrl != NULL, ("ctrl is NULL")); 482 /* Don't wait if h/w is not enabled. */ 483 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 484 return; 485 486 for (i = 0; i < 10; i++) { 487 if (amdvi_cmp_wait(softc)) 488 return; 489 } 490 491 device_printf(softc->dev, "Error: completion failed" 492 " tail:0x%x, head:0x%x.\n", 493 ctrl->cmd_tail, ctrl->cmd_head); 494 /* Dump the last command. */ 495 amdvi_dump_cmds(softc, 1); 496 } 497 498 static void 499 amdvi_dump_cmds(struct amdvi_softc *softc, int count) 500 { 501 struct amdvi_ctrl *ctrl; 502 struct amdvi_cmd *cmd; 503 int off, i; 504 505 ctrl = softc->ctrl; 506 device_printf(softc->dev, "Dump last %d command(s):\n", count); 507 /* 508 * If h/w is stuck in completion, it is the previous command, 509 * start dumping from previous command onward. 510 */ 511 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 512 softc->cmd_max); 513 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { 514 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 515 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 516 " 0x%x 0x%lx\n", i, off, cmd->opcode, 517 cmd->word0, cmd->word1, cmd->addr); 518 off = (off + sizeof(struct amdvi_cmd)) % 519 (softc->cmd_max * sizeof(struct amdvi_cmd)); 520 } 521 } 522 523 static int 524 amdvi_init_event(struct amdvi_softc *softc) 525 { 526 struct amdvi_ctrl *ctrl; 527 528 ctrl = softc->ctrl; 529 ctrl->event.len = 8; 530 softc->event_max = 1 << ctrl->event.len; 531 softc->event = malloc(sizeof(struct amdvi_event) * 532 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 533 if ((uintptr_t)softc->event & PAGE_MASK) { 534 device_printf(softc->dev, "Event buffer not aligned on page."); 535 return (false); 536 } 537 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 538 539 /* Reset the pointers. */ 540 ctrl->evt_head = 0; 541 ctrl->evt_tail = 0; 542 543 return (0); 544 } 545 546 static inline void 547 amdvi_decode_evt_flag(uint16_t flag) 548 { 549 550 flag &= AMDVI_EVENT_FLAG_MASK; 551 printf(" 0x%b]\n", flag, 552 "\020" 553 "\001GN" 554 "\002NX" 555 "\003US" 556 "\004I" 557 "\005PR" 558 "\006RW" 559 "\007PE" 560 "\010RZ" 561 "\011TR" 562 ); 563 } 564 565 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 566 static inline void 567 amdvi_decode_evt_flag_type(uint8_t type) 568 { 569 570 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 571 case 0: 572 printf("RSVD\n"); 573 break; 574 case 1: 575 printf("Master Abort\n"); 576 break; 577 case 2: 578 printf("Target Abort\n"); 579 break; 580 case 3: 581 printf("Data Err\n"); 582 break; 583 default: 584 break; 585 } 586 } 587 588 static void 589 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 590 uint16_t flag) 591 { 592 593 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 594 " Addr:0x%lx", 595 devid, domid, addr); 596 amdvi_decode_evt_flag(flag); 597 } 598 599 static void 600 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 601 uint16_t flag) 602 { 603 604 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 605 " Addr:0x%lx", 606 devid, domid, addr); 607 amdvi_decode_evt_flag(flag); 608 } 609 610 static void 611 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 612 uint64_t addr, uint16_t flag) 613 { 614 615 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 616 " Addr:0x%lx", devid, domid, addr); 617 amdvi_decode_evt_flag(flag); 618 amdvi_decode_evt_flag_type(flag); 619 } 620 621 static void 622 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 623 uint16_t flag) 624 { 625 626 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 627 " Addr:0x%lx", devid, domid, addr); 628 amdvi_decode_evt_flag(flag); 629 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 630 } 631 632 static void 633 amdvi_decode_evt(struct amdvi_event *evt) 634 { 635 struct amdvi_cmd *cmd; 636 637 switch (evt->opcode) { 638 case AMDVI_EVENT_INVALID_DTE: 639 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 640 evt->addr, evt->flag); 641 break; 642 643 case AMDVI_EVENT_PFAULT: 644 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 645 evt->addr, evt->flag); 646 break; 647 648 case AMDVI_EVENT_DTE_HW_ERROR: 649 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 650 evt->addr, evt->flag); 651 break; 652 653 case AMDVI_EVENT_PAGE_HW_ERROR: 654 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 655 evt->addr, evt->flag); 656 break; 657 658 case AMDVI_EVENT_ILLEGAL_CMD: 659 /* FALL THROUGH */ 660 case AMDVI_EVENT_CMD_HW_ERROR: 661 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 662 "ILLEGAL CMD" : "CMD HW ERR"); 663 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 664 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 665 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 666 break; 667 668 case AMDVI_EVENT_IOTLB_TIMEOUT: 669 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 670 evt->devid, evt->addr); 671 break; 672 673 case AMDVI_EVENT_INVALID_DTE_REQ: 674 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 675 evt->devid, evt->addr, evt->flag >> 9, 676 (evt->flag >> 8) & 1); 677 break; 678 679 case AMDVI_EVENT_INVALID_PPR_REQ: 680 case AMDVI_EVENT_COUNTER_ZERO: 681 printf("AMD-Vi: v2 events.\n"); 682 break; 683 684 default: 685 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 686 } 687 } 688 689 static void 690 amdvi_print_events(struct amdvi_softc *softc) 691 { 692 struct amdvi_ctrl *ctrl; 693 struct amdvi_event *event; 694 int i, size; 695 696 ctrl = softc->ctrl; 697 size = sizeof(struct amdvi_event); 698 for (i = 0; i < softc->event_max; i++) { 699 event = &softc->event[ctrl->evt_head / size]; 700 if (!event->opcode) 701 break; 702 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 703 i, ctrl->evt_head, ctrl->evt_tail); 704 amdvi_decode_evt(event); 705 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 706 softc->event_max); 707 } 708 } 709 710 static int 711 amdvi_init_dte(struct amdvi_softc *softc) 712 { 713 struct amdvi_ctrl *ctrl; 714 715 ctrl = softc->ctrl; 716 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 717 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 718 719 return (0); 720 } 721 722 /* 723 * Not all capabilities of IOMMU are available in ACPI IVHD flag 724 * or EFR entry, read directly from device. 725 */ 726 static int 727 amdvi_print_pci_cap(device_t dev) 728 { 729 struct amdvi_softc *softc; 730 uint32_t off, cap; 731 732 733 softc = device_get_softc(dev); 734 off = softc->cap_off; 735 736 /* 737 * Section 3.7.1 of IOMMU sepc rev 2.0. 738 * Read capability from device. 739 */ 740 cap = amdvi_pci_read(softc, off); 741 742 /* Make sure capability type[18:16] is 3. */ 743 KASSERT((((cap >> 16) & 0x7) == 0x3), 744 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 745 746 softc->pci_cap = cap >> 24; 747 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 748 cap, off, softc->pci_cap, 749 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 750 751 return (0); 752 } 753 754 static void 755 amdvi_event_intr(void *arg) 756 { 757 struct amdvi_softc *softc; 758 struct amdvi_ctrl *ctrl; 759 760 softc = (struct amdvi_softc *)arg; 761 ctrl = softc->ctrl; 762 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 763 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 764 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 765 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 766 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 767 768 amdvi_print_events(softc); 769 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 770 } 771 772 static void 773 amdvi_free_evt_intr_res(device_t dev) 774 { 775 776 struct amdvi_softc *softc; 777 778 softc = device_get_softc(dev); 779 if (softc->event_tag != NULL) { 780 bus_teardown_intr(dev, softc->event_res, softc->event_tag); 781 } 782 if (softc->event_res != NULL) { 783 bus_release_resource(dev, SYS_RES_IRQ, softc->event_rid, 784 softc->event_res); 785 } 786 bus_delete_resource(dev, SYS_RES_IRQ, softc->event_rid); 787 PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)), 788 dev, 1, &softc->event_irq); 789 } 790 791 static bool 792 amdvi_alloc_intr_resources(struct amdvi_softc *softc) 793 { 794 struct amdvi_ctrl *ctrl; 795 device_t dev, pcib; 796 device_t mmio_dev; 797 uint64_t msi_addr; 798 uint32_t msi_data; 799 int err; 800 801 dev = softc->dev; 802 pcib = device_get_parent(device_get_parent(dev)); 803 mmio_dev = pci_find_bsf(PCI_RID2BUS(softc->pci_rid), 804 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid)); 805 if (device_is_attached(mmio_dev)) { 806 device_printf(dev, 807 "warning: IOMMU device is claimed by another driver %s\n", 808 device_get_driver(mmio_dev)->name); 809 } 810 811 softc->event_irq = -1; 812 softc->event_rid = 0; 813 814 /* 815 * Section 3.7.1 of IOMMU rev 2.0. With MSI, there is only one 816 * interrupt. XXX: Enable MSI/X support. 817 */ 818 err = PCIB_ALLOC_MSI(pcib, dev, 1, 1, &softc->event_irq); 819 if (err) { 820 device_printf(dev, 821 "Couldn't find event MSI IRQ resource.\n"); 822 return (ENOENT); 823 } 824 825 err = bus_set_resource(dev, SYS_RES_IRQ, softc->event_rid, 826 softc->event_irq, 1); 827 if (err) { 828 device_printf(dev, "Couldn't set event MSI resource.\n"); 829 return (ENXIO); 830 } 831 832 softc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 833 &softc->event_rid, RF_ACTIVE); 834 if (!softc->event_res) { 835 device_printf(dev, 836 "Unable to allocate event INTR resource.\n"); 837 return (ENOMEM); 838 } 839 840 if (bus_setup_intr(dev, softc->event_res, 841 INTR_TYPE_MISC | INTR_MPSAFE, NULL, amdvi_event_intr, 842 softc, &softc->event_tag)) { 843 device_printf(dev, "Fail to setup event intr\n"); 844 bus_release_resource(softc->dev, SYS_RES_IRQ, 845 softc->event_rid, softc->event_res); 846 softc->event_res = NULL; 847 return (ENXIO); 848 } 849 850 bus_describe_intr(dev, softc->event_res, softc->event_tag, 851 "fault"); 852 853 err = PCIB_MAP_MSI(pcib, dev, softc->event_irq, &msi_addr, 854 &msi_data); 855 if (err) { 856 device_printf(dev, 857 "Event interrupt config failed, err=%d.\n", 858 err); 859 amdvi_free_evt_intr_res(softc->dev); 860 return (err); 861 } 862 863 /* Clear interrupt status bits. */ 864 ctrl = softc->ctrl; 865 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 866 867 /* Now enable MSI interrupt. */ 868 pci_enable_msi(mmio_dev, msi_addr, msi_data); 869 return (0); 870 } 871 872 873 static void 874 amdvi_print_dev_cap(struct amdvi_softc *softc) 875 { 876 struct ivhd_dev_cfg *cfg; 877 int i; 878 879 cfg = softc->dev_cfg; 880 for (i = 0; i < softc->dev_cfg_cnt; i++) { 881 device_printf(softc->dev, "device [0x%x - 0x%x]" 882 "config:%b%s\n", cfg->start_id, cfg->end_id, 883 cfg->data, 884 "\020\001INIT\002ExtInt\003NMI" 885 "\007LINT0\008LINT1", 886 cfg->enable_ats ? "ATS enabled" : ""); 887 cfg++; 888 } 889 } 890 891 static int 892 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 893 { 894 struct amdvi_softc *softc; 895 int result, type, error = 0; 896 897 softc = (struct amdvi_softc *)arg1; 898 type = arg2; 899 900 switch (type) { 901 case 0: 902 result = softc->ctrl->cmd_head; 903 error = sysctl_handle_int(oidp, &result, 0, 904 req); 905 break; 906 case 1: 907 result = softc->ctrl->cmd_tail; 908 error = sysctl_handle_int(oidp, &result, 0, 909 req); 910 break; 911 case 2: 912 result = softc->ctrl->evt_head; 913 error = sysctl_handle_int(oidp, &result, 0, 914 req); 915 break; 916 case 3: 917 result = softc->ctrl->evt_tail; 918 error = sysctl_handle_int(oidp, &result, 0, 919 req); 920 break; 921 922 default: 923 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 924 } 925 926 return (error); 927 } 928 929 static void 930 amdvi_add_sysctl(struct amdvi_softc *softc) 931 { 932 struct sysctl_oid_list *child; 933 struct sysctl_ctx_list *ctx; 934 device_t dev; 935 936 dev = softc->dev; 937 ctx = device_get_sysctl_ctx(dev); 938 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 939 940 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 941 &softc->event_intr_cnt, "Event interrupt count"); 942 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 943 &softc->total_cmd, "Command submitted count"); 944 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 945 &softc->pci_rid, 0, "IOMMU RID"); 946 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "start_dev_rid", CTLFLAG_RD, 947 &softc->start_dev_rid, 0, "Start of device under this IOMMU"); 948 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "end_dev_rid", CTLFLAG_RD, 949 &softc->end_dev_rid, 0, "End of device under this IOMMU"); 950 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 951 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 952 amdvi_handle_sysctl, "IU", "Command head"); 953 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 954 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 955 amdvi_handle_sysctl, "IU", "Command tail"); 956 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 957 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2, 958 amdvi_handle_sysctl, "IU", "Command head"); 959 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 960 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3, 961 amdvi_handle_sysctl, "IU", "Command tail"); 962 } 963 964 int 965 amdvi_setup_hw(struct amdvi_softc *softc) 966 { 967 device_t dev; 968 int status; 969 970 dev = softc->dev; 971 972 amdvi_hw_enable_iotlb(softc); 973 974 amdvi_print_dev_cap(softc); 975 976 if ((status = amdvi_print_pci_cap(dev)) != 0) { 977 device_printf(dev, "PCI capability.\n"); 978 return (status); 979 } 980 if ((status = amdvi_init_cmd(softc)) != 0) { 981 device_printf(dev, "Couldn't configure command buffer.\n"); 982 return (status); 983 } 984 if ((status = amdvi_init_event(softc)) != 0) { 985 device_printf(dev, "Couldn't configure event buffer.\n"); 986 return (status); 987 } 988 if ((status = amdvi_init_dte(softc)) != 0) { 989 device_printf(dev, "Couldn't configure device table.\n"); 990 return (status); 991 } 992 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 993 return (status); 994 } 995 amdvi_add_sysctl(softc); 996 return (0); 997 } 998 999 int 1000 amdvi_teardown_hw(struct amdvi_softc *softc) 1001 { 1002 device_t dev; 1003 1004 dev = softc->dev; 1005 1006 /* 1007 * Called after disable, h/w is stopped by now, free all the resources. 1008 */ 1009 amdvi_free_evt_intr_res(dev); 1010 1011 if (softc->cmd) 1012 free(softc->cmd, M_AMDVI); 1013 1014 if (softc->event) 1015 free(softc->event, M_AMDVI); 1016 1017 return (0); 1018 } 1019 1020 /*********** bhyve interfaces *********************/ 1021 static int 1022 amdvi_init(void) 1023 { 1024 if (!ivhd_count) { 1025 return (EIO); 1026 } 1027 if (!amdvi_enable_user && ivhd_count) { 1028 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 1029 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 1030 ivhd_count); 1031 return (EINVAL); 1032 } 1033 return (0); 1034 } 1035 1036 static void 1037 amdvi_cleanup(void) 1038 { 1039 /* Nothing. */ 1040 } 1041 1042 static uint16_t 1043 amdvi_domainId(void) 1044 { 1045 1046 /* 1047 * If we hit maximum domain limit, rollover leaving host 1048 * domain(0). 1049 * XXX: make sure that this domain is not used. 1050 */ 1051 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 1052 amdvi_dom_id = 1; 1053 1054 return ((uint16_t)amdvi_dom_id++); 1055 } 1056 1057 static void 1058 amdvi_do_inv_domain(uint16_t domain_id, bool create) 1059 { 1060 struct amdvi_softc *softc; 1061 int i; 1062 1063 for (i = 0; i < ivhd_count; i++) { 1064 softc = device_get_softc(ivhd_devs[i]); 1065 KASSERT(softc, ("softc is NULL")); 1066 /* 1067 * If not present pages are cached, invalidate page after 1068 * creating domain. 1069 */ 1070 #if 0 1071 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 1072 continue; 1073 #endif 1074 amdvi_inv_domain(softc, domain_id); 1075 amdvi_wait(softc); 1076 } 1077 } 1078 1079 static void * 1080 amdvi_create_domain(vm_paddr_t maxaddr) 1081 { 1082 struct amdvi_domain *dom; 1083 1084 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1085 dom->id = amdvi_domainId(); 1086 //dom->maxaddr = maxaddr; 1087 #ifdef AMDVI_DEBUG_CMD 1088 printf("Created domain #%d\n", dom->id); 1089 #endif 1090 /* 1091 * Host domain(#0) don't create translation table. 1092 */ 1093 if (dom->id || amdvi_host_ptp) 1094 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1095 1096 dom->ptp_level = amdvi_ptp_level; 1097 1098 amdvi_do_inv_domain(dom->id, true); 1099 SLIST_INSERT_HEAD(&dom_head, dom, next); 1100 1101 return (dom); 1102 } 1103 1104 static void 1105 amdvi_free_ptp(uint64_t *ptp, int level) 1106 { 1107 int i; 1108 1109 if (level < 1) 1110 return; 1111 1112 for (i = 0; i < NPTEPG ; i++) { 1113 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1114 continue; 1115 /* XXX: Add super-page or PTE mapping > 4KB. */ 1116 #ifdef notyet 1117 /* Super-page mapping. */ 1118 if (AMDVI_PD_SUPER(ptp[i])) 1119 continue; 1120 #endif 1121 1122 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1123 & AMDVI_PT_MASK), level - 1); 1124 1125 } 1126 1127 free(ptp, M_AMDVI); 1128 } 1129 1130 static void 1131 amdvi_destroy_domain(void *arg) 1132 { 1133 struct amdvi_domain *domain; 1134 1135 domain = (struct amdvi_domain *)arg; 1136 KASSERT(domain, ("domain is NULL")); 1137 #ifdef AMDVI_DEBUG_CMD 1138 printf("Destroying domain %d\n", domain->id); 1139 #endif 1140 if (domain->ptp) 1141 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1142 1143 amdvi_do_inv_domain(domain->id, false); 1144 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1145 free(domain, M_AMDVI); 1146 } 1147 1148 static uint64_t 1149 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1150 vm_paddr_t hpa, uint64_t pg_size, bool create) 1151 { 1152 uint64_t *page, pa; 1153 int shift, index; 1154 const int PT_SHIFT = 9; 1155 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1156 1157 if (!pg_size) 1158 return (0); 1159 1160 if (hpa & (pg_size - 1)) { 1161 printf("HPA is not size aligned.\n"); 1162 return (0); 1163 } 1164 if (gpa & (pg_size - 1)) { 1165 printf("HPA is not size aligned.\n"); 1166 return (0); 1167 } 1168 shift = PML4SHIFT; 1169 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1170 index = (gpa >> shift) & PT_INDEX_MASK; 1171 1172 if ((pt[index] == 0) && create) { 1173 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1174 pa = vtophys(page); 1175 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1176 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1177 } 1178 #ifdef AMDVI_DEBUG_PTE 1179 if ((gpa % 0x1000000) == 0) 1180 printf("[level%d, shift = %d]PTE:0x%lx\n", 1181 level, shift, pt[index]); 1182 #endif 1183 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1184 pa = PTE2PA(pt[index]); 1185 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1186 shift -= PT_SHIFT; 1187 level--; 1188 } 1189 1190 /* Leaf entry. */ 1191 index = (gpa >> shift) & PT_INDEX_MASK; 1192 1193 if (create) { 1194 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1195 } else 1196 pt[index] = 0; 1197 1198 #ifdef AMDVI_DEBUG_PTE 1199 if ((gpa % 0x1000000) == 0) 1200 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1201 level, shift, pt[index]); 1202 #endif 1203 return (1ULL << shift); 1204 } 1205 1206 static uint64_t 1207 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1208 vm_paddr_t hpa, uint64_t size, bool create) 1209 { 1210 uint64_t mapped, *ptp, len; 1211 int level; 1212 1213 KASSERT(domain, ("domain is NULL")); 1214 level = domain->ptp_level; 1215 KASSERT(level, ("Page table level is 0")); 1216 1217 ptp = domain->ptp; 1218 KASSERT(ptp, ("PTP is NULL")); 1219 mapped = 0; 1220 while (mapped < size) { 1221 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1222 PAGE_SIZE, create); 1223 if (!len) { 1224 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1225 hpa, gpa); 1226 return (0); 1227 } 1228 mapped += len; 1229 } 1230 1231 return (mapped); 1232 } 1233 1234 static uint64_t 1235 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1236 uint64_t len) 1237 { 1238 struct amdvi_domain *domain; 1239 1240 domain = (struct amdvi_domain *)arg; 1241 1242 if (domain->id && !domain->ptp) { 1243 printf("ptp is NULL"); 1244 return (-1); 1245 } 1246 1247 /* 1248 * If host domain is created w/o page table, skip IOMMU page 1249 * table set-up. 1250 */ 1251 if (domain->ptp) 1252 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1253 else 1254 return (len); 1255 } 1256 1257 static uint64_t 1258 amdvi_destroy_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1259 { 1260 struct amdvi_domain *domain; 1261 1262 domain = (struct amdvi_domain *)arg; 1263 /* 1264 * If host domain is created w/o page table, skip IOMMU page 1265 * table set-up. 1266 */ 1267 if (domain->ptp) 1268 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1269 return 1270 (len); 1271 } 1272 1273 static struct amdvi_softc * 1274 amdvi_find_iommu(uint16_t devid) 1275 { 1276 struct amdvi_softc *softc; 1277 int i; 1278 1279 for (i = 0; i < ivhd_count; i++) { 1280 softc = device_get_softc(ivhd_devs[i]); 1281 if ((devid >= softc->start_dev_rid) && 1282 (devid <= softc->end_dev_rid)) 1283 return (softc); 1284 } 1285 1286 /* 1287 * XXX: BIOS bug, device not in IVRS table, assume its from first IOMMU. 1288 */ 1289 printf("BIOS bug device(%d.%d.%d) doesn't have IVHD entry.\n", 1290 RID2PCI_STR(devid)); 1291 1292 return (device_get_softc(ivhd_devs[0])); 1293 } 1294 1295 /* 1296 * Set-up device table entry. 1297 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1298 * be set concurrently, e.g. read and write bits. 1299 */ 1300 static void 1301 amdvi_set_dte(struct amdvi_domain *domain, uint16_t devid, bool enable) 1302 { 1303 struct amdvi_softc *softc; 1304 struct amdvi_dte* temp; 1305 1306 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1307 1308 softc = amdvi_find_iommu(devid); 1309 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1310 1311 temp = &amdvi_dte[devid]; 1312 1313 #ifdef AMDVI_ATS_ENABLE 1314 /* If IOMMU and device support IOTLB, enable it. */ 1315 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1316 temp->iotlb_enable = 1; 1317 #endif 1318 1319 /* Avoid duplicate I/O faults. */ 1320 temp->sup_second_io_fault = 1; 1321 temp->sup_all_io_fault = amdvi_disable_io_fault; 1322 1323 temp->dt_valid = 1; 1324 temp->domain_id = domain->id; 1325 1326 if (enable) { 1327 if (domain->ptp) { 1328 temp->pt_base = vtophys(domain->ptp) >> 12; 1329 temp->pt_level = amdvi_ptp_level; 1330 } 1331 /* 1332 * XXX: Page table valid[TV] bit must be set even if host domain 1333 * page tables are not enabled. 1334 */ 1335 temp->pt_valid = 1; 1336 temp->read_allow = 1; 1337 temp->write_allow = 1; 1338 } 1339 } 1340 1341 static void 1342 amdvi_inv_device(uint16_t devid) 1343 { 1344 struct amdvi_softc *softc; 1345 1346 softc = amdvi_find_iommu(devid); 1347 KASSERT(softc, ("softc is NULL")); 1348 1349 amdvi_cmd_inv_dte(softc, devid); 1350 #ifdef AMDVI_ATS_ENABLE 1351 if (amdvi_dev_support_iotlb(softc, devid)) 1352 amdvi_cmd_inv_iotlb(softc, devid); 1353 #endif 1354 amdvi_wait(softc); 1355 } 1356 1357 static void 1358 amdvi_add_device(void *arg, uint16_t devid) 1359 { 1360 struct amdvi_domain *domain; 1361 1362 domain = (struct amdvi_domain *)arg; 1363 KASSERT(domain != NULL, ("domain is NULL")); 1364 #ifdef AMDVI_DEBUG_CMD 1365 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1366 RID2PCI_STR(devid), domain->id); 1367 #endif 1368 amdvi_set_dte(domain, devid, true); 1369 amdvi_inv_device(devid); 1370 } 1371 1372 static void 1373 amdvi_remove_device(void *arg, uint16_t devid) 1374 { 1375 struct amdvi_domain *domain; 1376 1377 domain = (struct amdvi_domain *)arg; 1378 #ifdef AMDVI_DEBUG_CMD 1379 printf("Remove device(0x%x) from domain:%d\n", 1380 devid, domain->id); 1381 #endif 1382 amdvi_set_dte(domain, devid, false); 1383 amdvi_inv_device(devid); 1384 } 1385 1386 static void 1387 amdvi_enable(void) 1388 { 1389 struct amdvi_ctrl *ctrl; 1390 struct amdvi_softc *softc; 1391 uint64_t val; 1392 int i; 1393 1394 for (i = 0; i < ivhd_count; i++) { 1395 softc = device_get_softc(ivhd_devs[i]); 1396 KASSERT(softc, ("softc is NULL\n")); 1397 ctrl = softc->ctrl; 1398 KASSERT(ctrl, ("ctrl is NULL\n")); 1399 1400 val = ( AMDVI_CTRL_EN | 1401 AMDVI_CTRL_CMD | 1402 AMDVI_CTRL_ELOG | 1403 AMDVI_CTRL_ELOGINT | 1404 AMDVI_CTRL_INV_TO_1S); 1405 1406 if (softc->ivhd_flag & IVHD_FLAG_COH) 1407 val |= AMDVI_CTRL_COH; 1408 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1409 val |= AMDVI_CTRL_HTT; 1410 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1411 val |= AMDVI_CTRL_RPPW; 1412 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1413 val |= AMDVI_CTRL_PPW; 1414 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1415 val |= AMDVI_CTRL_ISOC; 1416 1417 ctrl->control = val; 1418 } 1419 } 1420 1421 static void 1422 amdvi_disable(void) 1423 { 1424 struct amdvi_ctrl *ctrl; 1425 struct amdvi_softc *softc; 1426 int i; 1427 1428 for (i = 0; i < ivhd_count; i++) { 1429 softc = device_get_softc(ivhd_devs[i]); 1430 KASSERT(softc, ("softc is NULL\n")); 1431 ctrl = softc->ctrl; 1432 KASSERT(ctrl, ("ctrl is NULL\n")); 1433 1434 ctrl->control = 0; 1435 } 1436 } 1437 1438 static void 1439 amdvi_inv_tlb(void *arg) 1440 { 1441 struct amdvi_domain *domain; 1442 1443 domain = (struct amdvi_domain *)arg; 1444 KASSERT(domain, ("domain is NULL")); 1445 amdvi_do_inv_domain(domain->id, false); 1446 } 1447 1448 struct iommu_ops iommu_ops_amd = { 1449 amdvi_init, 1450 amdvi_cleanup, 1451 amdvi_enable, 1452 amdvi_disable, 1453 amdvi_create_domain, 1454 amdvi_destroy_domain, 1455 amdvi_create_mapping, 1456 amdvi_destroy_mapping, 1457 amdvi_add_device, 1458 amdvi_remove_device, 1459 amdvi_inv_tlb 1460 }; 1461