1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/sunddi.h> 28 #include <sys/sunndi.h> 29 #include <sys/acpi/acpi.h> 30 #include <sys/acpica.h> 31 #include <sys/amd_iommu.h> 32 #include <sys/bootconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/ddidmareq.h> 35 36 #include "amd_iommu_impl.h" 37 #include "amd_iommu_acpi.h" 38 #include "amd_iommu_page_tables.h" 39 40 ddi_dma_attr_t amd_iommu_pgtable_dma_attr = { 41 DMA_ATTR_V0, 42 0U, /* dma_attr_addr_lo */ 43 0xffffffffffffffffULL, /* dma_attr_addr_hi */ 44 0xffffffffU, /* dma_attr_count_max */ 45 (uint64_t)4096, /* dma_attr_align */ 46 1, /* dma_attr_burstsizes */ 47 64, /* dma_attr_minxfer */ 48 0xffffffffU, /* dma_attr_maxxfer */ 49 0xffffffffU, /* dma_attr_seg */ 50 1, /* dma_attr_sgllen, variable */ 51 64, /* dma_attr_granular */ 52 0 /* dma_attr_flags */ 53 }; 54 55 static amd_iommu_domain_t **amd_iommu_domain_table; 56 57 static struct { 58 int f_count; 59 amd_iommu_page_table_t *f_list; 60 } amd_iommu_pgtable_freelist; 61 int amd_iommu_no_pgtable_freelist; 62 63 /*ARGSUSED*/ 64 static int 65 amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp) 66 { 67 amd_iommu_acpi_ivhd_t *hinfop; 68 69 hinfop = amd_iommu_lookup_ivhd(bdf); 70 if (hinfop == NULL || hinfop->ach_src_deviceid == -1) 71 *src_bdfp = bdf; 72 else 73 *src_bdfp = hinfop->ach_src_deviceid; 74 75 return (DDI_SUCCESS); 76 } 77 78 static dev_info_t * 79 amd_iommu_pci_dip(dev_info_t *rdip, const char *path) 80 { 81 dev_info_t *pdip; 82 const char *driver = ddi_driver_name(rdip); 83 int instance = ddi_get_instance(rdip); 84 const char *f = "amd_iommu_pci_dip"; 85 86 /* Hold rdip so it and its parents don't go away */ 87 ndi_hold_devi(rdip); 88 89 if (ddi_is_pci_dip(rdip)) 90 return (rdip); 91 92 pdip = rdip; 93 while (pdip = ddi_get_parent(pdip)) { 94 if (ddi_is_pci_dip(pdip)) { 95 ndi_hold_devi(pdip); 96 ndi_rele_devi(rdip); 97 return (pdip); 98 } 99 } 100 101 cmn_err(CE_WARN, "%s: %s%d dip = %p has no PCI parent, path = %s", 102 f, driver, instance, (void *)rdip, path); 103 104 ndi_rele_devi(rdip); 105 106 ASSERT(0); 107 108 return (NULL); 109 } 110 111 /*ARGSUSED*/ 112 static int 113 amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias, 114 uint16_t deviceid, domain_id_t *domainid, const char *path) 115 { 116 const char *f = "amd_iommu_get_domain"; 117 118 *domainid = AMD_IOMMU_INVALID_DOMAIN; 119 120 ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0); 121 122 switch (deviceid) { 123 case AMD_IOMMU_INVALID_DOMAIN: 124 case AMD_IOMMU_IDENTITY_DOMAIN: 125 case AMD_IOMMU_PASSTHRU_DOMAIN: 126 case AMD_IOMMU_SYS_DOMAIN: 127 *domainid = AMD_IOMMU_SYS_DOMAIN; 128 break; 129 default: 130 *domainid = deviceid; 131 break; 132 } 133 134 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 135 cmn_err(CE_NOTE, "%s: domainid for %s = %d", 136 f, path, *domainid); 137 } 138 139 return (DDI_SUCCESS); 140 } 141 142 static uint16_t 143 hash_domain(domain_id_t domainid) 144 { 145 return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ); 146 } 147 148 /*ARGSUSED*/ 149 void 150 amd_iommu_init_page_tables(amd_iommu_t *iommu) 151 { 152 amd_iommu_domain_table = kmem_zalloc( 153 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP); 154 } 155 156 /*ARGSUSED*/ 157 void 158 amd_iommu_fini_page_tables(amd_iommu_t *iommu) 159 { 160 if (amd_iommu_domain_table) { 161 kmem_free(amd_iommu_domain_table, 162 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ); 163 amd_iommu_domain_table = NULL; 164 } 165 } 166 167 static amd_iommu_domain_t * 168 amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid, 169 map_type_t type, int km_flags) 170 { 171 uint16_t idx; 172 amd_iommu_domain_t *dp; 173 char name[AMD_IOMMU_VMEM_NAMELEN+1]; 174 175 ASSERT(amd_iommu_domain_table); 176 177 idx = hash_domain(domainid); 178 179 for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) { 180 if (dp->d_domainid == domainid) 181 return (dp); 182 } 183 184 ASSERT(type != AMD_IOMMU_INVALID_MAP); 185 186 dp = kmem_zalloc(sizeof (*dp), km_flags); 187 if (dp == NULL) 188 return (NULL); 189 dp->d_domainid = domainid; 190 dp->d_pgtable_root_4K = 0; /* make this explicit */ 191 192 if (type == AMD_IOMMU_VMEM_MAP) { 193 uint64_t base; 194 uint64_t size; 195 (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d", 196 iommu->aiomt_idx, domainid); 197 base = MMU_PAGESIZE; 198 size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE; 199 dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size, 200 MMU_PAGESIZE, NULL, NULL, NULL, 0, 201 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 202 if (dp->d_vmem == NULL) { 203 kmem_free(dp, sizeof (*dp)); 204 return (NULL); 205 } 206 } else { 207 dp->d_vmem = NULL; 208 } 209 210 dp->d_next = amd_iommu_domain_table[idx]; 211 dp->d_prev = NULL; 212 amd_iommu_domain_table[idx] = dp; 213 if (dp->d_next) 214 dp->d_next->d_prev = dp; 215 dp->d_ref = 0; 216 217 218 return (dp); 219 } 220 221 static void 222 amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp) 223 { 224 uint16_t idx; 225 int flags; 226 amd_iommu_cmdargs_t cmdargs = {0}; 227 domain_id_t domainid = dp->d_domainid; 228 const char *f = "amd_iommu_teardown_domain"; 229 230 ASSERT(dp->d_ref == 0); 231 232 idx = hash_domain(dp->d_domainid); 233 234 if (dp->d_prev == NULL) 235 amd_iommu_domain_table[idx] = dp->d_next; 236 else 237 dp->d_prev->d_next = dp->d_next; 238 239 if (dp->d_next) 240 dp->d_next->d_prev = dp->d_prev; 241 242 if (dp->d_vmem != NULL) { 243 vmem_destroy(dp->d_vmem); 244 dp->d_vmem = NULL; 245 } 246 247 kmem_free(dp, sizeof (*dp)); 248 249 cmdargs.ca_domainid = (uint16_t)domainid; 250 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 251 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 252 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 253 254 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 255 &cmdargs, flags, 0) != DDI_SUCCESS) { 256 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" 257 "Failed to invalidate domain in IOMMU HW cache", 258 f, iommu->aiomt_idx, cmdargs.ca_domainid); 259 } 260 } 261 262 static int 263 amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid, 264 int *aliasp, const char *path) 265 { 266 int bus = -1; 267 int device = -1; 268 int func = -1; 269 uint16_t bdf; 270 int32_t src_bdf; 271 dev_info_t *idip = iommu->aiomt_dip; 272 const char *driver = ddi_driver_name(idip); 273 int instance = ddi_get_instance(idip); 274 dev_info_t *pci_dip; 275 const char *f = "amd_iommu_get_deviceid"; 276 277 /* be conservative. Always assume an alias */ 278 *aliasp = 1; 279 *deviceid = 0; 280 281 /* Check for special special devices (rdip == NULL) */ 282 if (rdip == NULL) { 283 if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) { 284 cmn_err(CE_WARN, 285 "%s: %s%d: idx=%d, failed to get SRC BDF " 286 "for special-device", 287 f, driver, instance, iommu->aiomt_idx); 288 return (DDI_DMA_NOMAPPING); 289 } 290 *deviceid = src_bdf; 291 *aliasp = 1; 292 return (DDI_SUCCESS); 293 } 294 295 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 296 cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s", 297 f, path); 298 } 299 300 pci_dip = amd_iommu_pci_dip(rdip, path); 301 if (pci_dip == NULL) { 302 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip " 303 "for rdip=%p, path = %s", 304 f, driver, instance, iommu->aiomt_idx, (void *)rdip, 305 path); 306 return (DDI_DMA_NOMAPPING); 307 } 308 309 if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) { 310 ndi_rele_devi(pci_dip); 311 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for " 312 "PCI dip (%p). rdip path = %s", 313 f, driver, instance, iommu->aiomt_idx, 314 (void *)pci_dip, path); 315 return (DDI_DMA_NOMAPPING); 316 } 317 318 ndi_rele_devi(pci_dip); 319 320 if (bus > UINT8_MAX || bus < 0 || 321 device > UINT8_MAX || device < 0 || 322 func > UINT8_MAX || func < 0) { 323 cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) " 324 "for PCI dip (%p). rdip path = %s", f, driver, instance, 325 iommu->aiomt_idx, 326 bus, device, func, 327 (void *)pci_dip, path); 328 return (DDI_DMA_NOMAPPING); 329 } 330 331 bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func; 332 333 if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) { 334 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF " 335 "for PCI dip (%p) rdip path = %s.", 336 f, driver, instance, iommu->aiomt_idx, (void *)pci_dip, 337 path); 338 return (DDI_DMA_NOMAPPING); 339 } 340 341 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 342 cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s", 343 f, src_bdf, path); 344 } 345 346 *deviceid = src_bdf; 347 *aliasp = (src_bdf != bdf); 348 349 return (DDI_SUCCESS); 350 } 351 352 /*ARGSUSED*/ 353 static int 354 init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid, 355 amd_iommu_domain_t *dp) 356 { 357 uint64_t entry[4] = {0}; 358 int i; 359 360 /* If already passthru, don't touch */ 361 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 && 362 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { 363 return (0); 364 } 365 366 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 && 367 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) { 368 369 ASSERT(dp->d_pgtable_root_4K == 370 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), 371 AMD_IOMMU_DEVTBL_ROOT_PGTBL)); 372 373 ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), 374 AMD_IOMMU_DEVTBL_DOMAINID)); 375 376 return (0); 377 } 378 379 /* New devtbl entry for this domain. Bump up the domain ref-count */ 380 dp->d_ref++; 381 382 entry[3] = 0; 383 entry[2] = 0; 384 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SYSMGT, 1); 385 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1); 386 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0); 387 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0); 388 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1); 389 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0); 390 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1); 391 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOTLB, 1); 392 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID, 393 (uint16_t)domainid); 394 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1); 395 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1); 396 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 397 dp->d_pgtable_root_4K); 398 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE, 399 AMD_IOMMU_PGTABLE_MAXLEVEL); 400 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV, 401 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); 402 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V, 403 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); 404 405 for (i = 1; i < 4; i++) { 406 devtbl_entry[i] = entry[i]; 407 } 408 devtbl_entry[0] = entry[0]; 409 410 /* we did an actual init */ 411 return (1); 412 } 413 414 void 415 amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip) 416 { 417 int32_t deviceid; 418 int alias; 419 uint64_t *devtbl_entry; 420 amd_iommu_cmdargs_t cmdargs = {0}; 421 char *path; 422 int pathfree; 423 int V; 424 int TV; 425 int instance; 426 const char *driver; 427 const char *f = "amd_iommu_set_passthru"; 428 429 if (rdip) { 430 driver = ddi_driver_name(rdip); 431 instance = ddi_get_instance(rdip); 432 } else { 433 driver = "special-device"; 434 instance = 0; 435 } 436 437 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); 438 if (path) { 439 if (rdip) 440 (void) ddi_pathname(rdip, path); 441 else 442 (void) strcpy(path, "special-device"); 443 pathfree = 1; 444 } else { 445 pathfree = 0; 446 path = "<path-mem-alloc-failed>"; 447 } 448 449 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 450 != DDI_SUCCESS) { 451 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 452 "Failed to get device ID for device %s.", f, driver, 453 instance, 454 iommu->aiomt_idx, (void *)rdip, path); 455 goto out; 456 } 457 458 /* No deviceid */ 459 if (deviceid == -1) { 460 goto out; 461 } 462 463 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 464 iommu->aiomt_devtbl_sz) { 465 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 466 "for rdip (%p) exceeds device table size (%u), path=%s", 467 f, driver, 468 instance, iommu->aiomt_idx, deviceid, (void *)rdip, 469 iommu->aiomt_devtbl_sz, path); 470 goto out; 471 } 472 473 /*LINTED*/ 474 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 475 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 476 477 V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V); 478 TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV); 479 480 /* Already passthru */ 481 if (V == 0 && TV == 0) { 482 goto out; 483 } 484 485 /* Existing translations */ 486 if (V == 1 && TV == 1) { 487 goto out; 488 } 489 490 /* Invalid setting */ 491 if (V == 0 && TV == 1) { 492 goto out; 493 } 494 495 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0); 496 497 cmdargs.ca_deviceid = (uint16_t)deviceid; 498 (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 499 &cmdargs, 0, 0); 500 501 out: 502 if (pathfree) 503 kmem_free(path, MAXPATHLEN); 504 } 505 506 static int 507 amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, 508 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, 509 const char *path) 510 { 511 uint64_t *devtbl_entry; 512 amd_iommu_cmdargs_t cmdargs = {0}; 513 int error, flags; 514 dev_info_t *idip = iommu->aiomt_dip; 515 const char *driver = ddi_driver_name(idip); 516 int instance = ddi_get_instance(idip); 517 const char *f = "amd_iommu_set_devtbl_entry"; 518 519 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 520 cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s", 521 f, path); 522 } 523 524 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 525 iommu->aiomt_devtbl_sz) { 526 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 527 "for rdip (%p) exceeds device table size (%u), path=%s", 528 f, driver, 529 instance, iommu->aiomt_idx, deviceid, (void *)rdip, 530 iommu->aiomt_devtbl_sz, path); 531 return (DDI_DMA_NOMAPPING); 532 } 533 534 /*LINTED*/ 535 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 536 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 537 538 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 539 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", 540 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); 541 } 542 543 /* 544 * Flush internal caches, need to do this if we came up from 545 * fast boot 546 */ 547 cmdargs.ca_deviceid = deviceid; 548 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 549 &cmdargs, 0, 0); 550 if (error != DDI_SUCCESS) { 551 cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d" 552 "Failed to invalidate domain in IOMMU HW cache", 553 f, iommu->aiomt_idx, deviceid); 554 return (error); 555 } 556 557 cmdargs.ca_domainid = (uint16_t)domainid; 558 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 559 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 560 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 561 562 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 563 &cmdargs, flags, 0); 564 if (error != DDI_SUCCESS) { 565 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" 566 "Failed to invalidate translations in IOMMU HW cache", 567 f, iommu->aiomt_idx, cmdargs.ca_domainid); 568 return (error); 569 } 570 571 /* Initialize device table entry */ 572 if (init_devtbl(iommu, devtbl_entry, domainid, dp)) { 573 cmdargs.ca_deviceid = deviceid; 574 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 575 &cmdargs, 0, 0); 576 } 577 578 return (error); 579 } 580 581 int 582 amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, 583 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, 584 int *domain_freed, char *path) 585 { 586 uint64_t *devtbl_entry; 587 int error = DDI_SUCCESS; 588 amd_iommu_cmdargs_t cmdargs = {0}; 589 const char *driver = ddi_driver_name(iommu->aiomt_dip); 590 int instance = ddi_get_instance(iommu->aiomt_dip); 591 const char *f = "amd_iommu_clear_devtbl_entry"; 592 593 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 594 cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for " 595 "domainid = %d, deviceid = %u, path = %s", 596 f, domainid, deviceid, path); 597 } 598 599 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 600 iommu->aiomt_devtbl_sz) { 601 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 602 "for rdip (%p) exceeds device table size (%u), path = %s", 603 f, driver, instance, 604 iommu->aiomt_idx, deviceid, (void *)rdip, 605 iommu->aiomt_devtbl_sz, path); 606 return (DDI_FAILURE); 607 } 608 609 /*LINTED*/ 610 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 611 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 612 613 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 614 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", 615 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); 616 } 617 618 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { 619 /* Nothing to do */ 620 return (DDI_SUCCESS); 621 } 622 623 ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), 624 AMD_IOMMU_DEVTBL_ROOT_PGTBL)); 625 626 ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), 627 AMD_IOMMU_DEVTBL_DOMAINID)); 628 629 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0); 630 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0); 631 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1); 632 633 SYNC_FORDEV(iommu->aiomt_dmahdl); 634 635 dp->d_ref--; 636 ASSERT(dp->d_ref >= 0); 637 638 if (dp->d_ref == 0) { 639 *domain_freed = 1; 640 } 641 642 cmdargs.ca_deviceid = deviceid; 643 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 644 &cmdargs, 0, 0); 645 if (error != DDI_SUCCESS) 646 error = DDI_FAILURE; 647 648 return (error); 649 } 650 651 int 652 amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt) 653 { 654 ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) * 655 AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP); 656 return (DDI_SUCCESS); 657 } 658 659 void 660 amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt) 661 { 662 kmem_free(ampt->ampt_hash, 663 sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ); 664 ampt->ampt_hash = NULL; 665 } 666 667 static uint32_t 668 pt_hashfn(uint64_t pa_4K) 669 { 670 return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ); 671 } 672 673 static void 674 amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt) 675 { 676 uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12; 677 uint32_t idx = pt_hashfn(pa_4K); 678 679 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 680 681 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 682 683 pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx]; 684 pt->pt_prev = NULL; 685 amd_iommu_page_table_hash.ampt_hash[idx] = pt; 686 if (pt->pt_next) 687 pt->pt_next->pt_prev = pt; 688 689 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 690 } 691 692 static void 693 amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt) 694 { 695 uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12); 696 uint32_t idx = pt_hashfn(pa_4K); 697 698 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 699 700 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 701 702 if (pt->pt_next) 703 pt->pt_next->pt_prev = pt->pt_prev; 704 705 if (pt->pt_prev) 706 pt->pt_prev->pt_next = pt->pt_next; 707 else 708 amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next; 709 710 pt->pt_next = NULL; 711 pt->pt_prev = NULL; 712 713 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 714 } 715 716 static amd_iommu_page_table_t * 717 amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K) 718 { 719 amd_iommu_page_table_t *pt; 720 uint32_t idx = pt_hashfn(pgtable_pa_4K); 721 722 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 723 pt = amd_iommu_page_table_hash.ampt_hash[idx]; 724 for (; pt; pt = pt->pt_next) { 725 if (domainid != pt->pt_domainid) 726 continue; 727 ASSERT((pt->pt_cookie.dmac_cookie_addr & 728 AMD_IOMMU_PGTABLE_ALIGN) == 0); 729 if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) { 730 break; 731 } 732 } 733 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 734 735 return (pt); 736 } 737 738 /*ARGSUSED*/ 739 static amd_iommu_page_table_t * 740 amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt, 741 amd_iommu_domain_t *dp, int level, uint16_t index) 742 { 743 uint64_t *pdtep; 744 uint64_t pgtable_pa_4K; 745 746 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); 747 ASSERT(dp); 748 749 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 750 ASSERT(ppt == NULL); 751 ASSERT(index == 0); 752 pgtable_pa_4K = dp->d_pgtable_root_4K; 753 } else { 754 ASSERT(ppt); 755 pdtep = &(ppt->pt_pgtblva[index]); 756 if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) { 757 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 758 cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%" 759 PRIx64, *pdtep); 760 } 761 return (NULL); 762 } 763 pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR); 764 } 765 766 return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K)); 767 } 768 769 static amd_iommu_page_table_t * 770 amd_iommu_alloc_from_freelist(void) 771 { 772 int i; 773 uint64_t *pte_array; 774 amd_iommu_page_table_t *pt; 775 776 if (amd_iommu_no_pgtable_freelist == 1) 777 return (NULL); 778 779 if (amd_iommu_pgtable_freelist.f_count == 0) 780 return (NULL); 781 782 pt = amd_iommu_pgtable_freelist.f_list; 783 amd_iommu_pgtable_freelist.f_list = pt->pt_next; 784 amd_iommu_pgtable_freelist.f_count--; 785 786 pte_array = pt->pt_pgtblva; 787 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { 788 ASSERT(pt->pt_pte_ref[i] == 0); 789 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), 790 AMD_IOMMU_PTDE_PR) == 0); 791 } 792 793 return (pt); 794 } 795 796 static int 797 amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid, 798 const char *path, amd_iommu_page_table_t **ptp, int km_flags) 799 { 800 int err; 801 uint_t ncookies; 802 amd_iommu_page_table_t *pt; 803 dev_info_t *idip = iommu->aiomt_dip; 804 const char *driver = ddi_driver_name(idip); 805 int instance = ddi_get_instance(idip); 806 const char *f = "amd_iommu_alloc_pgtable"; 807 808 *ptp = NULL; 809 810 pt = amd_iommu_alloc_from_freelist(); 811 if (pt) 812 goto init_pgtable; 813 814 pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags); 815 if (pt == NULL) 816 return (DDI_DMA_NORESOURCES); 817 818 /* 819 * Each page table is 4K in size 820 */ 821 pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ; 822 823 /* 824 * Alloc a DMA handle. Use the IOMMU dip as we want this DMA 825 * to *not* enter the IOMMU - no recursive entrance. 826 */ 827 err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr, 828 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 829 NULL, &pt->pt_dma_hdl); 830 if (err != DDI_SUCCESS) { 831 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. " 832 "Cannot alloc DMA handle for IO Page Table", 833 f, driver, instance, domainid, path); 834 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 835 return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING); 836 } 837 838 /* 839 * Alloc memory for IO Page Table. 840 * XXX remove size_t cast kludge 841 */ 842 err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz, 843 &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED, 844 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 845 NULL, (caddr_t *)&pt->pt_pgtblva, 846 (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl); 847 if (err != DDI_SUCCESS) { 848 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " 849 "Cannot allocate DMA memory for IO Page table", 850 f, driver, instance, domainid, path); 851 ddi_dma_free_handle(&pt->pt_dma_hdl); 852 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 853 return (DDI_DMA_NORESOURCES); 854 } 855 856 /* 857 * The Page table DMA VA must be 4K aligned and 858 * size >= than requested memory. 859 * 860 */ 861 ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN) 862 == 0); 863 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); 864 865 /* 866 * Now bind the handle 867 */ 868 err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL, 869 (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz, 870 DDI_DMA_READ | DDI_DMA_CONSISTENT, 871 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 872 NULL, &pt->pt_cookie, &ncookies); 873 if (err != DDI_DMA_MAPPED) { 874 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " 875 "Cannot bind memory for DMA to IO Page Tables. " 876 "bufrealsz=%p", 877 f, driver, instance, domainid, path, 878 (void *)(uintptr_t)pt->pt_mem_realsz); 879 ddi_dma_mem_free(&pt->pt_mem_hdl); 880 ddi_dma_free_handle(&pt->pt_dma_hdl); 881 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 882 return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING : 883 err); 884 } 885 886 /* 887 * We assume the DMA engine on the IOMMU is capable of handling the 888 * whole page table in a single cookie. If not and multiple cookies 889 * are needed we fail. 890 */ 891 if (ncookies != 1) { 892 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s " 893 "Cannot handle multiple " 894 "cookies for DMA to IO page Table, #cookies=%u", 895 f, driver, instance, domainid, path, ncookies); 896 (void) ddi_dma_unbind_handle(pt->pt_dma_hdl); 897 ddi_dma_mem_free(&pt->pt_mem_hdl); 898 ddi_dma_free_handle(&pt->pt_dma_hdl); 899 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 900 return (DDI_DMA_NOMAPPING); 901 } 902 903 init_pgtable: 904 /* 905 * The address in the cookie must be 4K aligned and >= table size 906 */ 907 ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL); 908 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 909 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz); 910 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz); 911 ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE); 912 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); 913 ASSERT(pt->pt_pgtblva); 914 915 pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN; 916 pt->pt_level = 0x7; 917 pt->pt_index = 0; 918 pt->pt_ref = 0; 919 pt->pt_next = NULL; 920 pt->pt_prev = NULL; 921 pt->pt_parent = NULL; 922 923 bzero(pt->pt_pgtblva, pt->pt_mem_realsz); 924 SYNC_FORDEV(pt->pt_dma_hdl); 925 926 amd_iommu_insert_pgtable_hash(pt); 927 928 *ptp = pt; 929 930 return (DDI_SUCCESS); 931 } 932 933 static int 934 amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt) 935 { 936 if (amd_iommu_no_pgtable_freelist == 1) 937 return (DDI_FAILURE); 938 939 if (amd_iommu_pgtable_freelist.f_count == 940 AMD_IOMMU_PGTABLE_FREELIST_MAX) 941 return (DDI_FAILURE); 942 943 pt->pt_next = amd_iommu_pgtable_freelist.f_list; 944 amd_iommu_pgtable_freelist.f_list = pt; 945 amd_iommu_pgtable_freelist.f_count++; 946 947 return (DDI_SUCCESS); 948 } 949 950 static void 951 amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt) 952 { 953 int i; 954 uint64_t *pte_array; 955 dev_info_t *dip = iommu->aiomt_dip; 956 int instance = ddi_get_instance(dip); 957 const char *driver = ddi_driver_name(dip); 958 const char *f = "amd_iommu_free_pgtable"; 959 960 ASSERT(pt->pt_ref == 0); 961 962 amd_iommu_remove_pgtable_hash(pt); 963 964 pte_array = pt->pt_pgtblva; 965 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { 966 ASSERT(pt->pt_pte_ref[i] == 0); 967 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), 968 AMD_IOMMU_PTDE_PR) == 0); 969 } 970 971 if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS) 972 return; 973 974 /* Unbind the handle */ 975 if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) { 976 cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. " 977 "Failed to unbind handle: %p for IOMMU Page Table", 978 f, driver, instance, iommu->aiomt_idx, pt->pt_domainid, 979 (void *)pt->pt_dma_hdl); 980 } 981 /* Free the table memory allocated for DMA */ 982 ddi_dma_mem_free(&pt->pt_mem_hdl); 983 984 /* Free the DMA handle */ 985 ddi_dma_free_handle(&pt->pt_dma_hdl); 986 987 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 988 989 } 990 991 static int 992 init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt) 993 { 994 uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]); 995 uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; 996 997 /* nothing to set. PDE is already set */ 998 if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) { 999 ASSERT(PT_REF_VALID(ppt)); 1000 ASSERT(PT_REF_VALID(pt)); 1001 ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0); 1002 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR) 1003 == next_pgtable_pa_4K); 1004 return (DDI_SUCCESS); 1005 } 1006 1007 ppt->pt_ref++; 1008 ASSERT(PT_REF_VALID(ppt)); 1009 1010 /* Page Directories are always RW */ 1011 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1); 1012 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1); 1013 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR, 1014 next_pgtable_pa_4K); 1015 pt->pt_parent = ppt; 1016 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL, 1017 pt->pt_level); 1018 ppt->pt_pte_ref[pt->pt_index] = 0; 1019 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1); 1020 SYNC_FORDEV(ppt->pt_dma_hdl); 1021 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1); 1022 1023 return (DDI_SUCCESS); 1024 } 1025 1026 static int 1027 init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index, 1028 struct ddi_dma_req *dmareq) 1029 { 1030 uint64_t *ptep = &(pt->pt_pgtblva[index]); 1031 uint64_t pa_4K = pa >> 12; 1032 int R; 1033 int W; 1034 1035 /* nothing to set if PTE is already set */ 1036 if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) { 1037 /* 1038 * Adjust current permissions 1039 * DDI_DMA_WRITE means direction of DMA is MEM -> I/O 1040 * so that requires Memory READ permissions i.e. sense 1041 * is inverted. 1042 * Note: either or both of DD_DMA_READ/WRITE may be set 1043 */ 1044 if (amd_iommu_no_RW_perms == 0) { 1045 R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR); 1046 W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW); 1047 if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) || 1048 (dmareq->dmar_flags & DDI_DMA_RDWR))) { 1049 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1050 } 1051 if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) || 1052 (dmareq->dmar_flags & DDI_DMA_RDWR))) { 1053 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1054 } 1055 } 1056 ASSERT(PT_REF_VALID(pt)); 1057 pt->pt_pte_ref[index]++; 1058 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR) 1059 == pa_4K); 1060 return (DDI_SUCCESS); 1061 } 1062 1063 pt->pt_ref++; 1064 ASSERT(PT_REF_VALID(pt)); 1065 1066 /* see comment above about inverting sense of RD/WR */ 1067 if (amd_iommu_no_RW_perms == 0) { 1068 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0); 1069 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0); 1070 if (dmareq->dmar_flags & DDI_DMA_RDWR) { 1071 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1072 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1073 } else { 1074 if (dmareq->dmar_flags & DDI_DMA_WRITE) { 1075 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1076 } 1077 if (dmareq->dmar_flags & DDI_DMA_READ) { 1078 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1079 } 1080 } 1081 } else { 1082 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1083 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1084 } 1085 1086 /* TODO what is correct for FC and U */ 1087 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0); 1088 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0); 1089 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K); 1090 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0); 1091 ASSERT(pt->pt_pte_ref[index] == 0); 1092 pt->pt_pte_ref[index] = 1; 1093 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1); 1094 SYNC_FORDEV(pt->pt_dma_hdl); 1095 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1); 1096 1097 return (DDI_SUCCESS); 1098 } 1099 1100 1101 static void 1102 init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp, 1103 int level, uint16_t index) 1104 { 1105 ASSERT(dp); 1106 1107 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1108 dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; 1109 } else { 1110 ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL); 1111 } 1112 1113 pt->pt_domainid = dp->d_domainid; 1114 pt->pt_level = level; 1115 pt->pt_index = index; 1116 } 1117 1118 static int 1119 amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip, 1120 struct ddi_dma_req *dmareq, 1121 domain_id_t domainid, amd_iommu_domain_t *dp, 1122 amd_iommu_page_table_t *ppt, 1123 uint16_t index, int level, uint64_t va, uint64_t pa, 1124 amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path, 1125 int km_flags) 1126 { 1127 int error; 1128 amd_iommu_page_table_t *pt; 1129 const char *driver = ddi_driver_name(rdip); 1130 int instance = ddi_get_instance(rdip); 1131 const char *f = "amd_iommu_setup_1_pgtable"; 1132 1133 *ptp = NULL; 1134 *next_idxp = 0; 1135 error = DDI_SUCCESS; 1136 1137 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); 1138 1139 ASSERT(dp); 1140 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1141 ASSERT(ppt == NULL); 1142 ASSERT(index == 0); 1143 } else { 1144 ASSERT(ppt); 1145 } 1146 1147 /* Check if page table is already allocated */ 1148 if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) { 1149 ASSERT(pt->pt_domainid == domainid); 1150 ASSERT(pt->pt_level == level); 1151 ASSERT(pt->pt_index == index); 1152 goto out; 1153 } 1154 1155 if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt, 1156 km_flags)) != DDI_SUCCESS) { 1157 cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p " 1158 "path = %s", f, driver, instance, iommu->aiomt_idx, 1159 domainid, (void *)(uintptr_t)va, path); 1160 return (error); 1161 } 1162 1163 ASSERT(dp->d_domainid == domainid); 1164 1165 init_pt(pt, dp, level, index); 1166 1167 out: 1168 if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) { 1169 error = init_pde(ppt, pt); 1170 } 1171 1172 if (level == 1) { 1173 ASSERT(error == DDI_SUCCESS); 1174 error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq); 1175 } else { 1176 *next_idxp = AMD_IOMMU_VA_BITS(va, level); 1177 *ptp = pt; 1178 } 1179 1180 return (error); 1181 } 1182 1183 typedef enum { 1184 PDTE_NOT_TORN = 0x1, 1185 PDTE_TORN_DOWN = 0x2, 1186 PGTABLE_TORN_DOWN = 0x4 1187 } pdte_tear_t; 1188 1189 static pdte_tear_t 1190 amd_iommu_teardown_pdte(amd_iommu_t *iommu, 1191 amd_iommu_page_table_t *pt, int index) 1192 { 1193 uint8_t next_level; 1194 pdte_tear_t retval; 1195 uint64_t *ptdep = &(pt->pt_pgtblva[index]); 1196 1197 next_level = AMD_IOMMU_REG_GET64(ptdep, 1198 AMD_IOMMU_PTDE_NXT_LVL); 1199 1200 if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) { 1201 if (pt->pt_level == 1) { 1202 ASSERT(next_level == 0); 1203 /* PTE */ 1204 pt->pt_pte_ref[index]--; 1205 if (pt->pt_pte_ref[index] != 0) { 1206 return (PDTE_NOT_TORN); 1207 } 1208 } else { 1209 ASSERT(next_level != 0 && next_level != 7); 1210 } 1211 ASSERT(pt->pt_pte_ref[index] == 0); 1212 ASSERT(PT_REF_VALID(pt)); 1213 1214 AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0); 1215 SYNC_FORDEV(pt->pt_dma_hdl); 1216 ASSERT(AMD_IOMMU_REG_GET64(ptdep, 1217 AMD_IOMMU_PTDE_PR) == 0); 1218 pt->pt_ref--; 1219 ASSERT(PT_REF_VALID(pt)); 1220 retval = PDTE_TORN_DOWN; 1221 } else { 1222 ASSERT(0); 1223 ASSERT(pt->pt_pte_ref[index] == 0); 1224 ASSERT(PT_REF_VALID(pt)); 1225 retval = PDTE_NOT_TORN; 1226 } 1227 1228 if (pt->pt_ref == 0) { 1229 amd_iommu_free_pgtable(iommu, pt); 1230 return (PGTABLE_TORN_DOWN); 1231 } 1232 1233 return (retval); 1234 } 1235 1236 static int 1237 amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, 1238 struct ddi_dma_req *dmareq, uint64_t va, 1239 uint64_t pa, uint16_t deviceid, domain_id_t domainid, 1240 amd_iommu_domain_t *dp, const char *path, int km_flags) 1241 { 1242 int level; 1243 uint16_t index; 1244 uint16_t next_idx; 1245 amd_iommu_page_table_t *pt; 1246 amd_iommu_page_table_t *ppt; 1247 int error; 1248 const char *driver = ddi_driver_name(rdip); 1249 int instance = ddi_get_instance(rdip); 1250 const char *f = "amd_iommu_create_pgtables"; 1251 1252 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1253 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " 1254 "deviceid = %u, va = %p, pa = %p, path = %s", 1255 f, driver, instance, 1256 iommu->aiomt_idx, domainid, deviceid, 1257 (void *)(uintptr_t)va, 1258 (void *)(uintptr_t)pa, path); 1259 } 1260 1261 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { 1262 /* No need for pagetables. Just set up device table entry */ 1263 goto passthru; 1264 } 1265 1266 index = 0; 1267 ppt = NULL; 1268 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; 1269 level--, pt = NULL, next_idx = 0) { 1270 if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq, 1271 domainid, dp, ppt, index, level, va, pa, &pt, 1272 &next_idx, path, km_flags)) != DDI_SUCCESS) { 1273 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " 1274 "deviceid=%u, va= %p, pa = %p, Failed to setup " 1275 "page table(s) at level = %d, path = %s.", 1276 f, driver, instance, iommu->aiomt_idx, 1277 domainid, deviceid, (void *)(uintptr_t)va, 1278 (void *)(uintptr_t)pa, level, path); 1279 return (error); 1280 } 1281 1282 if (level > 1) { 1283 ASSERT(pt); 1284 ASSERT(pt->pt_domainid == domainid); 1285 ppt = pt; 1286 index = next_idx; 1287 } else { 1288 ASSERT(level == 1); 1289 ASSERT(pt == NULL); 1290 ASSERT(next_idx == 0); 1291 ppt = NULL; 1292 index = 0; 1293 } 1294 } 1295 1296 passthru: 1297 if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid, 1298 dp, path)) != DDI_SUCCESS) { 1299 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, " 1300 "domainid=%d." 1301 "Failed to set device table entry for path %s.", 1302 f, driver, instance, 1303 iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path); 1304 return (error); 1305 } 1306 1307 SYNC_FORDEV(iommu->aiomt_dmahdl); 1308 1309 return (DDI_SUCCESS); 1310 } 1311 1312 static int 1313 amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, 1314 uint64_t pageva, uint16_t deviceid, domain_id_t domainid, 1315 amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path) 1316 { 1317 int level; 1318 int flags; 1319 amd_iommu_cmdargs_t cmdargs = {0}; 1320 uint16_t index; 1321 uint16_t prev_index; 1322 amd_iommu_page_table_t *pt; 1323 amd_iommu_page_table_t *ppt; 1324 pdte_tear_t retval; 1325 int tear_level; 1326 int invalidate_pte; 1327 int invalidate_pde; 1328 int error = DDI_FAILURE; 1329 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1330 int instance = ddi_get_instance(iommu->aiomt_dip); 1331 const char *f = "amd_iommu_destroy_pgtables"; 1332 1333 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1334 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " 1335 "deviceid = %u, va = %p, path = %s", 1336 f, driver, instance, 1337 iommu->aiomt_idx, domainid, deviceid, 1338 (void *)(uintptr_t)pageva, path); 1339 } 1340 1341 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { 1342 /* 1343 * there are no pagetables for the passthru domain. 1344 * Just the device table entry 1345 */ 1346 error = DDI_SUCCESS; 1347 goto passthru; 1348 } 1349 1350 ppt = NULL; 1351 index = 0; 1352 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) { 1353 pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index); 1354 if (pt) { 1355 ppt = pt; 1356 index = AMD_IOMMU_VA_BITS(pageva, level); 1357 continue; 1358 } 1359 break; 1360 } 1361 1362 if (level == 0) { 1363 uint64_t *ptep; 1364 uint64_t pa_4K; 1365 1366 ASSERT(pt); 1367 ASSERT(pt == ppt); 1368 ASSERT(pt->pt_domainid == dp->d_domainid); 1369 1370 ptep = &(pt->pt_pgtblva[index]); 1371 1372 pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR); 1373 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1374 ASSERT(pageva == (pa_4K << MMU_PAGESHIFT)); 1375 } 1376 } 1377 1378 tear_level = -1; 1379 invalidate_pde = 0; 1380 invalidate_pte = 0; 1381 for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) { 1382 prev_index = pt->pt_index; 1383 ppt = pt->pt_parent; 1384 retval = amd_iommu_teardown_pdte(iommu, pt, index); 1385 switch (retval) { 1386 case PDTE_NOT_TORN: 1387 goto invalidate; 1388 case PDTE_TORN_DOWN: 1389 invalidate_pte = 1; 1390 goto invalidate; 1391 case PGTABLE_TORN_DOWN: 1392 invalidate_pte = 1; 1393 invalidate_pde = 1; 1394 tear_level = level; 1395 break; 1396 } 1397 index = prev_index; 1398 pt = ppt; 1399 } 1400 1401 invalidate: 1402 /* 1403 * Now teardown the IOMMU HW caches if applicable 1404 */ 1405 if (invalidate_pte) { 1406 cmdargs.ca_domainid = (uint16_t)domainid; 1407 if (amd_iommu_pageva_inval_all) { 1408 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 1409 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 1410 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 1411 } else if (invalidate_pde) { 1412 cmdargs.ca_addr = 1413 (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level); 1414 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 1415 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 1416 } else { 1417 cmdargs.ca_addr = (uintptr_t)pageva; 1418 flags = 0; 1419 } 1420 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 1421 &cmdargs, flags, 0) != DDI_SUCCESS) { 1422 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " 1423 "rdip=%p. Failed to invalidate IOMMU HW cache " 1424 "for %s", f, driver, instance, 1425 iommu->aiomt_idx, domainid, (void *)rdip, path); 1426 error = DDI_FAILURE; 1427 goto out; 1428 } 1429 } 1430 1431 passthru: 1432 if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1433 error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid, 1434 deviceid, dp, domain_freed, path); 1435 } else { 1436 error = DDI_SUCCESS; 1437 } 1438 1439 out: 1440 SYNC_FORDEV(iommu->aiomt_dmahdl); 1441 1442 return (error); 1443 } 1444 1445 static int 1446 cvt_bind_error(int error) 1447 { 1448 switch (error) { 1449 case DDI_DMA_MAPPED: 1450 case DDI_DMA_PARTIAL_MAP: 1451 case DDI_DMA_NORESOURCES: 1452 case DDI_DMA_NOMAPPING: 1453 break; 1454 default: 1455 cmn_err(CE_PANIC, "Unsupported error code: %d", error); 1456 /*NOTREACHED*/ 1457 } 1458 return (error); 1459 } 1460 1461 int 1462 amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp, 1463 struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz, 1464 map_type_t type, uint64_t *start_vap, int km_flags) 1465 { 1466 pfn_t pfn_start; 1467 pfn_t pfn_end; 1468 pfn_t pfn; 1469 int alias; 1470 int32_t deviceid; 1471 domain_id_t domainid; 1472 amd_iommu_domain_t *dp; 1473 uint64_t end_pa; 1474 uint64_t start_va; 1475 uint64_t end_va; 1476 uint64_t pg_start; 1477 uint64_t pg_end; 1478 uint64_t pg; 1479 uint64_t va_sz; 1480 char *path; 1481 int error = DDI_DMA_NOMAPPING; 1482 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1483 int instance = ddi_get_instance(iommu->aiomt_dip); 1484 const char *f = "amd_iommu_map_pa2va"; 1485 1486 ASSERT(pa_sz != 0); 1487 1488 *start_vap = 0; 1489 1490 ASSERT(rdip); 1491 1492 path = kmem_alloc(MAXPATHLEN, km_flags); 1493 if (path == NULL) { 1494 error = DDI_DMA_NORESOURCES; 1495 goto out; 1496 } 1497 (void) ddi_pathname(rdip, path); 1498 1499 /* 1500 * First get deviceid 1501 */ 1502 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 1503 != DDI_SUCCESS) { 1504 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 1505 "Failed to get device ID for %s.", f, driver, instance, 1506 iommu->aiomt_idx, (void *)rdip, path); 1507 error = DDI_DMA_NOMAPPING; 1508 goto out; 1509 } 1510 1511 /* 1512 * Next get the domain for this rdip 1513 */ 1514 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) 1515 != DDI_SUCCESS) { 1516 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " 1517 "Failed to get domain.", f, driver, instance, 1518 iommu->aiomt_idx, (void *)rdip, path); 1519 error = DDI_DMA_NOMAPPING; 1520 goto out; 1521 } 1522 1523 dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags); 1524 if (dp == NULL) { 1525 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " 1526 "Failed to get device ID for %s.", f, driver, instance, 1527 iommu->aiomt_idx, domainid, (void *)rdip, path); 1528 error = DDI_DMA_NORESOURCES; 1529 goto out; 1530 } 1531 1532 ASSERT(dp->d_domainid == domainid); 1533 1534 pfn_start = start_pa >> MMU_PAGESHIFT; 1535 1536 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1537 cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, " 1538 "pgshift = %d", 1539 (void *)(uintptr_t)start_pa, 1540 (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT), 1541 (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT); 1542 } 1543 1544 end_pa = start_pa + pa_sz - 1; 1545 pfn_end = end_pa >> MMU_PAGESHIFT; 1546 1547 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1548 start_va = start_pa; 1549 end_va = end_pa; 1550 va_sz = pa_sz; 1551 *start_vap = start_va; 1552 } else { 1553 va_sz = mmu_ptob(pfn_end - pfn_start + 1); 1554 start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz, 1555 MAX(attrp->dma_attr_align, MMU_PAGESIZE), 1556 0, 1557 attrp->dma_attr_seg + 1, 1558 (void *)(uintptr_t)attrp->dma_attr_addr_lo, 1559 (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1), 1560 AMD_IOMMU_SIZE_4G), /* XXX rollover */ 1561 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 1562 if (start_va == 0) { 1563 cmn_err(CE_WARN, "%s: No VA resources", 1564 amd_iommu_modname); 1565 error = DDI_DMA_NORESOURCES; 1566 goto out; 1567 } 1568 ASSERT((start_va & MMU_PAGEOFFSET) == 0); 1569 end_va = start_va + va_sz - 1; 1570 *start_vap = start_va + (start_pa & MMU_PAGEOFFSET); 1571 } 1572 1573 pg_start = start_va >> MMU_PAGESHIFT; 1574 pg_end = end_va >> MMU_PAGESHIFT; 1575 1576 pg = pg_start; 1577 for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) { 1578 1579 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1580 cmn_err(CE_NOTE, "%s: attempting to create page tables " 1581 "for pfn = %p, va = %p, path = %s", 1582 f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT), 1583 (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path); 1584 1585 } 1586 1587 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1588 ASSERT(pfn == pg); 1589 } 1590 1591 if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq, 1592 pg << MMU_PAGESHIFT, 1593 pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path, 1594 km_flags)) != DDI_SUCCESS) { 1595 cmn_err(CE_WARN, "Failed to create_pgtables"); 1596 goto out; 1597 } 1598 1599 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1600 cmn_err(CE_NOTE, "%s: successfuly created page tables " 1601 "for pfn = %p, vapg = %p, path = %s", 1602 f, (void *)(uintptr_t)pfn, 1603 (void *)(uintptr_t)pg, path); 1604 } 1605 1606 } 1607 ASSERT(pg == pg_end + 1); 1608 1609 1610 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) { 1611 cmn_err(CE_NOTE, "pa=%p, va=%p", 1612 (void *)(uintptr_t)start_pa, 1613 (void *)(uintptr_t)(*start_vap)); 1614 } 1615 error = DDI_DMA_MAPPED; 1616 1617 out: 1618 kmem_free(path, MAXPATHLEN); 1619 return (cvt_bind_error(error)); 1620 } 1621 1622 int 1623 amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va, 1624 uint64_t va_sz, map_type_t type) 1625 { 1626 uint64_t end_va; 1627 uint64_t pg_start; 1628 uint64_t pg_end; 1629 uint64_t pg; 1630 uint64_t actual_sz; 1631 char *path; 1632 int pathfree; 1633 int alias; 1634 int32_t deviceid; 1635 domain_id_t domainid; 1636 amd_iommu_domain_t *dp; 1637 int error; 1638 int domain_freed; 1639 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1640 int instance = ddi_get_instance(iommu->aiomt_dip); 1641 const char *f = "amd_iommu_unmap_va"; 1642 1643 if (amd_iommu_no_unmap) 1644 return (DDI_SUCCESS); 1645 1646 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); 1647 if (path) { 1648 (void) ddi_pathname(rdip, path); 1649 pathfree = 1; 1650 } else { 1651 pathfree = 0; 1652 path = "<path-mem-alloc-failed>"; 1653 } 1654 1655 /* 1656 * First get deviceid 1657 */ 1658 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 1659 != DDI_SUCCESS) { 1660 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 1661 "Failed to get device ID for %s.", f, driver, instance, 1662 iommu->aiomt_idx, (void *)rdip, path); 1663 error = DDI_FAILURE; 1664 goto out; 1665 } 1666 1667 /* 1668 * Next get the domain for this rdip 1669 */ 1670 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) 1671 != DDI_SUCCESS) { 1672 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " 1673 "Failed to get domain.", f, driver, instance, 1674 iommu->aiomt_idx, (void *)rdip, path); 1675 error = DDI_FAILURE; 1676 goto out; 1677 } 1678 1679 /* should never result in domain allocation/vmem_create */ 1680 dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP, 1681 KM_NOSLEEP); 1682 if (dp == NULL) { 1683 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " 1684 "Failed to get device ID for %s.", f, driver, instance, 1685 iommu->aiomt_idx, domainid, (void *)rdip, path); 1686 error = DDI_FAILURE; 1687 goto out; 1688 } 1689 1690 ASSERT(dp->d_domainid == domainid); 1691 1692 pg_start = start_va >> MMU_PAGESHIFT; 1693 end_va = start_va + va_sz - 1; 1694 pg_end = end_va >> MMU_PAGESHIFT; 1695 actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT; 1696 1697 domain_freed = 0; 1698 for (pg = pg_start; pg <= pg_end; pg++) { 1699 domain_freed = 0; 1700 if (amd_iommu_destroy_pgtables(iommu, rdip, 1701 pg << MMU_PAGESHIFT, deviceid, domainid, dp, type, 1702 &domain_freed, path) != DDI_SUCCESS) { 1703 error = DDI_FAILURE; 1704 goto out; 1705 } 1706 if (domain_freed) { 1707 ASSERT(pg == pg_end); 1708 break; 1709 } 1710 } 1711 1712 /* 1713 * vmem_xalloc() must be paired with vmem_xfree 1714 */ 1715 if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) { 1716 vmem_xfree(dp->d_vmem, 1717 (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz); 1718 } 1719 1720 if (domain_freed) 1721 amd_iommu_teardown_domain(iommu, dp); 1722 1723 error = DDI_SUCCESS; 1724 out: 1725 if (pathfree) 1726 kmem_free(path, MAXPATHLEN); 1727 return (error); 1728 } 1729