1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/malloc.h> 38 #include <sys/bus.h> 39 #include <sys/interrupt.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/limits.h> 43 #include <sys/lock.h> 44 #include <sys/memdesc.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/rman.h> 49 #include <sys/sysctl.h> 50 #include <sys/taskqueue.h> 51 #include <sys/tree.h> 52 #include <sys/uio.h> 53 #include <sys/vmem.h> 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_kern.h> 57 #include <vm/vm_object.h> 58 #include <vm/vm_page.h> 59 #include <vm/vm_pager.h> 60 #include <vm/vm_map.h> 61 #include <machine/atomic.h> 62 #include <machine/bus.h> 63 #include <machine/md_var.h> 64 #include <machine/specialreg.h> 65 #include <contrib/dev/acpica/include/acpi.h> 66 #include <contrib/dev/acpica/include/accommon.h> 67 #include <x86/include/busdma_impl.h> 68 #include <x86/iommu/intel_reg.h> 69 #include <x86/iommu/busdma_dmar.h> 70 #include <x86/iommu/intel_dmar.h> 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcivar.h> 73 74 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 75 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); 76 77 static void dmar_domain_unload_task(void *arg, int pending); 78 static void dmar_unref_domain_locked(struct dmar_unit *dmar, 79 struct dmar_domain *domain); 80 static void dmar_domain_destroy(struct dmar_domain *domain); 81 82 static void 83 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 84 { 85 struct sf_buf *sf; 86 dmar_root_entry_t *re; 87 vm_page_t ctxm; 88 89 /* 90 * Allocated context page must be linked. 91 */ 92 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_NOALLOC); 93 if (ctxm != NULL) 94 return; 95 96 /* 97 * Page not present, allocate and link. Note that other 98 * thread might execute this sequence in parallel. This 99 * should be safe, because the context entries written by both 100 * threads are equal. 101 */ 102 TD_PREP_PINNED_ASSERT; 103 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, DMAR_PGF_ZERO | 104 DMAR_PGF_WAITOK); 105 re = dmar_map_pgtbl(dmar->ctx_obj, 0, DMAR_PGF_NOALLOC, &sf); 106 re += bus; 107 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 108 VM_PAGE_TO_PHYS(ctxm))); 109 dmar_flush_root_to_ram(dmar, re); 110 dmar_unmap_pgtbl(sf); 111 TD_PINNED_ASSERT; 112 } 113 114 static dmar_ctx_entry_t * 115 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 116 { 117 dmar_ctx_entry_t *ctxp; 118 119 ctxp = dmar_map_pgtbl(ctx->domain->dmar->ctx_obj, 1 + 120 PCI_RID2BUS(ctx->rid), DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp); 121 ctxp += ctx->rid & 0xff; 122 return (ctxp); 123 } 124 125 static void 126 ctx_tag_init(struct dmar_ctx *ctx, device_t dev) 127 { 128 bus_addr_t maxaddr; 129 130 maxaddr = MIN(ctx->domain->end, BUS_SPACE_MAXADDR); 131 ctx->ctx_tag.common.ref_count = 1; /* Prevent free */ 132 ctx->ctx_tag.common.impl = &bus_dma_dmar_impl; 133 ctx->ctx_tag.common.boundary = 0; 134 ctx->ctx_tag.common.lowaddr = maxaddr; 135 ctx->ctx_tag.common.highaddr = maxaddr; 136 ctx->ctx_tag.common.maxsize = maxaddr; 137 ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED; 138 ctx->ctx_tag.common.maxsegsz = maxaddr; 139 ctx->ctx_tag.ctx = ctx; 140 ctx->ctx_tag.owner = dev; 141 } 142 143 static void 144 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move) 145 { 146 struct dmar_unit *unit; 147 struct dmar_domain *domain; 148 vm_page_t ctx_root; 149 150 domain = ctx->domain; 151 unit = domain->dmar; 152 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), 153 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 154 unit->unit, pci_get_bus(ctx->ctx_tag.owner), 155 pci_get_slot(ctx->ctx_tag.owner), 156 pci_get_function(ctx->ctx_tag.owner), 157 ctxp->ctx1, ctxp->ctx2)); 158 /* 159 * For update due to move, the store is not atomic. It is 160 * possible that DMAR read upper doubleword, while low 161 * doubleword is not yet updated. The domain id is stored in 162 * the upper doubleword, while the table pointer in the lower. 163 * 164 * There is no good solution, for the same reason it is wrong 165 * to clear P bit in the ctx entry for update. 166 */ 167 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | 168 domain->awlvl); 169 if ((domain->flags & DMAR_DOMAIN_IDMAP) != 0 && 170 (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 171 KASSERT(domain->pgtbl_obj == NULL, 172 ("ctx %p non-null pgtbl_obj", ctx)); 173 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 174 } else { 175 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, DMAR_PGF_NOALLOC); 176 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 177 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 178 DMAR_CTX1_P); 179 } 180 dmar_flush_ctx_to_ram(unit, ctxp); 181 } 182 183 static int 184 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) 185 { 186 int error; 187 188 /* 189 * If dmar declares Caching Mode as Set, follow 11.5 "Caching 190 * Mode Consideration" and do the (global) invalidation of the 191 * negative TLB entries. 192 */ 193 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force) 194 return (0); 195 if (dmar->qi_enabled) { 196 dmar_qi_invalidate_ctx_glob_locked(dmar); 197 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force) 198 dmar_qi_invalidate_iotlb_glob_locked(dmar); 199 return (0); 200 } 201 error = dmar_inv_ctx_glob(dmar); 202 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)) 203 error = dmar_inv_iotlb_glob(dmar); 204 return (error); 205 } 206 207 static int 208 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, 209 int slot, int func, int dev_domain, int dev_busno, 210 const void *dev_path, int dev_path_len) 211 { 212 struct dmar_map_entries_tailq rmrr_entries; 213 struct dmar_map_entry *entry, *entry1; 214 vm_page_t *ma; 215 dmar_gaddr_t start, end; 216 vm_pindex_t size, i; 217 int error, error1; 218 219 error = 0; 220 TAILQ_INIT(&rmrr_entries); 221 dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path, 222 dev_path_len, &rmrr_entries); 223 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 224 /* 225 * VT-d specification requires that the start of an 226 * RMRR entry is 4k-aligned. Buggy BIOSes put 227 * anything into the start and end fields. Truncate 228 * and round as neccesary. 229 * 230 * We also allow the overlapping RMRR entries, see 231 * dmar_gas_alloc_region(). 232 */ 233 start = entry->start; 234 end = entry->end; 235 if (bootverbose) 236 printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", 237 domain->dmar->unit, bus, slot, func, 238 (uintmax_t)start, (uintmax_t)end); 239 entry->start = trunc_page(start); 240 entry->end = round_page(end); 241 if (entry->start == entry->end) { 242 /* Workaround for some AMI (?) BIOSes */ 243 if (bootverbose) { 244 if (dev != NULL) 245 device_printf(dev, ""); 246 printf("pci%d:%d:%d ", bus, slot, func); 247 printf("BIOS bug: dmar%d RMRR " 248 "region (%jx, %jx) corrected\n", 249 domain->dmar->unit, start, end); 250 } 251 entry->end += DMAR_PAGE_SIZE * 0x20; 252 } 253 size = OFF_TO_IDX(entry->end - entry->start); 254 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 255 for (i = 0; i < size; i++) { 256 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 257 VM_MEMATTR_DEFAULT); 258 } 259 error1 = dmar_gas_map_region(domain, entry, 260 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE, 261 DMAR_GM_CANWAIT, ma); 262 /* 263 * Non-failed RMRR entries are owned by context rb 264 * tree. Get rid of the failed entry, but do not stop 265 * the loop. Rest of the parsed RMRR entries are 266 * loaded and removed on the context destruction. 267 */ 268 if (error1 == 0 && entry->end != entry->start) { 269 DMAR_LOCK(domain->dmar); 270 domain->refs++; /* XXXKIB prevent free */ 271 domain->flags |= DMAR_DOMAIN_RMRR; 272 DMAR_UNLOCK(domain->dmar); 273 } else { 274 if (error1 != 0) { 275 if (dev != NULL) 276 device_printf(dev, ""); 277 printf("pci%d:%d:%d ", bus, slot, func); 278 printf( 279 "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 280 domain->dmar->unit, start, end, 281 error1); 282 error = error1; 283 } 284 TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 285 dmar_gas_free_entry(domain, entry); 286 } 287 for (i = 0; i < size; i++) 288 vm_page_putfake(ma[i]); 289 free(ma, M_TEMP); 290 } 291 return (error); 292 } 293 294 static struct dmar_domain * 295 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) 296 { 297 struct dmar_domain *domain; 298 int error, id, mgaw; 299 300 id = alloc_unr(dmar->domids); 301 if (id == -1) 302 return (NULL); 303 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); 304 domain->domain = id; 305 LIST_INIT(&domain->contexts); 306 RB_INIT(&domain->rb_root); 307 TAILQ_INIT(&domain->unload_entries); 308 TASK_INIT(&domain->unload_task, 0, dmar_domain_unload_task, domain); 309 mtx_init(&domain->lock, "dmardom", NULL, MTX_DEF); 310 domain->dmar = dmar; 311 312 /* 313 * For now, use the maximal usable physical address of the 314 * installed memory to calculate the mgaw on id_mapped domain. 315 * It is useful for the identity mapping, and less so for the 316 * virtualized bus address space. 317 */ 318 domain->end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; 319 mgaw = dmar_maxaddr2mgaw(dmar, domain->end, !id_mapped); 320 error = domain_set_agaw(domain, mgaw); 321 if (error != 0) 322 goto fail; 323 if (!id_mapped) 324 /* Use all supported address space for remapping. */ 325 domain->end = 1ULL << (domain->agaw - 1); 326 327 dmar_gas_init_domain(domain); 328 329 if (id_mapped) { 330 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 331 domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, 332 domain->end); 333 } 334 domain->flags |= DMAR_DOMAIN_IDMAP; 335 } else { 336 error = domain_alloc_pgtbl(domain); 337 if (error != 0) 338 goto fail; 339 /* Disable local apic region access */ 340 error = dmar_gas_reserve_region(domain, 0xfee00000, 341 0xfeefffff + 1); 342 if (error != 0) 343 goto fail; 344 } 345 return (domain); 346 347 fail: 348 dmar_domain_destroy(domain); 349 return (NULL); 350 } 351 352 static struct dmar_ctx * 353 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) 354 { 355 struct dmar_ctx *ctx; 356 357 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 358 ctx->domain = domain; 359 ctx->rid = rid; 360 ctx->refs = 1; 361 return (ctx); 362 } 363 364 static void 365 dmar_ctx_link(struct dmar_ctx *ctx) 366 { 367 struct dmar_domain *domain; 368 369 domain = ctx->domain; 370 DMAR_ASSERT_LOCKED(domain->dmar); 371 KASSERT(domain->refs >= domain->ctx_cnt, 372 ("dom %p ref underflow %d %d", domain, domain->refs, 373 domain->ctx_cnt)); 374 domain->refs++; 375 domain->ctx_cnt++; 376 LIST_INSERT_HEAD(&domain->contexts, ctx, link); 377 } 378 379 static void 380 dmar_ctx_unlink(struct dmar_ctx *ctx) 381 { 382 struct dmar_domain *domain; 383 384 domain = ctx->domain; 385 DMAR_ASSERT_LOCKED(domain->dmar); 386 KASSERT(domain->refs > 0, 387 ("domain %p ctx dtr refs %d", domain, domain->refs)); 388 KASSERT(domain->ctx_cnt >= domain->refs, 389 ("domain %p ctx dtr refs %d ctx_cnt %d", domain, 390 domain->refs, domain->ctx_cnt)); 391 domain->refs--; 392 domain->ctx_cnt--; 393 LIST_REMOVE(ctx, link); 394 } 395 396 static void 397 dmar_domain_destroy(struct dmar_domain *domain) 398 { 399 400 KASSERT(TAILQ_EMPTY(&domain->unload_entries), 401 ("unfinished unloads %p", domain)); 402 KASSERT(LIST_EMPTY(&domain->contexts), 403 ("destroying dom %p with contexts", domain)); 404 KASSERT(domain->ctx_cnt == 0, 405 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); 406 KASSERT(domain->refs == 0, 407 ("destroying dom %p with refs %d", domain, domain->refs)); 408 if ((domain->flags & DMAR_DOMAIN_GAS_INITED) != 0) { 409 DMAR_DOMAIN_LOCK(domain); 410 dmar_gas_fini_domain(domain); 411 DMAR_DOMAIN_UNLOCK(domain); 412 } 413 if ((domain->flags & DMAR_DOMAIN_PGTBL_INITED) != 0) { 414 if (domain->pgtbl_obj != NULL) 415 DMAR_DOMAIN_PGLOCK(domain); 416 domain_free_pgtbl(domain); 417 } 418 mtx_destroy(&domain->lock); 419 free_unr(domain->dmar->domids, domain->domain); 420 free(domain, M_DMAR_DOMAIN); 421 } 422 423 static struct dmar_ctx * 424 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, 425 int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, 426 bool id_mapped, bool rmrr_init) 427 { 428 struct dmar_domain *domain, *domain1; 429 struct dmar_ctx *ctx, *ctx1; 430 dmar_ctx_entry_t *ctxp; 431 struct sf_buf *sf; 432 int bus, slot, func, error; 433 bool enable; 434 435 if (dev != NULL) { 436 bus = pci_get_bus(dev); 437 slot = pci_get_slot(dev); 438 func = pci_get_function(dev); 439 } else { 440 bus = PCI_RID2BUS(rid); 441 slot = PCI_RID2SLOT(rid); 442 func = PCI_RID2FUNC(rid); 443 } 444 enable = false; 445 TD_PREP_PINNED_ASSERT; 446 DMAR_LOCK(dmar); 447 ctx = dmar_find_ctx_locked(dmar, rid); 448 error = 0; 449 if (ctx == NULL) { 450 /* 451 * Perform the allocations which require sleep or have 452 * higher chance to succeed if the sleep is allowed. 453 */ 454 DMAR_UNLOCK(dmar); 455 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 456 domain1 = dmar_domain_alloc(dmar, id_mapped); 457 if (domain1 == NULL) { 458 TD_PINNED_ASSERT; 459 return (NULL); 460 } 461 if (!id_mapped) { 462 error = domain_init_rmrr(domain1, dev, bus, 463 slot, func, dev_domain, dev_busno, dev_path, 464 dev_path_len); 465 if (error != 0) { 466 dmar_domain_destroy(domain1); 467 TD_PINNED_ASSERT; 468 return (NULL); 469 } 470 } 471 ctx1 = dmar_ctx_alloc(domain1, rid); 472 ctxp = dmar_map_ctx_entry(ctx1, &sf); 473 DMAR_LOCK(dmar); 474 475 /* 476 * Recheck the contexts, other thread might have 477 * already allocated needed one. 478 */ 479 ctx = dmar_find_ctx_locked(dmar, rid); 480 if (ctx == NULL) { 481 domain = domain1; 482 ctx = ctx1; 483 dmar_ctx_link(ctx); 484 ctx->ctx_tag.owner = dev; 485 ctx_tag_init(ctx, dev); 486 487 /* 488 * This is the first activated context for the 489 * DMAR unit. Enable the translation after 490 * everything is set up. 491 */ 492 if (LIST_EMPTY(&dmar->domains)) 493 enable = true; 494 LIST_INSERT_HEAD(&dmar->domains, domain, link); 495 ctx_id_entry_init(ctx, ctxp, false); 496 if (dev != NULL) { 497 device_printf(dev, 498 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " 499 "agaw %d %s-mapped\n", 500 dmar->unit, dmar->segment, bus, slot, 501 func, rid, domain->domain, domain->mgaw, 502 domain->agaw, id_mapped ? "id" : "re"); 503 } 504 dmar_unmap_pgtbl(sf); 505 } else { 506 dmar_unmap_pgtbl(sf); 507 dmar_domain_destroy(domain1); 508 /* Nothing needs to be done to destroy ctx1. */ 509 free(ctx1, M_DMAR_CTX); 510 domain = ctx->domain; 511 ctx->refs++; /* tag referenced us */ 512 } 513 } else { 514 domain = ctx->domain; 515 if (ctx->ctx_tag.owner == NULL) 516 ctx->ctx_tag.owner = dev; 517 ctx->refs++; /* tag referenced us */ 518 } 519 520 error = dmar_flush_for_ctx_entry(dmar, enable); 521 if (error != 0) { 522 dmar_free_ctx_locked(dmar, ctx); 523 TD_PINNED_ASSERT; 524 return (NULL); 525 } 526 527 /* 528 * The dmar lock was potentially dropped between check for the 529 * empty context list and now. Recheck the state of GCMD_TE 530 * to avoid unneeded command. 531 */ 532 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 533 error = dmar_enable_translation(dmar); 534 if (error == 0) { 535 if (bootverbose) { 536 printf("dmar%d: enabled translation\n", 537 dmar->unit); 538 } 539 } else { 540 printf("dmar%d: enabling translation failed, " 541 "error %d\n", dmar->unit, error); 542 dmar_free_ctx_locked(dmar, ctx); 543 TD_PINNED_ASSERT; 544 return (NULL); 545 } 546 } 547 DMAR_UNLOCK(dmar); 548 TD_PINNED_ASSERT; 549 return (ctx); 550 } 551 552 struct dmar_ctx * 553 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, 554 bool id_mapped, bool rmrr_init) 555 { 556 int dev_domain, dev_path_len, dev_busno; 557 558 dev_domain = pci_get_domain(dev); 559 dev_path_len = dmar_dev_depth(dev); 560 ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; 561 dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); 562 return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno, 563 dev_path, dev_path_len, id_mapped, rmrr_init)); 564 } 565 566 struct dmar_ctx * 567 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, 568 int dev_domain, int dev_busno, 569 const void *dev_path, int dev_path_len, 570 bool id_mapped, bool rmrr_init) 571 { 572 573 return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno, 574 dev_path, dev_path_len, id_mapped, rmrr_init)); 575 } 576 577 int 578 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) 579 { 580 struct dmar_unit *dmar; 581 struct dmar_domain *old_domain; 582 dmar_ctx_entry_t *ctxp; 583 struct sf_buf *sf; 584 int error; 585 586 dmar = domain->dmar; 587 old_domain = ctx->domain; 588 if (domain == old_domain) 589 return (0); 590 KASSERT(old_domain->dmar == dmar, 591 ("domain %p %u moving between dmars %u %u", domain, 592 domain->domain, old_domain->dmar->unit, domain->dmar->unit)); 593 TD_PREP_PINNED_ASSERT; 594 595 ctxp = dmar_map_ctx_entry(ctx, &sf); 596 DMAR_LOCK(dmar); 597 dmar_ctx_unlink(ctx); 598 ctx->domain = domain; 599 dmar_ctx_link(ctx); 600 ctx_id_entry_init(ctx, ctxp, true); 601 dmar_unmap_pgtbl(sf); 602 error = dmar_flush_for_ctx_entry(dmar, true); 603 /* If flush failed, rolling back would not work as well. */ 604 printf("dmar%d rid %x domain %d->%d %s-mapped\n", 605 dmar->unit, ctx->rid, old_domain->domain, domain->domain, 606 (domain->flags & DMAR_DOMAIN_IDMAP) != 0 ? "id" : "re"); 607 dmar_unref_domain_locked(dmar, old_domain); 608 TD_PINNED_ASSERT; 609 return (error); 610 } 611 612 static void 613 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) 614 { 615 616 DMAR_ASSERT_LOCKED(dmar); 617 KASSERT(domain->refs >= 1, 618 ("dmar %d domain %p refs %u", dmar->unit, domain, domain->refs)); 619 KASSERT(domain->refs > domain->ctx_cnt, 620 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->unit, domain, 621 domain->refs, domain->ctx_cnt)); 622 623 if (domain->refs > 1) { 624 domain->refs--; 625 DMAR_UNLOCK(dmar); 626 return; 627 } 628 629 KASSERT((domain->flags & DMAR_DOMAIN_RMRR) == 0, 630 ("lost ref on RMRR domain %p", domain)); 631 632 LIST_REMOVE(domain, link); 633 DMAR_UNLOCK(dmar); 634 635 taskqueue_drain(dmar->delayed_taskqueue, &domain->unload_task); 636 dmar_domain_destroy(domain); 637 } 638 639 void 640 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 641 { 642 struct sf_buf *sf; 643 dmar_ctx_entry_t *ctxp; 644 struct dmar_domain *domain; 645 646 DMAR_ASSERT_LOCKED(dmar); 647 KASSERT(ctx->refs >= 1, 648 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 649 650 /* 651 * If our reference is not last, only the dereference should 652 * be performed. 653 */ 654 if (ctx->refs > 1) { 655 ctx->refs--; 656 DMAR_UNLOCK(dmar); 657 return; 658 } 659 660 KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 661 ("lost ref on disabled ctx %p", ctx)); 662 663 /* 664 * Otherwise, the context entry must be cleared before the 665 * page table is destroyed. The mapping of the context 666 * entries page could require sleep, unlock the dmar. 667 */ 668 DMAR_UNLOCK(dmar); 669 TD_PREP_PINNED_ASSERT; 670 ctxp = dmar_map_ctx_entry(ctx, &sf); 671 DMAR_LOCK(dmar); 672 KASSERT(ctx->refs >= 1, 673 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 674 675 /* 676 * Other thread might have referenced the context, in which 677 * case again only the dereference should be performed. 678 */ 679 if (ctx->refs > 1) { 680 ctx->refs--; 681 DMAR_UNLOCK(dmar); 682 dmar_unmap_pgtbl(sf); 683 TD_PINNED_ASSERT; 684 return; 685 } 686 687 KASSERT((ctx->flags & DMAR_CTX_DISABLED) == 0, 688 ("lost ref on disabled ctx %p", ctx)); 689 690 /* 691 * Clear the context pointer and flush the caches. 692 * XXXKIB: cannot do this if any RMRR entries are still present. 693 */ 694 dmar_pte_clear(&ctxp->ctx1); 695 ctxp->ctx2 = 0; 696 dmar_flush_ctx_to_ram(dmar, ctxp); 697 dmar_inv_ctx_glob(dmar); 698 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 699 if (dmar->qi_enabled) 700 dmar_qi_invalidate_iotlb_glob_locked(dmar); 701 else 702 dmar_inv_iotlb_glob(dmar); 703 } 704 dmar_unmap_pgtbl(sf); 705 domain = ctx->domain; 706 dmar_ctx_unlink(ctx); 707 free(ctx, M_DMAR_CTX); 708 dmar_unref_domain_locked(dmar, domain); 709 TD_PINNED_ASSERT; 710 } 711 712 void 713 dmar_free_ctx(struct dmar_ctx *ctx) 714 { 715 struct dmar_unit *dmar; 716 717 dmar = ctx->domain->dmar; 718 DMAR_LOCK(dmar); 719 dmar_free_ctx_locked(dmar, ctx); 720 } 721 722 /* 723 * Returns with the domain locked. 724 */ 725 struct dmar_ctx * 726 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 727 { 728 struct dmar_domain *domain; 729 struct dmar_ctx *ctx; 730 731 DMAR_ASSERT_LOCKED(dmar); 732 733 LIST_FOREACH(domain, &dmar->domains, link) { 734 LIST_FOREACH(ctx, &domain->contexts, link) { 735 if (ctx->rid == rid) 736 return (ctx); 737 } 738 } 739 return (NULL); 740 } 741 742 void 743 dmar_domain_free_entry(struct dmar_map_entry *entry, bool free) 744 { 745 struct dmar_domain *domain; 746 747 domain = entry->domain; 748 DMAR_DOMAIN_LOCK(domain); 749 if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0) 750 dmar_gas_free_region(domain, entry); 751 else 752 dmar_gas_free_space(domain, entry); 753 DMAR_DOMAIN_UNLOCK(domain); 754 if (free) 755 dmar_gas_free_entry(domain, entry); 756 else 757 entry->flags = 0; 758 } 759 760 void 761 dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free) 762 { 763 struct dmar_unit *unit; 764 765 unit = entry->domain->dmar; 766 if (unit->qi_enabled) { 767 DMAR_LOCK(unit); 768 dmar_qi_invalidate_locked(entry->domain, entry->start, 769 entry->end - entry->start, &entry->gseq, true); 770 if (!free) 771 entry->flags |= DMAR_MAP_ENTRY_QI_NF; 772 TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 773 DMAR_UNLOCK(unit); 774 } else { 775 domain_flush_iotlb_sync(entry->domain, entry->start, 776 entry->end - entry->start); 777 dmar_domain_free_entry(entry, free); 778 } 779 } 780 781 static bool 782 dmar_domain_unload_emit_wait(struct dmar_domain *domain, 783 struct dmar_map_entry *entry) 784 { 785 786 if (TAILQ_NEXT(entry, dmamap_link) == NULL) 787 return (true); 788 return (domain->batch_no++ % dmar_batch_coalesce == 0); 789 } 790 791 void 792 dmar_domain_unload(struct dmar_domain *domain, 793 struct dmar_map_entries_tailq *entries, bool cansleep) 794 { 795 struct dmar_unit *unit; 796 struct dmar_map_entry *entry, *entry1; 797 int error; 798 799 unit = domain->dmar; 800 801 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 802 KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0, 803 ("not mapped entry %p %p", domain, entry)); 804 error = domain_unmap_buf(domain, entry->start, entry->end - 805 entry->start, cansleep ? DMAR_PGF_WAITOK : 0); 806 KASSERT(error == 0, ("unmap %p error %d", domain, error)); 807 if (!unit->qi_enabled) { 808 domain_flush_iotlb_sync(domain, entry->start, 809 entry->end - entry->start); 810 TAILQ_REMOVE(entries, entry, dmamap_link); 811 dmar_domain_free_entry(entry, true); 812 } 813 } 814 if (TAILQ_EMPTY(entries)) 815 return; 816 817 KASSERT(unit->qi_enabled, ("loaded entry left")); 818 DMAR_LOCK(unit); 819 TAILQ_FOREACH(entry, entries, dmamap_link) { 820 dmar_qi_invalidate_locked(domain, entry->start, entry->end - 821 entry->start, &entry->gseq, 822 dmar_domain_unload_emit_wait(domain, entry)); 823 } 824 TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link); 825 DMAR_UNLOCK(unit); 826 } 827 828 static void 829 dmar_domain_unload_task(void *arg, int pending) 830 { 831 struct dmar_domain *domain; 832 struct dmar_map_entries_tailq entries; 833 834 domain = arg; 835 TAILQ_INIT(&entries); 836 837 for (;;) { 838 DMAR_DOMAIN_LOCK(domain); 839 TAILQ_SWAP(&domain->unload_entries, &entries, dmar_map_entry, 840 dmamap_link); 841 DMAR_DOMAIN_UNLOCK(domain); 842 if (TAILQ_EMPTY(&entries)) 843 break; 844 dmar_domain_unload(domain, &entries, true); 845 } 846 } 847