1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020-2021 Ruslan Bukin <br@bsdpad.com> 5 * Copyright (c) 2014-2021 Andrew Turner 6 * Copyright (c) 2014-2016 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * This work was supported by Innovate UK project 105694, "Digital Security 10 * by Design (DSbD) Technology Platform Prototype". 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 /* 38 * Manages physical address maps for ARM SMMUv3 and ARM Mali GPU. 39 */ 40 41 #include "opt_vm.h" 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/ktr.h> 46 #include <sys/lock.h> 47 #include <sys/mutex.h> 48 #include <sys/rwlock.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_param.h> 52 #include <vm/vm_page.h> 53 #include <vm/vm_map.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_pageout.h> 56 #include <vm/vm_radix.h> 57 58 #include <machine/machdep.h> 59 60 #include <arm64/iommu/iommu_pmap.h> 61 #include <arm64/iommu/iommu_pte.h> 62 63 #define IOMMU_PAGE_SIZE 4096 64 65 #define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx) 66 #define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx) 67 #define SMMU_PMAP_LOCK_ASSERT(pmap, type) \ 68 mtx_assert(&(pmap)->sp_mtx, (type)) 69 70 #define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 71 #define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 72 #define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 73 #define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t))) 74 75 #define NUL0E IOMMU_L0_ENTRIES 76 #define NUL1E (NUL0E * NL1PG) 77 #define NUL2E (NUL1E * NL2PG) 78 79 #define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) 80 #define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) 81 #define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) 82 83 #define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK) 84 #define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK) 85 #define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK) 86 #define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK) 87 88 static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex); 89 static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, 90 vm_page_t m, struct spglist *free); 91 92 /* 93 * These load the old table data and store the new value. 94 * They need to be atomic as the System MMU may write to the table at 95 * the same time as the CPU. 96 */ 97 #define smmu_pmap_load(table) (*table) 98 #define smmu_pmap_clear(table) atomic_store_64(table, 0) 99 #define smmu_pmap_store(table, entry) atomic_store_64(table, entry) 100 101 /********************/ 102 /* Inline functions */ 103 /********************/ 104 105 static __inline pd_entry_t * 106 smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va) 107 { 108 109 return (&pmap->sp_l0[smmu_l0_index(va)]); 110 } 111 112 static __inline pd_entry_t * 113 smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) 114 { 115 pd_entry_t *l1; 116 117 l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK); 118 return (&l1[smmu_l1_index(va)]); 119 } 120 121 static __inline pd_entry_t * 122 smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va) 123 { 124 pd_entry_t *l0; 125 126 l0 = smmu_pmap_l0(pmap, va); 127 if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) 128 return (NULL); 129 130 return (smmu_pmap_l0_to_l1(l0, va)); 131 } 132 133 static __inline pd_entry_t * 134 smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) 135 { 136 pd_entry_t l1, *l2p; 137 138 l1 = smmu_pmap_load(l1p); 139 140 /* 141 * The valid bit may be clear if pmap_update_entry() is concurrently 142 * modifying the entry, so for KVA only the entry type may be checked. 143 */ 144 KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, 145 ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); 146 KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 147 ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); 148 l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); 149 return (&l2p[smmu_l2_index(va)]); 150 } 151 152 static __inline pd_entry_t * 153 smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va) 154 { 155 pd_entry_t *l1; 156 157 l1 = smmu_pmap_l1(pmap, va); 158 if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) 159 return (NULL); 160 161 return (smmu_pmap_l1_to_l2(l1, va)); 162 } 163 164 static __inline pt_entry_t * 165 smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) 166 { 167 pd_entry_t l2; 168 pt_entry_t *l3p; 169 170 l2 = smmu_pmap_load(l2p); 171 172 /* 173 * The valid bit may be clear if pmap_update_entry() is concurrently 174 * modifying the entry, so for KVA only the entry type may be checked. 175 */ 176 KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, 177 ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); 178 KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 179 ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); 180 l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); 181 return (&l3p[smmu_l3_index(va)]); 182 } 183 184 /* 185 * Returns the lowest valid pde for a given virtual address. 186 * The next level may or may not point to a valid page or block. 187 */ 188 static __inline pd_entry_t * 189 smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level) 190 { 191 pd_entry_t *l0, *l1, *l2, desc; 192 193 l0 = smmu_pmap_l0(pmap, va); 194 desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK; 195 if (desc != IOMMU_L0_TABLE) { 196 *level = -1; 197 return (NULL); 198 } 199 200 l1 = smmu_pmap_l0_to_l1(l0, va); 201 desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; 202 if (desc != IOMMU_L1_TABLE) { 203 *level = 0; 204 return (l0); 205 } 206 207 l2 = smmu_pmap_l1_to_l2(l1, va); 208 desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; 209 if (desc != IOMMU_L2_TABLE) { 210 *level = 1; 211 return (l1); 212 } 213 214 *level = 2; 215 return (l2); 216 } 217 218 /* 219 * Returns the lowest valid pte block or table entry for a given virtual 220 * address. If there are no valid entries return NULL and set the level to 221 * the first invalid level. 222 */ 223 static __inline pt_entry_t * 224 smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level) 225 { 226 pd_entry_t *l1, *l2, desc; 227 pt_entry_t *l3; 228 229 l1 = smmu_pmap_l1(pmap, va); 230 if (l1 == NULL) { 231 *level = 0; 232 return (NULL); 233 } 234 desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; 235 if (desc == IOMMU_L1_BLOCK) { 236 *level = 1; 237 return (l1); 238 } 239 240 if (desc != IOMMU_L1_TABLE) { 241 *level = 1; 242 return (NULL); 243 } 244 245 l2 = smmu_pmap_l1_to_l2(l1, va); 246 desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; 247 if (desc == IOMMU_L2_BLOCK) { 248 *level = 2; 249 return (l2); 250 } 251 252 if (desc != IOMMU_L2_TABLE) { 253 *level = 2; 254 return (NULL); 255 } 256 257 *level = 3; 258 l3 = smmu_pmap_l2_to_l3(l2, va); 259 if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) 260 return (NULL); 261 262 return (l3); 263 } 264 265 static __inline int 266 smmu_pmap_l3_valid(pt_entry_t l3) 267 { 268 269 return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE); 270 } 271 272 CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK); 273 274 #ifdef INVARIANTS 275 static __inline void 276 smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) 277 { 278 279 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 280 pmap->sp_resident_count += count; 281 } 282 283 static __inline void 284 smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) 285 { 286 287 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 288 KASSERT(pmap->sp_resident_count >= count, 289 ("pmap %p resident count underflow %ld %d", pmap, 290 pmap->sp_resident_count, count)); 291 pmap->sp_resident_count -= count; 292 } 293 #else 294 static __inline void 295 smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) 296 { 297 } 298 299 static __inline void 300 smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) 301 { 302 } 303 #endif 304 305 /*************************************************** 306 * Page table page management routines..... 307 ***************************************************/ 308 /* 309 * Schedule the specified unused page table page to be freed. Specifically, 310 * add the page to the specified list of pages that will be released to the 311 * physical memory manager after the TLB has been updated. 312 */ 313 static __inline void 314 smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 315 boolean_t set_PG_ZERO) 316 { 317 318 if (set_PG_ZERO) 319 m->flags |= PG_ZERO; 320 else 321 m->flags &= ~PG_ZERO; 322 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 323 } 324 325 /*************************************************** 326 * Low level mapping routines..... 327 ***************************************************/ 328 329 /* 330 * Decrements a page table page's reference count, which is used to record the 331 * number of valid page table entries within the page. If the reference count 332 * drops to zero, then the page table page is unmapped. Returns TRUE if the 333 * page table page was unmapped and FALSE otherwise. 334 */ 335 static inline boolean_t 336 smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, 337 struct spglist *free) 338 { 339 340 --m->ref_count; 341 if (m->ref_count == 0) { 342 _smmu_pmap_unwire_l3(pmap, va, m, free); 343 return (TRUE); 344 } else 345 return (FALSE); 346 } 347 348 static void 349 _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, 350 struct spglist *free) 351 { 352 353 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 354 /* 355 * unmap the page table page 356 */ 357 if (m->pindex >= (NUL2E + NUL1E)) { 358 /* l1 page */ 359 pd_entry_t *l0; 360 361 l0 = smmu_pmap_l0(pmap, va); 362 smmu_pmap_clear(l0); 363 } else if (m->pindex >= NUL2E) { 364 /* l2 page */ 365 pd_entry_t *l1; 366 367 l1 = smmu_pmap_l1(pmap, va); 368 smmu_pmap_clear(l1); 369 } else { 370 /* l3 page */ 371 pd_entry_t *l2; 372 373 l2 = smmu_pmap_l2(pmap, va); 374 smmu_pmap_clear(l2); 375 } 376 smmu_pmap_resident_count_dec(pmap, 1); 377 if (m->pindex < NUL2E) { 378 /* We just released an l3, unhold the matching l2 */ 379 pd_entry_t *l1, tl1; 380 vm_page_t l2pg; 381 382 l1 = smmu_pmap_l1(pmap, va); 383 tl1 = smmu_pmap_load(l1); 384 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 385 smmu_pmap_unwire_l3(pmap, va, l2pg, free); 386 } else if (m->pindex < (NUL2E + NUL1E)) { 387 /* We just released an l2, unhold the matching l1 */ 388 pd_entry_t *l0, tl0; 389 vm_page_t l1pg; 390 391 l0 = smmu_pmap_l0(pmap, va); 392 tl0 = smmu_pmap_load(l0); 393 l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 394 smmu_pmap_unwire_l3(pmap, va, l1pg, free); 395 } 396 397 /* 398 * Put page on a list so that it is released after 399 * *ALL* TLB shootdown is done 400 */ 401 smmu_pmap_add_delayed_free_list(m, free, TRUE); 402 } 403 404 int 405 smmu_pmap_pinit(struct smmu_pmap *pmap) 406 { 407 vm_page_t m; 408 409 /* 410 * allocate the l0 page 411 */ 412 m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED | 413 VM_ALLOC_ZERO); 414 pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m); 415 pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr); 416 417 pmap->sp_resident_count = 0; 418 mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF); 419 420 return (1); 421 } 422 423 /* 424 * This routine is called if the desired page table page does not exist. 425 * 426 * If page table page allocation fails, this routine may sleep before 427 * returning NULL. It sleeps only if a lock pointer was given. 428 * 429 * Note: If a page allocation fails at page table level two or three, 430 * one or two pages may be held during the wait, only to be released 431 * afterwards. This conservative approach is easily argued to avoid 432 * race conditions. 433 */ 434 static vm_page_t 435 _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex) 436 { 437 vm_page_t m, l1pg, l2pg; 438 439 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 440 441 /* 442 * Allocate a page table page. 443 */ 444 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 445 /* 446 * Indicate the need to retry. While waiting, the page table 447 * page may have been allocated. 448 */ 449 return (NULL); 450 } 451 m->pindex = ptepindex; 452 453 /* 454 * Because of AArch64's weak memory consistency model, we must have a 455 * barrier here to ensure that the stores for zeroing "m", whether by 456 * pmap_zero_page() or an earlier function, are visible before adding 457 * "m" to the page table. Otherwise, a page table walk by another 458 * processor's MMU could see the mapping to "m" and a stale, non-zero 459 * PTE within "m". 460 */ 461 dmb(ishst); 462 463 /* 464 * Map the pagetable page into the process address space, if 465 * it isn't already there. 466 */ 467 468 if (ptepindex >= (NUL2E + NUL1E)) { 469 pd_entry_t *l0; 470 vm_pindex_t l0index; 471 472 l0index = ptepindex - (NUL2E + NUL1E); 473 l0 = &pmap->sp_l0[l0index]; 474 smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); 475 } else if (ptepindex >= NUL2E) { 476 vm_pindex_t l0index, l1index; 477 pd_entry_t *l0, *l1; 478 pd_entry_t tl0; 479 480 l1index = ptepindex - NUL2E; 481 l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 482 483 l0 = &pmap->sp_l0[l0index]; 484 tl0 = smmu_pmap_load(l0); 485 if (tl0 == 0) { 486 /* recurse for allocating page dir */ 487 if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index) 488 == NULL) { 489 vm_page_unwire_noq(m); 490 vm_page_free_zero(m); 491 return (NULL); 492 } 493 } else { 494 l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 495 l1pg->ref_count++; 496 } 497 498 l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK); 499 l1 = &l1[ptepindex & Ln_ADDR_MASK]; 500 smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); 501 } else { 502 vm_pindex_t l0index, l1index; 503 pd_entry_t *l0, *l1, *l2; 504 pd_entry_t tl0, tl1; 505 506 l1index = ptepindex >> IOMMU_Ln_ENTRIES_SHIFT; 507 l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 508 509 l0 = &pmap->sp_l0[l0index]; 510 tl0 = smmu_pmap_load(l0); 511 if (tl0 == 0) { 512 /* recurse for allocating page dir */ 513 if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { 514 vm_page_unwire_noq(m); 515 vm_page_free_zero(m); 516 return (NULL); 517 } 518 tl0 = smmu_pmap_load(l0); 519 l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 520 l1 = &l1[l1index & Ln_ADDR_MASK]; 521 } else { 522 l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 523 l1 = &l1[l1index & Ln_ADDR_MASK]; 524 tl1 = smmu_pmap_load(l1); 525 if (tl1 == 0) { 526 /* recurse for allocating page dir */ 527 if (_pmap_alloc_l3(pmap, NUL2E + l1index) 528 == NULL) { 529 vm_page_unwire_noq(m); 530 vm_page_free_zero(m); 531 return (NULL); 532 } 533 } else { 534 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 535 l2pg->ref_count++; 536 } 537 } 538 539 l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK); 540 l2 = &l2[ptepindex & Ln_ADDR_MASK]; 541 smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); 542 } 543 544 smmu_pmap_resident_count_inc(pmap, 1); 545 546 return (m); 547 } 548 549 /*************************************************** 550 * Pmap allocation/deallocation routines. 551 ***************************************************/ 552 553 /* 554 * Release any resources held by the given physical map. 555 * Called when a pmap initialized by pmap_pinit is being released. 556 * Should only be called if the map contains no valid mappings. 557 */ 558 void 559 smmu_pmap_release(struct smmu_pmap *pmap) 560 { 561 vm_page_t m; 562 563 KASSERT(pmap->sp_resident_count == 0, 564 ("pmap_release: pmap resident count %ld != 0", 565 pmap->sp_resident_count)); 566 567 m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr); 568 vm_page_unwire_noq(m); 569 vm_page_free_zero(m); 570 mtx_destroy(&pmap->sp_mtx); 571 } 572 573 /*************************************************** 574 * page management routines. 575 ***************************************************/ 576 577 /* 578 * Add a single Mali GPU entry. This function does not sleep. 579 */ 580 int 581 pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, 582 vm_prot_t prot, u_int flags) 583 { 584 pd_entry_t *pde; 585 pt_entry_t new_l3; 586 pt_entry_t orig_l3 __diagused; 587 pt_entry_t *l3; 588 vm_page_t mpte; 589 pd_entry_t *l1p; 590 pd_entry_t *l2p; 591 int lvl; 592 int rv; 593 594 KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 595 KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 596 KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); 597 598 new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK); 599 600 if ((prot & VM_PROT_WRITE) != 0) 601 new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 602 if ((prot & VM_PROT_READ) != 0) 603 new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); 604 if ((prot & VM_PROT_EXECUTE) == 0) 605 new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL); 606 607 CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa); 608 609 SMMU_PMAP_LOCK(pmap); 610 611 /* 612 * In the case that a page table page is not 613 * resident, we are creating it here. 614 */ 615 retry: 616 pde = smmu_pmap_pde(pmap, va, &lvl); 617 if (pde != NULL && lvl == 2) { 618 l3 = smmu_pmap_l2_to_l3(pde, va); 619 } else { 620 mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); 621 if (mpte == NULL) { 622 CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 623 rv = KERN_RESOURCE_SHORTAGE; 624 goto out; 625 } 626 627 /* 628 * Ensure newly created l1, l2 are visible to GPU. 629 * l0 is already visible by similar call in panfrost driver. 630 * The cache entry for l3 handled below. 631 */ 632 633 l1p = smmu_pmap_l1(pmap, va); 634 l2p = smmu_pmap_l2(pmap, va); 635 cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t)); 636 cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t)); 637 638 goto retry; 639 } 640 641 orig_l3 = smmu_pmap_load(l3); 642 KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); 643 644 /* New mapping */ 645 smmu_pmap_store(l3, new_l3); 646 647 cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t)); 648 649 smmu_pmap_resident_count_inc(pmap, 1); 650 dsb(ishst); 651 652 rv = KERN_SUCCESS; 653 out: 654 SMMU_PMAP_UNLOCK(pmap); 655 656 return (rv); 657 } 658 659 /* 660 * Remove a single Mali GPU entry. 661 */ 662 int 663 pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va) 664 { 665 pd_entry_t *pde; 666 pt_entry_t *pte; 667 int lvl; 668 int rc; 669 670 KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 671 672 SMMU_PMAP_LOCK(pmap); 673 674 pde = smmu_pmap_pde(pmap, va, &lvl); 675 if (pde == NULL || lvl != 2) { 676 rc = KERN_FAILURE; 677 goto out; 678 } 679 680 pte = smmu_pmap_l2_to_l3(pde, va); 681 682 smmu_pmap_resident_count_dec(pmap, 1); 683 smmu_pmap_clear(pte); 684 cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t)); 685 rc = KERN_SUCCESS; 686 687 out: 688 SMMU_PMAP_UNLOCK(pmap); 689 690 return (rc); 691 } 692 693 /* 694 * Add a single SMMU entry. This function does not sleep. 695 */ 696 int 697 smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, 698 vm_prot_t prot, u_int flags) 699 { 700 pd_entry_t *pde; 701 pt_entry_t new_l3; 702 pt_entry_t orig_l3 __diagused; 703 pt_entry_t *l3; 704 vm_page_t mpte; 705 int lvl; 706 int rv; 707 708 KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 709 710 va = trunc_page(va); 711 new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | 712 ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE); 713 if ((prot & VM_PROT_WRITE) == 0) 714 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); 715 new_l3 |= ATTR_S1_XN; /* Execute never. */ 716 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER); 717 new_l3 |= ATTR_S1_nG; /* Non global. */ 718 719 CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa); 720 721 SMMU_PMAP_LOCK(pmap); 722 723 /* 724 * In the case that a page table page is not 725 * resident, we are creating it here. 726 */ 727 retry: 728 pde = smmu_pmap_pde(pmap, va, &lvl); 729 if (pde != NULL && lvl == 2) { 730 l3 = smmu_pmap_l2_to_l3(pde, va); 731 } else { 732 mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); 733 if (mpte == NULL) { 734 CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 735 rv = KERN_RESOURCE_SHORTAGE; 736 goto out; 737 } 738 goto retry; 739 } 740 741 orig_l3 = smmu_pmap_load(l3); 742 KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); 743 744 /* New mapping */ 745 smmu_pmap_store(l3, new_l3); 746 smmu_pmap_resident_count_inc(pmap, 1); 747 dsb(ishst); 748 749 rv = KERN_SUCCESS; 750 out: 751 SMMU_PMAP_UNLOCK(pmap); 752 753 return (rv); 754 } 755 756 /* 757 * Remove a single SMMU entry. 758 */ 759 int 760 smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va) 761 { 762 pt_entry_t *pte; 763 int lvl; 764 int rc; 765 766 SMMU_PMAP_LOCK(pmap); 767 768 pte = smmu_pmap_pte(pmap, va, &lvl); 769 KASSERT(lvl == 3, 770 ("Invalid SMMU pagetable level: %d != 3", lvl)); 771 772 if (pte != NULL) { 773 smmu_pmap_resident_count_dec(pmap, 1); 774 smmu_pmap_clear(pte); 775 rc = KERN_SUCCESS; 776 } else 777 rc = KERN_FAILURE; 778 779 SMMU_PMAP_UNLOCK(pmap); 780 781 return (rc); 782 } 783 784 /* 785 * Remove all the allocated L1, L2 pages from SMMU pmap. 786 * All the L3 entires must be cleared in advance, otherwise 787 * this function panics. 788 */ 789 void 790 smmu_pmap_remove_pages(struct smmu_pmap *pmap) 791 { 792 pd_entry_t l0e, *l1, l1e, *l2, l2e; 793 pt_entry_t *l3, l3e; 794 vm_page_t m, m0, m1; 795 vm_paddr_t pa; 796 vm_paddr_t pa0; 797 vm_paddr_t pa1; 798 int i, j, k, l; 799 800 SMMU_PMAP_LOCK(pmap); 801 802 for (i = 0; i < IOMMU_L0_ENTRIES; i++) { 803 l0e = pmap->sp_l0[i]; 804 if ((l0e & ATTR_DESCR_VALID) == 0) { 805 continue; 806 } 807 pa0 = l0e & ~ATTR_MASK; 808 m0 = PHYS_TO_VM_PAGE(pa0); 809 l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0); 810 811 for (j = 0; j < IOMMU_Ln_ENTRIES; j++) { 812 l1e = l1[j]; 813 if ((l1e & ATTR_DESCR_VALID) == 0) { 814 continue; 815 } 816 if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) { 817 continue; 818 } 819 pa1 = l1e & ~ATTR_MASK; 820 m1 = PHYS_TO_VM_PAGE(pa1); 821 l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1); 822 823 for (k = 0; k < IOMMU_Ln_ENTRIES; k++) { 824 l2e = l2[k]; 825 if ((l2e & ATTR_DESCR_VALID) == 0) { 826 continue; 827 } 828 pa = l2e & ~ATTR_MASK; 829 m = PHYS_TO_VM_PAGE(pa); 830 l3 = (pt_entry_t *)PHYS_TO_DMAP(pa); 831 832 for (l = 0; l < IOMMU_Ln_ENTRIES; l++) { 833 l3e = l3[l]; 834 if ((l3e & ATTR_DESCR_VALID) == 0) 835 continue; 836 panic( 837 "%s: l3e found (indexes %d %d %d %d)", 838 __func__, i, j, k, l); 839 } 840 841 vm_page_unwire_noq(m1); 842 vm_page_unwire_noq(m); 843 smmu_pmap_resident_count_dec(pmap, 1); 844 vm_page_free(m); 845 smmu_pmap_clear(&l2[k]); 846 } 847 848 vm_page_unwire_noq(m0); 849 smmu_pmap_resident_count_dec(pmap, 1); 850 vm_page_free(m1); 851 smmu_pmap_clear(&l1[j]); 852 } 853 854 smmu_pmap_resident_count_dec(pmap, 1); 855 vm_page_free(m0); 856 smmu_pmap_clear(&pmap->sp_l0[i]); 857 } 858 859 KASSERT(pmap->sp_resident_count == 0, 860 ("Invalid resident count %jd", pmap->sp_resident_count)); 861 862 SMMU_PMAP_UNLOCK(pmap); 863 } 864