1*f17c4e38SRuslan Bukin /*- 2*f17c4e38SRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause 3*f17c4e38SRuslan Bukin * 4*f17c4e38SRuslan Bukin * Copyright (c) 2020-2021 Ruslan Bukin <br@bsdpad.com> 5*f17c4e38SRuslan Bukin * Copyright (c) 2014-2021 Andrew Turner 6*f17c4e38SRuslan Bukin * Copyright (c) 2014-2016 The FreeBSD Foundation 7*f17c4e38SRuslan Bukin * All rights reserved. 8*f17c4e38SRuslan Bukin * 9*f17c4e38SRuslan Bukin * This work was supported by Innovate UK project 105694, "Digital Security 10*f17c4e38SRuslan Bukin * by Design (DSbD) Technology Platform Prototype". 11*f17c4e38SRuslan Bukin * 12*f17c4e38SRuslan Bukin * Redistribution and use in source and binary forms, with or without 13*f17c4e38SRuslan Bukin * modification, are permitted provided that the following conditions 14*f17c4e38SRuslan Bukin * are met: 15*f17c4e38SRuslan Bukin * 1. Redistributions of source code must retain the above copyright 16*f17c4e38SRuslan Bukin * notice, this list of conditions and the following disclaimer. 17*f17c4e38SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright 18*f17c4e38SRuslan Bukin * notice, this list of conditions and the following disclaimer in the 19*f17c4e38SRuslan Bukin * documentation and/or other materials provided with the distribution. 20*f17c4e38SRuslan Bukin * 21*f17c4e38SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22*f17c4e38SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23*f17c4e38SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24*f17c4e38SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25*f17c4e38SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26*f17c4e38SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27*f17c4e38SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28*f17c4e38SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29*f17c4e38SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30*f17c4e38SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31*f17c4e38SRuslan Bukin * SUCH DAMAGE. 32*f17c4e38SRuslan Bukin */ 33*f17c4e38SRuslan Bukin 34*f17c4e38SRuslan Bukin #include <sys/cdefs.h> 35*f17c4e38SRuslan Bukin __FBSDID("$FreeBSD$"); 36*f17c4e38SRuslan Bukin 37*f17c4e38SRuslan Bukin /* 38*f17c4e38SRuslan Bukin * Manages physical address maps for ARM SMMUv3 and ARM Mali GPU. 39*f17c4e38SRuslan Bukin */ 40*f17c4e38SRuslan Bukin 41*f17c4e38SRuslan Bukin #include "opt_vm.h" 42*f17c4e38SRuslan Bukin 43*f17c4e38SRuslan Bukin #include <sys/param.h> 44*f17c4e38SRuslan Bukin #include <sys/ktr.h> 45*f17c4e38SRuslan Bukin #include <sys/mutex.h> 46*f17c4e38SRuslan Bukin #include <sys/rwlock.h> 47*f17c4e38SRuslan Bukin 48*f17c4e38SRuslan Bukin #include <vm/vm.h> 49*f17c4e38SRuslan Bukin #include <vm/vm_param.h> 50*f17c4e38SRuslan Bukin #include <vm/vm_page.h> 51*f17c4e38SRuslan Bukin #include <vm/vm_map.h> 52*f17c4e38SRuslan Bukin #include <vm/vm_object.h> 53*f17c4e38SRuslan Bukin #include <vm/vm_pageout.h> 54*f17c4e38SRuslan Bukin #include <vm/vm_radix.h> 55*f17c4e38SRuslan Bukin 56*f17c4e38SRuslan Bukin #include <machine/machdep.h> 57*f17c4e38SRuslan Bukin 58*f17c4e38SRuslan Bukin #include <arm64/iommu/iommu_pmap.h> 59*f17c4e38SRuslan Bukin #include <arm64/iommu/iommu_pte.h> 60*f17c4e38SRuslan Bukin 61*f17c4e38SRuslan Bukin #define IOMMU_PAGE_SIZE 4096 62*f17c4e38SRuslan Bukin 63*f17c4e38SRuslan Bukin #define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 64*f17c4e38SRuslan Bukin #define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 65*f17c4e38SRuslan Bukin #define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 66*f17c4e38SRuslan Bukin #define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t))) 67*f17c4e38SRuslan Bukin 68*f17c4e38SRuslan Bukin #define NUL0E IOMMU_L0_ENTRIES 69*f17c4e38SRuslan Bukin #define NUL1E (NUL0E * NL1PG) 70*f17c4e38SRuslan Bukin #define NUL2E (NUL1E * NL2PG) 71*f17c4e38SRuslan Bukin 72*f17c4e38SRuslan Bukin #define iommu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) 73*f17c4e38SRuslan Bukin #define iommu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) 74*f17c4e38SRuslan Bukin #define iommu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) 75*f17c4e38SRuslan Bukin 76*f17c4e38SRuslan Bukin /* This code assumes all L1 DMAP entries will be used */ 77*f17c4e38SRuslan Bukin CTASSERT((DMAP_MIN_ADDRESS & ~IOMMU_L0_OFFSET) == DMAP_MIN_ADDRESS); 78*f17c4e38SRuslan Bukin CTASSERT((DMAP_MAX_ADDRESS & ~IOMMU_L0_OFFSET) == DMAP_MAX_ADDRESS); 79*f17c4e38SRuslan Bukin 80*f17c4e38SRuslan Bukin static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex); 81*f17c4e38SRuslan Bukin static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, 82*f17c4e38SRuslan Bukin struct spglist *free); 83*f17c4e38SRuslan Bukin 84*f17c4e38SRuslan Bukin /* 85*f17c4e38SRuslan Bukin * These load the old table data and store the new value. 86*f17c4e38SRuslan Bukin * They need to be atomic as the System MMU may write to the table at 87*f17c4e38SRuslan Bukin * the same time as the CPU. 88*f17c4e38SRuslan Bukin */ 89*f17c4e38SRuslan Bukin #define pmap_load(table) (*table) 90*f17c4e38SRuslan Bukin #define pmap_clear(table) atomic_store_64(table, 0) 91*f17c4e38SRuslan Bukin #define pmap_store(table, entry) atomic_store_64(table, entry) 92*f17c4e38SRuslan Bukin 93*f17c4e38SRuslan Bukin /********************/ 94*f17c4e38SRuslan Bukin /* Inline functions */ 95*f17c4e38SRuslan Bukin /********************/ 96*f17c4e38SRuslan Bukin 97*f17c4e38SRuslan Bukin static __inline pd_entry_t * 98*f17c4e38SRuslan Bukin pmap_l0(pmap_t pmap, vm_offset_t va) 99*f17c4e38SRuslan Bukin { 100*f17c4e38SRuslan Bukin 101*f17c4e38SRuslan Bukin return (&pmap->pm_l0[iommu_l0_index(va)]); 102*f17c4e38SRuslan Bukin } 103*f17c4e38SRuslan Bukin 104*f17c4e38SRuslan Bukin static __inline pd_entry_t * 105*f17c4e38SRuslan Bukin pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) 106*f17c4e38SRuslan Bukin { 107*f17c4e38SRuslan Bukin pd_entry_t *l1; 108*f17c4e38SRuslan Bukin 109*f17c4e38SRuslan Bukin l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); 110*f17c4e38SRuslan Bukin return (&l1[iommu_l1_index(va)]); 111*f17c4e38SRuslan Bukin } 112*f17c4e38SRuslan Bukin 113*f17c4e38SRuslan Bukin static __inline pd_entry_t * 114*f17c4e38SRuslan Bukin pmap_l1(pmap_t pmap, vm_offset_t va) 115*f17c4e38SRuslan Bukin { 116*f17c4e38SRuslan Bukin pd_entry_t *l0; 117*f17c4e38SRuslan Bukin 118*f17c4e38SRuslan Bukin l0 = pmap_l0(pmap, va); 119*f17c4e38SRuslan Bukin if ((pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) 120*f17c4e38SRuslan Bukin return (NULL); 121*f17c4e38SRuslan Bukin 122*f17c4e38SRuslan Bukin return (pmap_l0_to_l1(l0, va)); 123*f17c4e38SRuslan Bukin } 124*f17c4e38SRuslan Bukin 125*f17c4e38SRuslan Bukin static __inline pd_entry_t * 126*f17c4e38SRuslan Bukin pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) 127*f17c4e38SRuslan Bukin { 128*f17c4e38SRuslan Bukin pd_entry_t l1, *l2p; 129*f17c4e38SRuslan Bukin 130*f17c4e38SRuslan Bukin l1 = pmap_load(l1p); 131*f17c4e38SRuslan Bukin 132*f17c4e38SRuslan Bukin /* 133*f17c4e38SRuslan Bukin * The valid bit may be clear if pmap_update_entry() is concurrently 134*f17c4e38SRuslan Bukin * modifying the entry, so for KVA only the entry type may be checked. 135*f17c4e38SRuslan Bukin */ 136*f17c4e38SRuslan Bukin KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, 137*f17c4e38SRuslan Bukin ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); 138*f17c4e38SRuslan Bukin KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 139*f17c4e38SRuslan Bukin ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); 140*f17c4e38SRuslan Bukin l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); 141*f17c4e38SRuslan Bukin return (&l2p[iommu_l2_index(va)]); 142*f17c4e38SRuslan Bukin } 143*f17c4e38SRuslan Bukin 144*f17c4e38SRuslan Bukin static __inline pd_entry_t * 145*f17c4e38SRuslan Bukin pmap_l2(pmap_t pmap, vm_offset_t va) 146*f17c4e38SRuslan Bukin { 147*f17c4e38SRuslan Bukin pd_entry_t *l1; 148*f17c4e38SRuslan Bukin 149*f17c4e38SRuslan Bukin l1 = pmap_l1(pmap, va); 150*f17c4e38SRuslan Bukin if ((pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) 151*f17c4e38SRuslan Bukin return (NULL); 152*f17c4e38SRuslan Bukin 153*f17c4e38SRuslan Bukin return (pmap_l1_to_l2(l1, va)); 154*f17c4e38SRuslan Bukin } 155*f17c4e38SRuslan Bukin 156*f17c4e38SRuslan Bukin static __inline pt_entry_t * 157*f17c4e38SRuslan Bukin pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) 158*f17c4e38SRuslan Bukin { 159*f17c4e38SRuslan Bukin pd_entry_t l2; 160*f17c4e38SRuslan Bukin pt_entry_t *l3p; 161*f17c4e38SRuslan Bukin 162*f17c4e38SRuslan Bukin l2 = pmap_load(l2p); 163*f17c4e38SRuslan Bukin 164*f17c4e38SRuslan Bukin /* 165*f17c4e38SRuslan Bukin * The valid bit may be clear if pmap_update_entry() is concurrently 166*f17c4e38SRuslan Bukin * modifying the entry, so for KVA only the entry type may be checked. 167*f17c4e38SRuslan Bukin */ 168*f17c4e38SRuslan Bukin KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, 169*f17c4e38SRuslan Bukin ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); 170*f17c4e38SRuslan Bukin KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 171*f17c4e38SRuslan Bukin ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); 172*f17c4e38SRuslan Bukin l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); 173*f17c4e38SRuslan Bukin return (&l3p[iommu_l3_index(va)]); 174*f17c4e38SRuslan Bukin } 175*f17c4e38SRuslan Bukin 176*f17c4e38SRuslan Bukin /* 177*f17c4e38SRuslan Bukin * Returns the lowest valid pde for a given virtual address. 178*f17c4e38SRuslan Bukin * The next level may or may not point to a valid page or block. 179*f17c4e38SRuslan Bukin */ 180*f17c4e38SRuslan Bukin static __inline pd_entry_t * 181*f17c4e38SRuslan Bukin pmap_pde(pmap_t pmap, vm_offset_t va, int *level) 182*f17c4e38SRuslan Bukin { 183*f17c4e38SRuslan Bukin pd_entry_t *l0, *l1, *l2, desc; 184*f17c4e38SRuslan Bukin 185*f17c4e38SRuslan Bukin l0 = pmap_l0(pmap, va); 186*f17c4e38SRuslan Bukin desc = pmap_load(l0) & ATTR_DESCR_MASK; 187*f17c4e38SRuslan Bukin if (desc != IOMMU_L0_TABLE) { 188*f17c4e38SRuslan Bukin *level = -1; 189*f17c4e38SRuslan Bukin return (NULL); 190*f17c4e38SRuslan Bukin } 191*f17c4e38SRuslan Bukin 192*f17c4e38SRuslan Bukin l1 = pmap_l0_to_l1(l0, va); 193*f17c4e38SRuslan Bukin desc = pmap_load(l1) & ATTR_DESCR_MASK; 194*f17c4e38SRuslan Bukin if (desc != IOMMU_L1_TABLE) { 195*f17c4e38SRuslan Bukin *level = 0; 196*f17c4e38SRuslan Bukin return (l0); 197*f17c4e38SRuslan Bukin } 198*f17c4e38SRuslan Bukin 199*f17c4e38SRuslan Bukin l2 = pmap_l1_to_l2(l1, va); 200*f17c4e38SRuslan Bukin desc = pmap_load(l2) & ATTR_DESCR_MASK; 201*f17c4e38SRuslan Bukin if (desc != IOMMU_L2_TABLE) { 202*f17c4e38SRuslan Bukin *level = 1; 203*f17c4e38SRuslan Bukin return (l1); 204*f17c4e38SRuslan Bukin } 205*f17c4e38SRuslan Bukin 206*f17c4e38SRuslan Bukin *level = 2; 207*f17c4e38SRuslan Bukin return (l2); 208*f17c4e38SRuslan Bukin } 209*f17c4e38SRuslan Bukin 210*f17c4e38SRuslan Bukin /* 211*f17c4e38SRuslan Bukin * Returns the lowest valid pte block or table entry for a given virtual 212*f17c4e38SRuslan Bukin * address. If there are no valid entries return NULL and set the level to 213*f17c4e38SRuslan Bukin * the first invalid level. 214*f17c4e38SRuslan Bukin */ 215*f17c4e38SRuslan Bukin static __inline pt_entry_t * 216*f17c4e38SRuslan Bukin pmap_pte(pmap_t pmap, vm_offset_t va, int *level) 217*f17c4e38SRuslan Bukin { 218*f17c4e38SRuslan Bukin pd_entry_t *l1, *l2, desc; 219*f17c4e38SRuslan Bukin pt_entry_t *l3; 220*f17c4e38SRuslan Bukin 221*f17c4e38SRuslan Bukin l1 = pmap_l1(pmap, va); 222*f17c4e38SRuslan Bukin if (l1 == NULL) { 223*f17c4e38SRuslan Bukin *level = 0; 224*f17c4e38SRuslan Bukin return (NULL); 225*f17c4e38SRuslan Bukin } 226*f17c4e38SRuslan Bukin desc = pmap_load(l1) & ATTR_DESCR_MASK; 227*f17c4e38SRuslan Bukin if (desc == IOMMU_L1_BLOCK) { 228*f17c4e38SRuslan Bukin *level = 1; 229*f17c4e38SRuslan Bukin return (l1); 230*f17c4e38SRuslan Bukin } 231*f17c4e38SRuslan Bukin 232*f17c4e38SRuslan Bukin if (desc != IOMMU_L1_TABLE) { 233*f17c4e38SRuslan Bukin *level = 1; 234*f17c4e38SRuslan Bukin return (NULL); 235*f17c4e38SRuslan Bukin } 236*f17c4e38SRuslan Bukin 237*f17c4e38SRuslan Bukin l2 = pmap_l1_to_l2(l1, va); 238*f17c4e38SRuslan Bukin desc = pmap_load(l2) & ATTR_DESCR_MASK; 239*f17c4e38SRuslan Bukin if (desc == IOMMU_L2_BLOCK) { 240*f17c4e38SRuslan Bukin *level = 2; 241*f17c4e38SRuslan Bukin return (l2); 242*f17c4e38SRuslan Bukin } 243*f17c4e38SRuslan Bukin 244*f17c4e38SRuslan Bukin if (desc != IOMMU_L2_TABLE) { 245*f17c4e38SRuslan Bukin *level = 2; 246*f17c4e38SRuslan Bukin return (NULL); 247*f17c4e38SRuslan Bukin } 248*f17c4e38SRuslan Bukin 249*f17c4e38SRuslan Bukin *level = 3; 250*f17c4e38SRuslan Bukin l3 = pmap_l2_to_l3(l2, va); 251*f17c4e38SRuslan Bukin if ((pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) 252*f17c4e38SRuslan Bukin return (NULL); 253*f17c4e38SRuslan Bukin 254*f17c4e38SRuslan Bukin return (l3); 255*f17c4e38SRuslan Bukin } 256*f17c4e38SRuslan Bukin 257*f17c4e38SRuslan Bukin static __inline int 258*f17c4e38SRuslan Bukin pmap_l3_valid(pt_entry_t l3) 259*f17c4e38SRuslan Bukin { 260*f17c4e38SRuslan Bukin 261*f17c4e38SRuslan Bukin return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE); 262*f17c4e38SRuslan Bukin } 263*f17c4e38SRuslan Bukin 264*f17c4e38SRuslan Bukin CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK); 265*f17c4e38SRuslan Bukin 266*f17c4e38SRuslan Bukin static __inline void 267*f17c4e38SRuslan Bukin pmap_resident_count_inc(pmap_t pmap, int count) 268*f17c4e38SRuslan Bukin { 269*f17c4e38SRuslan Bukin 270*f17c4e38SRuslan Bukin PMAP_LOCK_ASSERT(pmap, MA_OWNED); 271*f17c4e38SRuslan Bukin pmap->pm_stats.resident_count += count; 272*f17c4e38SRuslan Bukin } 273*f17c4e38SRuslan Bukin 274*f17c4e38SRuslan Bukin static __inline void 275*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap_t pmap, int count) 276*f17c4e38SRuslan Bukin { 277*f17c4e38SRuslan Bukin 278*f17c4e38SRuslan Bukin PMAP_LOCK_ASSERT(pmap, MA_OWNED); 279*f17c4e38SRuslan Bukin KASSERT(pmap->pm_stats.resident_count >= count, 280*f17c4e38SRuslan Bukin ("pmap %p resident count underflow %ld %d", pmap, 281*f17c4e38SRuslan Bukin pmap->pm_stats.resident_count, count)); 282*f17c4e38SRuslan Bukin pmap->pm_stats.resident_count -= count; 283*f17c4e38SRuslan Bukin } 284*f17c4e38SRuslan Bukin 285*f17c4e38SRuslan Bukin /*************************************************** 286*f17c4e38SRuslan Bukin * Page table page management routines..... 287*f17c4e38SRuslan Bukin ***************************************************/ 288*f17c4e38SRuslan Bukin /* 289*f17c4e38SRuslan Bukin * Schedule the specified unused page table page to be freed. Specifically, 290*f17c4e38SRuslan Bukin * add the page to the specified list of pages that will be released to the 291*f17c4e38SRuslan Bukin * physical memory manager after the TLB has been updated. 292*f17c4e38SRuslan Bukin */ 293*f17c4e38SRuslan Bukin static __inline void 294*f17c4e38SRuslan Bukin pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 295*f17c4e38SRuslan Bukin boolean_t set_PG_ZERO) 296*f17c4e38SRuslan Bukin { 297*f17c4e38SRuslan Bukin 298*f17c4e38SRuslan Bukin if (set_PG_ZERO) 299*f17c4e38SRuslan Bukin m->flags |= PG_ZERO; 300*f17c4e38SRuslan Bukin else 301*f17c4e38SRuslan Bukin m->flags &= ~PG_ZERO; 302*f17c4e38SRuslan Bukin SLIST_INSERT_HEAD(free, m, plinks.s.ss); 303*f17c4e38SRuslan Bukin } 304*f17c4e38SRuslan Bukin 305*f17c4e38SRuslan Bukin /*************************************************** 306*f17c4e38SRuslan Bukin * Low level mapping routines..... 307*f17c4e38SRuslan Bukin ***************************************************/ 308*f17c4e38SRuslan Bukin 309*f17c4e38SRuslan Bukin /* 310*f17c4e38SRuslan Bukin * Decrements a page table page's reference count, which is used to record the 311*f17c4e38SRuslan Bukin * number of valid page table entries within the page. If the reference count 312*f17c4e38SRuslan Bukin * drops to zero, then the page table page is unmapped. Returns TRUE if the 313*f17c4e38SRuslan Bukin * page table page was unmapped and FALSE otherwise. 314*f17c4e38SRuslan Bukin */ 315*f17c4e38SRuslan Bukin static inline boolean_t 316*f17c4e38SRuslan Bukin pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 317*f17c4e38SRuslan Bukin { 318*f17c4e38SRuslan Bukin 319*f17c4e38SRuslan Bukin --m->ref_count; 320*f17c4e38SRuslan Bukin if (m->ref_count == 0) { 321*f17c4e38SRuslan Bukin _pmap_unwire_l3(pmap, va, m, free); 322*f17c4e38SRuslan Bukin return (TRUE); 323*f17c4e38SRuslan Bukin } else 324*f17c4e38SRuslan Bukin return (FALSE); 325*f17c4e38SRuslan Bukin } 326*f17c4e38SRuslan Bukin 327*f17c4e38SRuslan Bukin static void 328*f17c4e38SRuslan Bukin _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 329*f17c4e38SRuslan Bukin { 330*f17c4e38SRuslan Bukin 331*f17c4e38SRuslan Bukin PMAP_LOCK_ASSERT(pmap, MA_OWNED); 332*f17c4e38SRuslan Bukin /* 333*f17c4e38SRuslan Bukin * unmap the page table page 334*f17c4e38SRuslan Bukin */ 335*f17c4e38SRuslan Bukin if (m->pindex >= (NUL2E + NUL1E)) { 336*f17c4e38SRuslan Bukin /* l1 page */ 337*f17c4e38SRuslan Bukin pd_entry_t *l0; 338*f17c4e38SRuslan Bukin 339*f17c4e38SRuslan Bukin l0 = pmap_l0(pmap, va); 340*f17c4e38SRuslan Bukin pmap_clear(l0); 341*f17c4e38SRuslan Bukin } else if (m->pindex >= NUL2E) { 342*f17c4e38SRuslan Bukin /* l2 page */ 343*f17c4e38SRuslan Bukin pd_entry_t *l1; 344*f17c4e38SRuslan Bukin 345*f17c4e38SRuslan Bukin l1 = pmap_l1(pmap, va); 346*f17c4e38SRuslan Bukin pmap_clear(l1); 347*f17c4e38SRuslan Bukin } else { 348*f17c4e38SRuslan Bukin /* l3 page */ 349*f17c4e38SRuslan Bukin pd_entry_t *l2; 350*f17c4e38SRuslan Bukin 351*f17c4e38SRuslan Bukin l2 = pmap_l2(pmap, va); 352*f17c4e38SRuslan Bukin pmap_clear(l2); 353*f17c4e38SRuslan Bukin } 354*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 355*f17c4e38SRuslan Bukin if (m->pindex < NUL2E) { 356*f17c4e38SRuslan Bukin /* We just released an l3, unhold the matching l2 */ 357*f17c4e38SRuslan Bukin pd_entry_t *l1, tl1; 358*f17c4e38SRuslan Bukin vm_page_t l2pg; 359*f17c4e38SRuslan Bukin 360*f17c4e38SRuslan Bukin l1 = pmap_l1(pmap, va); 361*f17c4e38SRuslan Bukin tl1 = pmap_load(l1); 362*f17c4e38SRuslan Bukin l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 363*f17c4e38SRuslan Bukin pmap_unwire_l3(pmap, va, l2pg, free); 364*f17c4e38SRuslan Bukin } else if (m->pindex < (NUL2E + NUL1E)) { 365*f17c4e38SRuslan Bukin /* We just released an l2, unhold the matching l1 */ 366*f17c4e38SRuslan Bukin pd_entry_t *l0, tl0; 367*f17c4e38SRuslan Bukin vm_page_t l1pg; 368*f17c4e38SRuslan Bukin 369*f17c4e38SRuslan Bukin l0 = pmap_l0(pmap, va); 370*f17c4e38SRuslan Bukin tl0 = pmap_load(l0); 371*f17c4e38SRuslan Bukin l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 372*f17c4e38SRuslan Bukin pmap_unwire_l3(pmap, va, l1pg, free); 373*f17c4e38SRuslan Bukin } 374*f17c4e38SRuslan Bukin 375*f17c4e38SRuslan Bukin /* 376*f17c4e38SRuslan Bukin * Put page on a list so that it is released after 377*f17c4e38SRuslan Bukin * *ALL* TLB shootdown is done 378*f17c4e38SRuslan Bukin */ 379*f17c4e38SRuslan Bukin pmap_add_delayed_free_list(m, free, TRUE); 380*f17c4e38SRuslan Bukin } 381*f17c4e38SRuslan Bukin 382*f17c4e38SRuslan Bukin static int 383*f17c4e38SRuslan Bukin iommu_pmap_pinit_levels(pmap_t pmap, int levels) 384*f17c4e38SRuslan Bukin { 385*f17c4e38SRuslan Bukin vm_page_t m; 386*f17c4e38SRuslan Bukin 387*f17c4e38SRuslan Bukin /* 388*f17c4e38SRuslan Bukin * allocate the l0 page 389*f17c4e38SRuslan Bukin */ 390*f17c4e38SRuslan Bukin while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 391*f17c4e38SRuslan Bukin VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) 392*f17c4e38SRuslan Bukin vm_wait(NULL); 393*f17c4e38SRuslan Bukin 394*f17c4e38SRuslan Bukin pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); 395*f17c4e38SRuslan Bukin pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); 396*f17c4e38SRuslan Bukin 397*f17c4e38SRuslan Bukin if ((m->flags & PG_ZERO) == 0) 398*f17c4e38SRuslan Bukin pagezero(pmap->pm_l0); 399*f17c4e38SRuslan Bukin 400*f17c4e38SRuslan Bukin pmap->pm_root.rt_root = 0; 401*f17c4e38SRuslan Bukin bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 402*f17c4e38SRuslan Bukin 403*f17c4e38SRuslan Bukin MPASS(levels == 3 || levels == 4); 404*f17c4e38SRuslan Bukin pmap->pm_levels = levels; 405*f17c4e38SRuslan Bukin 406*f17c4e38SRuslan Bukin /* 407*f17c4e38SRuslan Bukin * Allocate the level 1 entry to use as the root. This will increase 408*f17c4e38SRuslan Bukin * the refcount on the level 1 page so it won't be removed until 409*f17c4e38SRuslan Bukin * pmap_release() is called. 410*f17c4e38SRuslan Bukin */ 411*f17c4e38SRuslan Bukin if (pmap->pm_levels == 3) { 412*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 413*f17c4e38SRuslan Bukin m = _pmap_alloc_l3(pmap, NUL2E + NUL1E); 414*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 415*f17c4e38SRuslan Bukin } 416*f17c4e38SRuslan Bukin pmap->pm_ttbr = VM_PAGE_TO_PHYS(m); 417*f17c4e38SRuslan Bukin 418*f17c4e38SRuslan Bukin return (1); 419*f17c4e38SRuslan Bukin } 420*f17c4e38SRuslan Bukin 421*f17c4e38SRuslan Bukin int 422*f17c4e38SRuslan Bukin iommu_pmap_pinit(pmap_t pmap) 423*f17c4e38SRuslan Bukin { 424*f17c4e38SRuslan Bukin 425*f17c4e38SRuslan Bukin return (iommu_pmap_pinit_levels(pmap, 4)); 426*f17c4e38SRuslan Bukin } 427*f17c4e38SRuslan Bukin 428*f17c4e38SRuslan Bukin /* 429*f17c4e38SRuslan Bukin * This routine is called if the desired page table page does not exist. 430*f17c4e38SRuslan Bukin * 431*f17c4e38SRuslan Bukin * If page table page allocation fails, this routine may sleep before 432*f17c4e38SRuslan Bukin * returning NULL. It sleeps only if a lock pointer was given. 433*f17c4e38SRuslan Bukin * 434*f17c4e38SRuslan Bukin * Note: If a page allocation fails at page table level two or three, 435*f17c4e38SRuslan Bukin * one or two pages may be held during the wait, only to be released 436*f17c4e38SRuslan Bukin * afterwards. This conservative approach is easily argued to avoid 437*f17c4e38SRuslan Bukin * race conditions. 438*f17c4e38SRuslan Bukin */ 439*f17c4e38SRuslan Bukin static vm_page_t 440*f17c4e38SRuslan Bukin _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex) 441*f17c4e38SRuslan Bukin { 442*f17c4e38SRuslan Bukin vm_page_t m, l1pg, l2pg; 443*f17c4e38SRuslan Bukin 444*f17c4e38SRuslan Bukin PMAP_LOCK_ASSERT(pmap, MA_OWNED); 445*f17c4e38SRuslan Bukin 446*f17c4e38SRuslan Bukin /* 447*f17c4e38SRuslan Bukin * Allocate a page table page. 448*f17c4e38SRuslan Bukin */ 449*f17c4e38SRuslan Bukin if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 450*f17c4e38SRuslan Bukin VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 451*f17c4e38SRuslan Bukin /* 452*f17c4e38SRuslan Bukin * Indicate the need to retry. While waiting, the page table 453*f17c4e38SRuslan Bukin * page may have been allocated. 454*f17c4e38SRuslan Bukin */ 455*f17c4e38SRuslan Bukin return (NULL); 456*f17c4e38SRuslan Bukin } 457*f17c4e38SRuslan Bukin if ((m->flags & PG_ZERO) == 0) 458*f17c4e38SRuslan Bukin pmap_zero_page(m); 459*f17c4e38SRuslan Bukin 460*f17c4e38SRuslan Bukin /* 461*f17c4e38SRuslan Bukin * Because of AArch64's weak memory consistency model, we must have a 462*f17c4e38SRuslan Bukin * barrier here to ensure that the stores for zeroing "m", whether by 463*f17c4e38SRuslan Bukin * pmap_zero_page() or an earlier function, are visible before adding 464*f17c4e38SRuslan Bukin * "m" to the page table. Otherwise, a page table walk by another 465*f17c4e38SRuslan Bukin * processor's MMU could see the mapping to "m" and a stale, non-zero 466*f17c4e38SRuslan Bukin * PTE within "m". 467*f17c4e38SRuslan Bukin */ 468*f17c4e38SRuslan Bukin dmb(ishst); 469*f17c4e38SRuslan Bukin 470*f17c4e38SRuslan Bukin /* 471*f17c4e38SRuslan Bukin * Map the pagetable page into the process address space, if 472*f17c4e38SRuslan Bukin * it isn't already there. 473*f17c4e38SRuslan Bukin */ 474*f17c4e38SRuslan Bukin 475*f17c4e38SRuslan Bukin if (ptepindex >= (NUL2E + NUL1E)) { 476*f17c4e38SRuslan Bukin pd_entry_t *l0; 477*f17c4e38SRuslan Bukin vm_pindex_t l0index; 478*f17c4e38SRuslan Bukin 479*f17c4e38SRuslan Bukin l0index = ptepindex - (NUL2E + NUL1E); 480*f17c4e38SRuslan Bukin l0 = &pmap->pm_l0[l0index]; 481*f17c4e38SRuslan Bukin pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); 482*f17c4e38SRuslan Bukin } else if (ptepindex >= NUL2E) { 483*f17c4e38SRuslan Bukin vm_pindex_t l0index, l1index; 484*f17c4e38SRuslan Bukin pd_entry_t *l0, *l1; 485*f17c4e38SRuslan Bukin pd_entry_t tl0; 486*f17c4e38SRuslan Bukin 487*f17c4e38SRuslan Bukin l1index = ptepindex - NUL2E; 488*f17c4e38SRuslan Bukin l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 489*f17c4e38SRuslan Bukin 490*f17c4e38SRuslan Bukin l0 = &pmap->pm_l0[l0index]; 491*f17c4e38SRuslan Bukin tl0 = pmap_load(l0); 492*f17c4e38SRuslan Bukin if (tl0 == 0) { 493*f17c4e38SRuslan Bukin /* recurse for allocating page dir */ 494*f17c4e38SRuslan Bukin if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index) 495*f17c4e38SRuslan Bukin == NULL) { 496*f17c4e38SRuslan Bukin vm_page_unwire_noq(m); 497*f17c4e38SRuslan Bukin vm_page_free_zero(m); 498*f17c4e38SRuslan Bukin return (NULL); 499*f17c4e38SRuslan Bukin } 500*f17c4e38SRuslan Bukin } else { 501*f17c4e38SRuslan Bukin l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 502*f17c4e38SRuslan Bukin l1pg->ref_count++; 503*f17c4e38SRuslan Bukin } 504*f17c4e38SRuslan Bukin 505*f17c4e38SRuslan Bukin l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); 506*f17c4e38SRuslan Bukin l1 = &l1[ptepindex & Ln_ADDR_MASK]; 507*f17c4e38SRuslan Bukin pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); 508*f17c4e38SRuslan Bukin } else { 509*f17c4e38SRuslan Bukin vm_pindex_t l0index, l1index; 510*f17c4e38SRuslan Bukin pd_entry_t *l0, *l1, *l2; 511*f17c4e38SRuslan Bukin pd_entry_t tl0, tl1; 512*f17c4e38SRuslan Bukin 513*f17c4e38SRuslan Bukin l1index = ptepindex >> Ln_ENTRIES_SHIFT; 514*f17c4e38SRuslan Bukin l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 515*f17c4e38SRuslan Bukin 516*f17c4e38SRuslan Bukin l0 = &pmap->pm_l0[l0index]; 517*f17c4e38SRuslan Bukin tl0 = pmap_load(l0); 518*f17c4e38SRuslan Bukin if (tl0 == 0) { 519*f17c4e38SRuslan Bukin /* recurse for allocating page dir */ 520*f17c4e38SRuslan Bukin if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { 521*f17c4e38SRuslan Bukin vm_page_unwire_noq(m); 522*f17c4e38SRuslan Bukin vm_page_free_zero(m); 523*f17c4e38SRuslan Bukin return (NULL); 524*f17c4e38SRuslan Bukin } 525*f17c4e38SRuslan Bukin tl0 = pmap_load(l0); 526*f17c4e38SRuslan Bukin l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 527*f17c4e38SRuslan Bukin l1 = &l1[l1index & Ln_ADDR_MASK]; 528*f17c4e38SRuslan Bukin } else { 529*f17c4e38SRuslan Bukin l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 530*f17c4e38SRuslan Bukin l1 = &l1[l1index & Ln_ADDR_MASK]; 531*f17c4e38SRuslan Bukin tl1 = pmap_load(l1); 532*f17c4e38SRuslan Bukin if (tl1 == 0) { 533*f17c4e38SRuslan Bukin /* recurse for allocating page dir */ 534*f17c4e38SRuslan Bukin if (_pmap_alloc_l3(pmap, NUL2E + l1index) 535*f17c4e38SRuslan Bukin == NULL) { 536*f17c4e38SRuslan Bukin vm_page_unwire_noq(m); 537*f17c4e38SRuslan Bukin vm_page_free_zero(m); 538*f17c4e38SRuslan Bukin return (NULL); 539*f17c4e38SRuslan Bukin } 540*f17c4e38SRuslan Bukin } else { 541*f17c4e38SRuslan Bukin l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 542*f17c4e38SRuslan Bukin l2pg->ref_count++; 543*f17c4e38SRuslan Bukin } 544*f17c4e38SRuslan Bukin } 545*f17c4e38SRuslan Bukin 546*f17c4e38SRuslan Bukin l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); 547*f17c4e38SRuslan Bukin l2 = &l2[ptepindex & Ln_ADDR_MASK]; 548*f17c4e38SRuslan Bukin pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); 549*f17c4e38SRuslan Bukin } 550*f17c4e38SRuslan Bukin 551*f17c4e38SRuslan Bukin pmap_resident_count_inc(pmap, 1); 552*f17c4e38SRuslan Bukin 553*f17c4e38SRuslan Bukin return (m); 554*f17c4e38SRuslan Bukin } 555*f17c4e38SRuslan Bukin 556*f17c4e38SRuslan Bukin /*************************************************** 557*f17c4e38SRuslan Bukin * Pmap allocation/deallocation routines. 558*f17c4e38SRuslan Bukin ***************************************************/ 559*f17c4e38SRuslan Bukin 560*f17c4e38SRuslan Bukin /* 561*f17c4e38SRuslan Bukin * Release any resources held by the given physical map. 562*f17c4e38SRuslan Bukin * Called when a pmap initialized by pmap_pinit is being released. 563*f17c4e38SRuslan Bukin * Should only be called if the map contains no valid mappings. 564*f17c4e38SRuslan Bukin */ 565*f17c4e38SRuslan Bukin void 566*f17c4e38SRuslan Bukin iommu_pmap_release(pmap_t pmap) 567*f17c4e38SRuslan Bukin { 568*f17c4e38SRuslan Bukin boolean_t rv; 569*f17c4e38SRuslan Bukin struct spglist free; 570*f17c4e38SRuslan Bukin vm_page_t m; 571*f17c4e38SRuslan Bukin 572*f17c4e38SRuslan Bukin if (pmap->pm_levels != 4) { 573*f17c4e38SRuslan Bukin KASSERT(pmap->pm_stats.resident_count == 1, 574*f17c4e38SRuslan Bukin ("pmap_release: pmap resident count %ld != 0", 575*f17c4e38SRuslan Bukin pmap->pm_stats.resident_count)); 576*f17c4e38SRuslan Bukin KASSERT((pmap->pm_l0[0] & ATTR_DESCR_VALID) == ATTR_DESCR_VALID, 577*f17c4e38SRuslan Bukin ("pmap_release: Invalid l0 entry: %lx", pmap->pm_l0[0])); 578*f17c4e38SRuslan Bukin 579*f17c4e38SRuslan Bukin SLIST_INIT(&free); 580*f17c4e38SRuslan Bukin m = PHYS_TO_VM_PAGE(pmap->pm_ttbr); 581*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 582*f17c4e38SRuslan Bukin rv = pmap_unwire_l3(pmap, 0, m, &free); 583*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 584*f17c4e38SRuslan Bukin MPASS(rv == TRUE); 585*f17c4e38SRuslan Bukin vm_page_free_pages_toq(&free, true); 586*f17c4e38SRuslan Bukin } 587*f17c4e38SRuslan Bukin 588*f17c4e38SRuslan Bukin KASSERT(pmap->pm_stats.resident_count == 0, 589*f17c4e38SRuslan Bukin ("pmap_release: pmap resident count %ld != 0", 590*f17c4e38SRuslan Bukin pmap->pm_stats.resident_count)); 591*f17c4e38SRuslan Bukin KASSERT(vm_radix_is_empty(&pmap->pm_root), 592*f17c4e38SRuslan Bukin ("pmap_release: pmap has reserved page table page(s)")); 593*f17c4e38SRuslan Bukin 594*f17c4e38SRuslan Bukin m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr); 595*f17c4e38SRuslan Bukin vm_page_unwire_noq(m); 596*f17c4e38SRuslan Bukin vm_page_free_zero(m); 597*f17c4e38SRuslan Bukin } 598*f17c4e38SRuslan Bukin 599*f17c4e38SRuslan Bukin /*************************************************** 600*f17c4e38SRuslan Bukin * page management routines. 601*f17c4e38SRuslan Bukin ***************************************************/ 602*f17c4e38SRuslan Bukin 603*f17c4e38SRuslan Bukin /* 604*f17c4e38SRuslan Bukin * Add a single Mali GPU entry. This function does not sleep. 605*f17c4e38SRuslan Bukin */ 606*f17c4e38SRuslan Bukin int 607*f17c4e38SRuslan Bukin pmap_gpu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 608*f17c4e38SRuslan Bukin vm_prot_t prot, u_int flags) 609*f17c4e38SRuslan Bukin { 610*f17c4e38SRuslan Bukin pd_entry_t *pde; 611*f17c4e38SRuslan Bukin pt_entry_t new_l3, orig_l3; 612*f17c4e38SRuslan Bukin pt_entry_t *l3; 613*f17c4e38SRuslan Bukin vm_page_t mpte; 614*f17c4e38SRuslan Bukin pd_entry_t *l1p; 615*f17c4e38SRuslan Bukin pd_entry_t *l2p; 616*f17c4e38SRuslan Bukin int lvl; 617*f17c4e38SRuslan Bukin int rv; 618*f17c4e38SRuslan Bukin 619*f17c4e38SRuslan Bukin KASSERT(pmap != kernel_pmap, ("kernel pmap used for GPU")); 620*f17c4e38SRuslan Bukin KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 621*f17c4e38SRuslan Bukin KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 622*f17c4e38SRuslan Bukin KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); 623*f17c4e38SRuslan Bukin 624*f17c4e38SRuslan Bukin new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK); 625*f17c4e38SRuslan Bukin 626*f17c4e38SRuslan Bukin if ((prot & VM_PROT_WRITE) != 0) 627*f17c4e38SRuslan Bukin new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 628*f17c4e38SRuslan Bukin if ((prot & VM_PROT_READ) != 0) 629*f17c4e38SRuslan Bukin new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); 630*f17c4e38SRuslan Bukin if ((prot & VM_PROT_EXECUTE) == 0) 631*f17c4e38SRuslan Bukin new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL); 632*f17c4e38SRuslan Bukin 633*f17c4e38SRuslan Bukin CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa); 634*f17c4e38SRuslan Bukin 635*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 636*f17c4e38SRuslan Bukin 637*f17c4e38SRuslan Bukin /* 638*f17c4e38SRuslan Bukin * In the case that a page table page is not 639*f17c4e38SRuslan Bukin * resident, we are creating it here. 640*f17c4e38SRuslan Bukin */ 641*f17c4e38SRuslan Bukin retry: 642*f17c4e38SRuslan Bukin pde = pmap_pde(pmap, va, &lvl); 643*f17c4e38SRuslan Bukin if (pde != NULL && lvl == 2) { 644*f17c4e38SRuslan Bukin l3 = pmap_l2_to_l3(pde, va); 645*f17c4e38SRuslan Bukin } else { 646*f17c4e38SRuslan Bukin mpte = _pmap_alloc_l3(pmap, iommu_l2_pindex(va)); 647*f17c4e38SRuslan Bukin if (mpte == NULL) { 648*f17c4e38SRuslan Bukin CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 649*f17c4e38SRuslan Bukin rv = KERN_RESOURCE_SHORTAGE; 650*f17c4e38SRuslan Bukin goto out; 651*f17c4e38SRuslan Bukin } 652*f17c4e38SRuslan Bukin 653*f17c4e38SRuslan Bukin /* 654*f17c4e38SRuslan Bukin * Ensure newly created l1, l2 are visible to GPU. 655*f17c4e38SRuslan Bukin * l0 is already visible by similar call in panfrost driver. 656*f17c4e38SRuslan Bukin * The cache entry for l3 handled below. 657*f17c4e38SRuslan Bukin */ 658*f17c4e38SRuslan Bukin 659*f17c4e38SRuslan Bukin l1p = pmap_l1(pmap, va); 660*f17c4e38SRuslan Bukin l2p = pmap_l2(pmap, va); 661*f17c4e38SRuslan Bukin cpu_dcache_wb_range((vm_offset_t)l1p, sizeof(pd_entry_t)); 662*f17c4e38SRuslan Bukin cpu_dcache_wb_range((vm_offset_t)l2p, sizeof(pd_entry_t)); 663*f17c4e38SRuslan Bukin 664*f17c4e38SRuslan Bukin goto retry; 665*f17c4e38SRuslan Bukin } 666*f17c4e38SRuslan Bukin 667*f17c4e38SRuslan Bukin orig_l3 = pmap_load(l3); 668*f17c4e38SRuslan Bukin KASSERT(!pmap_l3_valid(orig_l3), ("l3 is valid")); 669*f17c4e38SRuslan Bukin 670*f17c4e38SRuslan Bukin /* New mapping */ 671*f17c4e38SRuslan Bukin pmap_store(l3, new_l3); 672*f17c4e38SRuslan Bukin 673*f17c4e38SRuslan Bukin cpu_dcache_wb_range((vm_offset_t)l3, sizeof(pt_entry_t)); 674*f17c4e38SRuslan Bukin 675*f17c4e38SRuslan Bukin pmap_resident_count_inc(pmap, 1); 676*f17c4e38SRuslan Bukin dsb(ishst); 677*f17c4e38SRuslan Bukin 678*f17c4e38SRuslan Bukin rv = KERN_SUCCESS; 679*f17c4e38SRuslan Bukin out: 680*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 681*f17c4e38SRuslan Bukin 682*f17c4e38SRuslan Bukin return (rv); 683*f17c4e38SRuslan Bukin } 684*f17c4e38SRuslan Bukin 685*f17c4e38SRuslan Bukin /* 686*f17c4e38SRuslan Bukin * Remove a single Mali GPU entry. 687*f17c4e38SRuslan Bukin */ 688*f17c4e38SRuslan Bukin int 689*f17c4e38SRuslan Bukin pmap_gpu_remove(pmap_t pmap, vm_offset_t va) 690*f17c4e38SRuslan Bukin { 691*f17c4e38SRuslan Bukin pd_entry_t *pde; 692*f17c4e38SRuslan Bukin pt_entry_t *pte; 693*f17c4e38SRuslan Bukin int lvl; 694*f17c4e38SRuslan Bukin int rc; 695*f17c4e38SRuslan Bukin 696*f17c4e38SRuslan Bukin KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 697*f17c4e38SRuslan Bukin KASSERT(pmap != kernel_pmap, ("kernel pmap used for GPU")); 698*f17c4e38SRuslan Bukin 699*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 700*f17c4e38SRuslan Bukin 701*f17c4e38SRuslan Bukin pde = pmap_pde(pmap, va, &lvl); 702*f17c4e38SRuslan Bukin if (pde == NULL || lvl != 2) { 703*f17c4e38SRuslan Bukin rc = KERN_FAILURE; 704*f17c4e38SRuslan Bukin goto out; 705*f17c4e38SRuslan Bukin } 706*f17c4e38SRuslan Bukin 707*f17c4e38SRuslan Bukin pte = pmap_l2_to_l3(pde, va); 708*f17c4e38SRuslan Bukin 709*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 710*f17c4e38SRuslan Bukin pmap_clear(pte); 711*f17c4e38SRuslan Bukin cpu_dcache_wb_range((vm_offset_t)pte, sizeof(pt_entry_t)); 712*f17c4e38SRuslan Bukin rc = KERN_SUCCESS; 713*f17c4e38SRuslan Bukin 714*f17c4e38SRuslan Bukin out: 715*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 716*f17c4e38SRuslan Bukin 717*f17c4e38SRuslan Bukin return (rc); 718*f17c4e38SRuslan Bukin } 719*f17c4e38SRuslan Bukin 720*f17c4e38SRuslan Bukin /* 721*f17c4e38SRuslan Bukin * Add a single SMMU entry. This function does not sleep. 722*f17c4e38SRuslan Bukin */ 723*f17c4e38SRuslan Bukin int 724*f17c4e38SRuslan Bukin pmap_smmu_enter(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 725*f17c4e38SRuslan Bukin vm_prot_t prot, u_int flags) 726*f17c4e38SRuslan Bukin { 727*f17c4e38SRuslan Bukin pd_entry_t *pde; 728*f17c4e38SRuslan Bukin pt_entry_t new_l3, orig_l3; 729*f17c4e38SRuslan Bukin pt_entry_t *l3; 730*f17c4e38SRuslan Bukin vm_page_t mpte; 731*f17c4e38SRuslan Bukin int lvl; 732*f17c4e38SRuslan Bukin int rv; 733*f17c4e38SRuslan Bukin 734*f17c4e38SRuslan Bukin KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 735*f17c4e38SRuslan Bukin 736*f17c4e38SRuslan Bukin va = trunc_page(va); 737*f17c4e38SRuslan Bukin new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | 738*f17c4e38SRuslan Bukin ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE); 739*f17c4e38SRuslan Bukin if ((prot & VM_PROT_WRITE) == 0) 740*f17c4e38SRuslan Bukin new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); 741*f17c4e38SRuslan Bukin new_l3 |= ATTR_S1_XN; /* Execute never. */ 742*f17c4e38SRuslan Bukin new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER); 743*f17c4e38SRuslan Bukin new_l3 |= ATTR_S1_nG; /* Non global. */ 744*f17c4e38SRuslan Bukin 745*f17c4e38SRuslan Bukin CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa); 746*f17c4e38SRuslan Bukin 747*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 748*f17c4e38SRuslan Bukin 749*f17c4e38SRuslan Bukin /* 750*f17c4e38SRuslan Bukin * In the case that a page table page is not 751*f17c4e38SRuslan Bukin * resident, we are creating it here. 752*f17c4e38SRuslan Bukin */ 753*f17c4e38SRuslan Bukin retry: 754*f17c4e38SRuslan Bukin pde = pmap_pde(pmap, va, &lvl); 755*f17c4e38SRuslan Bukin if (pde != NULL && lvl == 2) { 756*f17c4e38SRuslan Bukin l3 = pmap_l2_to_l3(pde, va); 757*f17c4e38SRuslan Bukin } else { 758*f17c4e38SRuslan Bukin mpte = _pmap_alloc_l3(pmap, iommu_l2_pindex(va)); 759*f17c4e38SRuslan Bukin if (mpte == NULL) { 760*f17c4e38SRuslan Bukin CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 761*f17c4e38SRuslan Bukin rv = KERN_RESOURCE_SHORTAGE; 762*f17c4e38SRuslan Bukin goto out; 763*f17c4e38SRuslan Bukin } 764*f17c4e38SRuslan Bukin goto retry; 765*f17c4e38SRuslan Bukin } 766*f17c4e38SRuslan Bukin 767*f17c4e38SRuslan Bukin orig_l3 = pmap_load(l3); 768*f17c4e38SRuslan Bukin KASSERT(!pmap_l3_valid(orig_l3), ("l3 is valid")); 769*f17c4e38SRuslan Bukin 770*f17c4e38SRuslan Bukin /* New mapping */ 771*f17c4e38SRuslan Bukin pmap_store(l3, new_l3); 772*f17c4e38SRuslan Bukin pmap_resident_count_inc(pmap, 1); 773*f17c4e38SRuslan Bukin dsb(ishst); 774*f17c4e38SRuslan Bukin 775*f17c4e38SRuslan Bukin rv = KERN_SUCCESS; 776*f17c4e38SRuslan Bukin out: 777*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 778*f17c4e38SRuslan Bukin 779*f17c4e38SRuslan Bukin return (rv); 780*f17c4e38SRuslan Bukin } 781*f17c4e38SRuslan Bukin 782*f17c4e38SRuslan Bukin /* 783*f17c4e38SRuslan Bukin * Remove a single SMMU entry. 784*f17c4e38SRuslan Bukin */ 785*f17c4e38SRuslan Bukin int 786*f17c4e38SRuslan Bukin pmap_smmu_remove(pmap_t pmap, vm_offset_t va) 787*f17c4e38SRuslan Bukin { 788*f17c4e38SRuslan Bukin pt_entry_t *pte; 789*f17c4e38SRuslan Bukin int lvl; 790*f17c4e38SRuslan Bukin int rc; 791*f17c4e38SRuslan Bukin 792*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 793*f17c4e38SRuslan Bukin 794*f17c4e38SRuslan Bukin pte = pmap_pte(pmap, va, &lvl); 795*f17c4e38SRuslan Bukin KASSERT(lvl == 3, 796*f17c4e38SRuslan Bukin ("Invalid SMMU pagetable level: %d != 3", lvl)); 797*f17c4e38SRuslan Bukin 798*f17c4e38SRuslan Bukin if (pte != NULL) { 799*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 800*f17c4e38SRuslan Bukin pmap_clear(pte); 801*f17c4e38SRuslan Bukin rc = KERN_SUCCESS; 802*f17c4e38SRuslan Bukin } else 803*f17c4e38SRuslan Bukin rc = KERN_FAILURE; 804*f17c4e38SRuslan Bukin 805*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 806*f17c4e38SRuslan Bukin 807*f17c4e38SRuslan Bukin return (rc); 808*f17c4e38SRuslan Bukin } 809*f17c4e38SRuslan Bukin 810*f17c4e38SRuslan Bukin /* 811*f17c4e38SRuslan Bukin * Remove all the allocated L1, L2 pages from SMMU pmap. 812*f17c4e38SRuslan Bukin * All the L3 entires must be cleared in advance, otherwise 813*f17c4e38SRuslan Bukin * this function panics. 814*f17c4e38SRuslan Bukin */ 815*f17c4e38SRuslan Bukin void 816*f17c4e38SRuslan Bukin iommu_pmap_remove_pages(pmap_t pmap) 817*f17c4e38SRuslan Bukin { 818*f17c4e38SRuslan Bukin pd_entry_t l0e, *l1, l1e, *l2, l2e; 819*f17c4e38SRuslan Bukin pt_entry_t *l3, l3e; 820*f17c4e38SRuslan Bukin vm_page_t m, m0, m1; 821*f17c4e38SRuslan Bukin vm_offset_t sva; 822*f17c4e38SRuslan Bukin vm_paddr_t pa; 823*f17c4e38SRuslan Bukin vm_paddr_t pa0; 824*f17c4e38SRuslan Bukin vm_paddr_t pa1; 825*f17c4e38SRuslan Bukin int i, j, k, l; 826*f17c4e38SRuslan Bukin 827*f17c4e38SRuslan Bukin PMAP_LOCK(pmap); 828*f17c4e38SRuslan Bukin 829*f17c4e38SRuslan Bukin for (sva = VM_MINUSER_ADDRESS, i = iommu_l0_index(sva); 830*f17c4e38SRuslan Bukin (i < Ln_ENTRIES && sva < VM_MAXUSER_ADDRESS); i++) { 831*f17c4e38SRuslan Bukin l0e = pmap->pm_l0[i]; 832*f17c4e38SRuslan Bukin if ((l0e & ATTR_DESCR_VALID) == 0) { 833*f17c4e38SRuslan Bukin sva += IOMMU_L0_SIZE; 834*f17c4e38SRuslan Bukin continue; 835*f17c4e38SRuslan Bukin } 836*f17c4e38SRuslan Bukin pa0 = l0e & ~ATTR_MASK; 837*f17c4e38SRuslan Bukin m0 = PHYS_TO_VM_PAGE(pa0); 838*f17c4e38SRuslan Bukin l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0); 839*f17c4e38SRuslan Bukin 840*f17c4e38SRuslan Bukin for (j = iommu_l1_index(sva); j < Ln_ENTRIES; j++) { 841*f17c4e38SRuslan Bukin l1e = l1[j]; 842*f17c4e38SRuslan Bukin if ((l1e & ATTR_DESCR_VALID) == 0) { 843*f17c4e38SRuslan Bukin sva += IOMMU_L1_SIZE; 844*f17c4e38SRuslan Bukin continue; 845*f17c4e38SRuslan Bukin } 846*f17c4e38SRuslan Bukin if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) { 847*f17c4e38SRuslan Bukin sva += IOMMU_L1_SIZE; 848*f17c4e38SRuslan Bukin continue; 849*f17c4e38SRuslan Bukin } 850*f17c4e38SRuslan Bukin pa1 = l1e & ~ATTR_MASK; 851*f17c4e38SRuslan Bukin m1 = PHYS_TO_VM_PAGE(pa1); 852*f17c4e38SRuslan Bukin l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1); 853*f17c4e38SRuslan Bukin 854*f17c4e38SRuslan Bukin for (k = iommu_l2_index(sva); k < Ln_ENTRIES; k++) { 855*f17c4e38SRuslan Bukin l2e = l2[k]; 856*f17c4e38SRuslan Bukin if ((l2e & ATTR_DESCR_VALID) == 0) { 857*f17c4e38SRuslan Bukin sva += IOMMU_L2_SIZE; 858*f17c4e38SRuslan Bukin continue; 859*f17c4e38SRuslan Bukin } 860*f17c4e38SRuslan Bukin pa = l2e & ~ATTR_MASK; 861*f17c4e38SRuslan Bukin m = PHYS_TO_VM_PAGE(pa); 862*f17c4e38SRuslan Bukin l3 = (pt_entry_t *)PHYS_TO_DMAP(pa); 863*f17c4e38SRuslan Bukin 864*f17c4e38SRuslan Bukin for (l = iommu_l3_index(sva); l < Ln_ENTRIES; 865*f17c4e38SRuslan Bukin l++, sva += IOMMU_L3_SIZE) { 866*f17c4e38SRuslan Bukin l3e = l3[l]; 867*f17c4e38SRuslan Bukin if ((l3e & ATTR_DESCR_VALID) == 0) 868*f17c4e38SRuslan Bukin continue; 869*f17c4e38SRuslan Bukin panic("%s: l3e found for va %jx\n", 870*f17c4e38SRuslan Bukin __func__, sva); 871*f17c4e38SRuslan Bukin } 872*f17c4e38SRuslan Bukin 873*f17c4e38SRuslan Bukin vm_page_unwire_noq(m1); 874*f17c4e38SRuslan Bukin vm_page_unwire_noq(m); 875*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 876*f17c4e38SRuslan Bukin vm_page_free(m); 877*f17c4e38SRuslan Bukin pmap_clear(&l2[k]); 878*f17c4e38SRuslan Bukin } 879*f17c4e38SRuslan Bukin 880*f17c4e38SRuslan Bukin vm_page_unwire_noq(m0); 881*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 882*f17c4e38SRuslan Bukin vm_page_free(m1); 883*f17c4e38SRuslan Bukin pmap_clear(&l1[j]); 884*f17c4e38SRuslan Bukin } 885*f17c4e38SRuslan Bukin 886*f17c4e38SRuslan Bukin pmap_resident_count_dec(pmap, 1); 887*f17c4e38SRuslan Bukin vm_page_free(m0); 888*f17c4e38SRuslan Bukin pmap_clear(&pmap->pm_l0[i]); 889*f17c4e38SRuslan Bukin } 890*f17c4e38SRuslan Bukin 891*f17c4e38SRuslan Bukin KASSERT(pmap->pm_stats.resident_count == 0, 892*f17c4e38SRuslan Bukin ("Invalid resident count %jd", pmap->pm_stats.resident_count)); 893*f17c4e38SRuslan Bukin 894*f17c4e38SRuslan Bukin PMAP_UNLOCK(pmap); 895*f17c4e38SRuslan Bukin } 896