1 /******************************************************************************* 2 Copyright (c) 2017-2021 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #include "uvm_types.h" 25 #include "uvm_forward_decl.h" 26 #include "uvm_global.h" 27 #include "uvm_hal.h" 28 #include "uvm_mmu.h" 29 #include "uvm_volta_fault_buffer.h" 30 #include "hwref/volta/gv100/dev_mmu.h" 31 #include "hwref/volta/gv100/dev_fault.h" 32 33 // Direct copy of make_pde_pascal and helpers, but adds NO_ATS in PDE1 34 #define MMU_BIG 0 35 #define MMU_SMALL 1 36 37 static NvU32 entries_per_index_volta(NvU32 depth) 38 { 39 UVM_ASSERT(depth < 5); 40 if (depth == 3) 41 return 2; 42 return 1; 43 } 44 45 static NvLength entry_offset_volta(NvU32 depth, NvU32 page_size) 46 { 47 UVM_ASSERT(depth < 5); 48 if (page_size == UVM_PAGE_SIZE_4K && depth == 3) 49 return MMU_SMALL; 50 return MMU_BIG; 51 } 52 53 static NvU64 single_pde_volta(uvm_mmu_page_table_alloc_t *phys_alloc, NvU32 depth) 54 { 55 NvU64 pde_bits = 0; 56 57 if (phys_alloc != NULL) { 58 NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_PDE_ADDRESS_SHIFT; 59 pde_bits |= HWCONST64(_MMU_VER2, PDE, IS_PDE, TRUE) | 60 HWCONST64(_MMU_VER2, PDE, VOL, TRUE); 61 62 switch (phys_alloc->addr.aperture) { 63 case UVM_APERTURE_SYS: 64 pde_bits |= HWCONST64(_MMU_VER2, PDE, APERTURE, SYSTEM_COHERENT_MEMORY) | 65 HWVALUE64(_MMU_VER2, PDE, ADDRESS_SYS, address); 66 break; 67 case UVM_APERTURE_VID: 68 pde_bits |= HWCONST64(_MMU_VER2, PDE, APERTURE, VIDEO_MEMORY) | 69 HWVALUE64(_MMU_VER2, PDE, ADDRESS_VID, address); 70 break; 71 default: 72 UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", phys_alloc->addr.aperture); 73 break; 74 } 75 76 // Volta GPUs on ATS-enabled systems, perform a parallel lookup on both 77 // ATS and GMMU page tables. For managed memory we need to prevent this 78 // parallel lookup since we would not get any GPU fault if the CPU has 79 // a valid mapping. Also, for external ranges that are known to be 80 // mapped entirely on the GMMU page table we can skip the ATS lookup 81 // for performance reasons. This bit is set in PDE1 (depth 2) and, 82 // therefore, it applies to the underlying 512MB VA range. 83 // 84 // UVM sets NO_ATS for all Volta+ mappings on ATS systems. This is fine 85 // because CUDA ensures that all managed and external allocations are 86 // properly compartmentalized in 512MB-aligned VA regions. For 87 // cudaHostRegister CUDA cannot control the VA range, but we rely on 88 // ATS for those allocations so they can't use the NO_ATS bit. 89 if (depth == 2 && g_uvm_global.ats.enabled) 90 pde_bits |= HWCONST64(_MMU_VER2, PDE, NO_ATS, TRUE); 91 } 92 93 return pde_bits; 94 } 95 96 static NvU64 big_half_pde_volta(uvm_mmu_page_table_alloc_t *phys_alloc) 97 { 98 NvU64 pde_bits = 0; 99 100 if (phys_alloc != NULL) { 101 NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_DUAL_PDE_ADDRESS_BIG_SHIFT; 102 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, VOL_BIG, TRUE); 103 104 switch (phys_alloc->addr.aperture) { 105 case UVM_APERTURE_SYS: 106 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_BIG, SYSTEM_COHERENT_MEMORY) | 107 HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_BIG_SYS, address); 108 break; 109 case UVM_APERTURE_VID: 110 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_BIG, VIDEO_MEMORY) | 111 HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_BIG_VID, address); 112 break; 113 default: 114 UVM_ASSERT_MSG(0, "Invalid big aperture %d\n", phys_alloc->addr.aperture); 115 break; 116 } 117 } 118 119 return pde_bits; 120 } 121 122 static NvU64 small_half_pde_volta(uvm_mmu_page_table_alloc_t *phys_alloc) 123 { 124 NvU64 pde_bits = 0; 125 126 if (phys_alloc != NULL) { 127 NvU64 address = phys_alloc->addr.address >> NV_MMU_VER2_DUAL_PDE_ADDRESS_SHIFT; 128 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, VOL_SMALL, TRUE); 129 130 switch (phys_alloc->addr.aperture) { 131 case UVM_APERTURE_SYS: 132 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_SMALL, SYSTEM_COHERENT_MEMORY); 133 pde_bits |= HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_SMALL_SYS, address); 134 break; 135 case UVM_APERTURE_VID: 136 pde_bits |= HWCONST64(_MMU_VER2, DUAL_PDE, APERTURE_SMALL, VIDEO_MEMORY); 137 pde_bits |= HWVALUE64(_MMU_VER2, DUAL_PDE, ADDRESS_SMALL_VID, address); 138 break; 139 default: 140 UVM_ASSERT_MSG(0, "Invalid small aperture %d\n", phys_alloc->addr.aperture); 141 break; 142 } 143 } 144 145 return pde_bits; 146 } 147 148 static void make_pde_volta(void *entry, uvm_mmu_page_table_alloc_t **phys_allocs, NvU32 depth) 149 { 150 NvU32 entry_count = entries_per_index_volta(depth); 151 NvU64 *entry_bits = (NvU64 *)entry; 152 153 if (entry_count == 1) { 154 *entry_bits = single_pde_volta(*phys_allocs, depth); 155 } 156 else if (entry_count == 2) { 157 entry_bits[MMU_BIG] = big_half_pde_volta(phys_allocs[MMU_BIG]); 158 entry_bits[MMU_SMALL] = small_half_pde_volta(phys_allocs[MMU_SMALL]); 159 160 // This entry applies to the whole dual PDE but is stored in the lower 161 // bits 162 entry_bits[MMU_BIG] |= HWCONST64(_MMU_VER2, DUAL_PDE, IS_PDE, TRUE); 163 } 164 else { 165 UVM_ASSERT_MSG(0, "Invalid number of entries per index: %d\n", entry_count); 166 } 167 } 168 169 // Direct copy of make_pte_pascal, but adds the bits necessary for 47-bit 170 // physical addressing 171 static NvU64 make_pte_volta(uvm_aperture_t aperture, NvU64 address, uvm_prot_t prot, NvU64 flags) 172 { 173 NvU8 aperture_bits = 0; 174 NvU64 pte_bits = 0; 175 176 UVM_ASSERT(prot != UVM_PROT_NONE); 177 UVM_ASSERT((flags & ~UVM_MMU_PTE_FLAGS_MASK) == 0); 178 179 // valid 0:0 180 pte_bits |= HWCONST64(_MMU_VER2, PTE, VALID, TRUE); 181 182 // aperture 2:1 183 if (aperture == UVM_APERTURE_SYS) 184 aperture_bits = NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY; 185 else if (aperture == UVM_APERTURE_VID) 186 aperture_bits = NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY; 187 else if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7) 188 aperture_bits = NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY; 189 else 190 UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", aperture); 191 192 pte_bits |= HWVALUE64(_MMU_VER2, PTE, APERTURE, aperture_bits); 193 194 // volatile 3:3 195 if (flags & UVM_MMU_PTE_FLAGS_CACHED) 196 pte_bits |= HWCONST64(_MMU_VER2, PTE, VOL, FALSE); 197 else 198 pte_bits |= HWCONST64(_MMU_VER2, PTE, VOL, TRUE); 199 200 // encrypted 4:4 201 pte_bits |= HWCONST64(_MMU_VER2, PTE, ENCRYPTED, FALSE); 202 203 // privilege 5:5 204 pte_bits |= HWCONST64(_MMU_VER2, PTE, PRIVILEGE, FALSE); 205 206 // read only 6:6 207 if (prot == UVM_PROT_READ_ONLY) 208 pte_bits |= HWCONST64(_MMU_VER2, PTE, READ_ONLY, TRUE); 209 else 210 pte_bits |= HWCONST64(_MMU_VER2, PTE, READ_ONLY, FALSE); 211 212 // atomic disable 7:7 213 if (prot == UVM_PROT_READ_WRITE_ATOMIC) 214 pte_bits |= HWCONST64(_MMU_VER2, PTE, ATOMIC_DISABLE, FALSE); 215 else 216 pte_bits |= HWCONST64(_MMU_VER2, PTE, ATOMIC_DISABLE, TRUE); 217 218 address >>= NV_MMU_VER2_PTE_ADDRESS_SHIFT; 219 if (aperture == UVM_APERTURE_SYS) { 220 // sys address 53:8 221 pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_SYS, address); 222 } 223 else { 224 NvU64 addr_lo = address & HWMASK64(_MMU_VER2, PTE, ADDRESS_VID); 225 NvU64 addr_hi = address >> HWSIZE(_MMU_VER2, PTE, ADDRESS_VID); 226 227 // vid address 32:8 for bits 36:12 of the physical address 228 pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_VID, addr_lo); 229 230 // comptagline 53:36 - this can be overloaded in some cases to reference 231 // a 47-bit physical address. Currently, the only known cases of this 232 // is for nvswitch, where peer id is the fabric id programmed for 233 // such peer mappings 234 pte_bits |= HWVALUE64(_MMU_VER2, PTE, COMPTAGLINE, addr_hi); 235 236 // peer id 35:33 237 if (aperture != UVM_APERTURE_VID) 238 pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_VID_PEER, UVM_APERTURE_PEER_ID(aperture)); 239 } 240 241 pte_bits |= HWVALUE64(_MMU_VER2, PTE, KIND, NV_MMU_PTE_KIND_PITCH); 242 243 return pte_bits; 244 } 245 246 static uvm_mmu_mode_hal_t volta_mmu_mode_hal; 247 248 uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_volta(NvU32 big_page_size) 249 { 250 static bool initialized = false; 251 252 UVM_ASSERT(big_page_size == UVM_PAGE_SIZE_64K || big_page_size == UVM_PAGE_SIZE_128K); 253 254 // TODO: Bug 1789555: RM should reject the creation of GPU VA spaces with 255 // 128K big page size for Pascal+ GPUs 256 if (big_page_size == UVM_PAGE_SIZE_128K) 257 return NULL; 258 259 if (!initialized) { 260 uvm_mmu_mode_hal_t *pascal_mmu_mode_hal = uvm_hal_mmu_mode_pascal(big_page_size); 261 UVM_ASSERT(pascal_mmu_mode_hal); 262 263 // The assumption made is that arch_hal->mmu_mode_hal() will be 264 // called under the global lock the first time, so check it here. 265 uvm_assert_mutex_locked(&g_uvm_global.global_lock); 266 267 volta_mmu_mode_hal = *pascal_mmu_mode_hal; 268 volta_mmu_mode_hal.make_pte = make_pte_volta; 269 volta_mmu_mode_hal.make_pde = make_pde_volta; 270 271 initialized = true; 272 } 273 274 return &volta_mmu_mode_hal; 275 } 276 277 uvm_mmu_engine_type_t uvm_hal_volta_mmu_engine_id_to_type(NvU16 mmu_engine_id) 278 { 279 if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_HOST0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_HOST13) 280 return UVM_MMU_ENGINE_TYPE_HOST; 281 282 if (mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_CE0 && mmu_engine_id <= NV_PFAULT_MMU_ENG_ID_CE8) 283 return UVM_MMU_ENGINE_TYPE_CE; 284 285 // We shouldn't be servicing faults from any other engines 286 UVM_ASSERT_MSG(mmu_engine_id >= NV_PFAULT_MMU_ENG_ID_GRAPHICS, "Unexpected engine ID: 0x%x\n", mmu_engine_id); 287 288 return UVM_MMU_ENGINE_TYPE_GRAPHICS; 289 } 290 291 NvU16 uvm_hal_volta_mmu_client_id_to_utlb_id(NvU16 client_id) 292 { 293 switch (client_id) { 294 case NV_PFAULT_CLIENT_GPC_RAST: 295 case NV_PFAULT_CLIENT_GPC_GCC: 296 case NV_PFAULT_CLIENT_GPC_GPCCS: 297 return UVM_VOLTA_GPC_UTLB_ID_RGG; 298 case NV_PFAULT_CLIENT_GPC_PE_0: 299 case NV_PFAULT_CLIENT_GPC_TPCCS_0: 300 case NV_PFAULT_CLIENT_GPC_T1_0: 301 case NV_PFAULT_CLIENT_GPC_T1_1: 302 return UVM_VOLTA_GPC_UTLB_ID_LTP0; 303 case NV_PFAULT_CLIENT_GPC_PE_1: 304 case NV_PFAULT_CLIENT_GPC_TPCCS_1: 305 case NV_PFAULT_CLIENT_GPC_T1_2: 306 case NV_PFAULT_CLIENT_GPC_T1_3: 307 return UVM_VOLTA_GPC_UTLB_ID_LTP1; 308 case NV_PFAULT_CLIENT_GPC_PE_2: 309 case NV_PFAULT_CLIENT_GPC_TPCCS_2: 310 case NV_PFAULT_CLIENT_GPC_T1_4: 311 case NV_PFAULT_CLIENT_GPC_T1_5: 312 return UVM_VOLTA_GPC_UTLB_ID_LTP2; 313 case NV_PFAULT_CLIENT_GPC_PE_3: 314 case NV_PFAULT_CLIENT_GPC_TPCCS_3: 315 case NV_PFAULT_CLIENT_GPC_T1_6: 316 case NV_PFAULT_CLIENT_GPC_T1_7: 317 return UVM_VOLTA_GPC_UTLB_ID_LTP3; 318 case NV_PFAULT_CLIENT_GPC_PE_4: 319 case NV_PFAULT_CLIENT_GPC_TPCCS_4: 320 case NV_PFAULT_CLIENT_GPC_T1_8: 321 case NV_PFAULT_CLIENT_GPC_T1_9: 322 return UVM_VOLTA_GPC_UTLB_ID_LTP4; 323 case NV_PFAULT_CLIENT_GPC_PE_5: 324 case NV_PFAULT_CLIENT_GPC_TPCCS_5: 325 case NV_PFAULT_CLIENT_GPC_T1_10: 326 case NV_PFAULT_CLIENT_GPC_T1_11: 327 return UVM_VOLTA_GPC_UTLB_ID_LTP5; 328 case NV_PFAULT_CLIENT_GPC_PE_6: 329 case NV_PFAULT_CLIENT_GPC_TPCCS_6: 330 case NV_PFAULT_CLIENT_GPC_T1_12: 331 case NV_PFAULT_CLIENT_GPC_T1_13: 332 return UVM_VOLTA_GPC_UTLB_ID_LTP6; 333 case NV_PFAULT_CLIENT_GPC_PE_7: 334 case NV_PFAULT_CLIENT_GPC_TPCCS_7: 335 case NV_PFAULT_CLIENT_GPC_T1_14: 336 case NV_PFAULT_CLIENT_GPC_T1_15: 337 return UVM_VOLTA_GPC_UTLB_ID_LTP7; 338 default: 339 UVM_ASSERT_MSG(false, "Invalid client value: 0x%x\n", client_id); 340 } 341 342 return 0; 343 } 344