1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERN_GMMU_H_PRIVATE_ACCESS_ALLOWED 25 26 #include "gpu/mmu/kern_gmmu.h" 27 #include "gpu/mem_mgr/mem_mgr.h" 28 #include "vgpu/vgpu_events.h" 29 #include "nv_sriov_defines.h" 30 31 #include "mmu/gmmu_fmt.h" 32 #include "published/hopper/gh100/dev_mmu.h" 33 #include "published/hopper/gh100/dev_fault.h" 34 35 /*! 36 * Check if a specific GMMU format version is supported. 37 */ 38 NvBool 39 kgmmuFmtIsVersionSupported_GH10X(KernelGmmu *pKernelGmmu, NvU32 version) 40 { 41 return (version == GMMU_FMT_VERSION_3); 42 } 43 44 /*! 45 * Initialize the GMMU format families. 46 */ 47 NV_STATUS 48 kgmmuFmtFamiliesInit_GH100(OBJGPU *pGpu, KernelGmmu* pKernelGmmu) 49 { 50 NvU32 i; 51 NvU32 pdePcfHw = 0; 52 NvU32 pdePcfSw = 0; 53 NvU32 ptePcfHw = 0; 54 NvU32 ptePcfSw = 0; 55 56 // Initialize the sparse encoding in the PDE PCF field for V3 57 GMMU_FMT_FAMILY *pFam = pKernelGmmu->pFmtFamilies[GMMU_FMT_VERSION_3 - 1]; 58 59 if (pFam != NULL) 60 { 61 // 1.Initialize sparsePde 62 pdePcfSw |= (1 << SW_MMU_PCF_SPARSE_IDX); 63 pdePcfSw |= (1 << SW_MMU_PCF_ATS_ALLOWED_IDX); 64 NV_ASSERT_OR_RETURN((kgmmuTranslatePdePcfFromSw_HAL(pKernelGmmu, pdePcfSw, &pdePcfHw) == NV_OK), 65 NV_ERR_INVALID_ARGUMENT); 66 gmmuFieldSetAperture(&pFam->pde.fldAperture, GMMU_APERTURE_INVALID, 67 pFam->sparsePde.v8); 68 nvFieldSet32(&pFam->pde.fldPdePcf, pdePcfHw, pFam->sparsePde.v8); 69 70 // 2.Initialize sparsePdeMulti 71 for (i = 0; i < MMU_FMT_MAX_SUB_LEVELS; ++i) 72 { 73 const GMMU_FMT_PDE *pPdeFmt = &pFam->pdeMulti.subLevels[i]; 74 gmmuFieldSetAperture(&pPdeFmt->fldAperture, GMMU_APERTURE_INVALID, 75 pFam->sparsePdeMulti.v8); 76 // Set PDE PCF sparse bit only for sub-level 0 for PdeMulti 77 if (i == 0) 78 { 79 nvFieldSet32(&pPdeFmt->fldPdePcf, pdePcfHw, pFam->sparsePdeMulti.v8); 80 } 81 } 82 83 // 3.Initialize nv4kPte 84 ptePcfSw |= (1 << SW_MMU_PCF_NV4K_IDX); 85 nvFieldSetBool(&pFam->pte.fldValid, NV_FALSE, pFam->nv4kPte.v8); 86 NV_ASSERT_OR_RETURN((kgmmuTranslatePtePcfFromSw_HAL(pKernelGmmu, ptePcfSw, &ptePcfHw) == NV_OK), 87 NV_ERR_INVALID_ARGUMENT); 88 nvFieldSet32(&pFam->pte.fldPtePcf, ptePcfHw, pFam->nv4kPte.v8); 89 90 // 4.Initialize sparsePte 91 ptePcfSw = (1 << SW_MMU_PCF_SPARSE_IDX); 92 nvFieldSetBool(&pFam->pte.fldValid, NV_FALSE, pFam->sparsePte.v8); 93 NV_ASSERT_OR_RETURN((kgmmuTranslatePtePcfFromSw_HAL(pKernelGmmu, ptePcfSw, &ptePcfHw) == NV_OK), 94 NV_ERR_INVALID_ARGUMENT); 95 nvFieldSet32(&pFam->pte.fldPtePcf, ptePcfHw, pFam->sparsePte.v8); 96 } 97 98 return NV_OK; 99 } 100 101 #define PTE_PCF_INVALID_LIST(fn) \ 102 fn(INVALID) \ 103 fn(NO_VALID_4KB_PAGE) \ 104 fn(SPARSE) \ 105 fn(MAPPING_NOWHERE) 106 107 #define PTE_PCF_VALID_LIST(fn) \ 108 fn(PRIVILEGE_RW_ATOMIC_CACHED_ACD) \ 109 fn(PRIVILEGE_RW_ATOMIC_CACHED_ACE) \ 110 fn(PRIVILEGE_RW_ATOMIC_UNCACHED_ACD) \ 111 fn(PRIVILEGE_RW_ATOMIC_UNCACHED_ACE) \ 112 fn(PRIVILEGE_RW_NO_ATOMIC_UNCACHED_ACE) \ 113 fn(PRIVILEGE_RW_NO_ATOMIC_CACHED_ACE) \ 114 fn(PRIVILEGE_RO_ATOMIC_UNCACHED_ACE) \ 115 fn(PRIVILEGE_RO_NO_ATOMIC_UNCACHED_ACE) \ 116 fn(PRIVILEGE_RO_NO_ATOMIC_CACHED_ACE) \ 117 fn(REGULAR_RW_ATOMIC_CACHED_ACD) \ 118 fn(REGULAR_RW_ATOMIC_CACHED_ACE) \ 119 fn(REGULAR_RW_ATOMIC_UNCACHED_ACD) \ 120 fn(REGULAR_RW_ATOMIC_UNCACHED_ACE) \ 121 fn(REGULAR_RW_NO_ATOMIC_CACHED_ACD) \ 122 fn(REGULAR_RW_NO_ATOMIC_CACHED_ACE) \ 123 fn(REGULAR_RW_NO_ATOMIC_UNCACHED_ACD) \ 124 fn(REGULAR_RW_NO_ATOMIC_UNCACHED_ACE) \ 125 fn(REGULAR_RO_ATOMIC_CACHED_ACD) \ 126 fn(REGULAR_RO_ATOMIC_CACHED_ACE) \ 127 fn(REGULAR_RO_ATOMIC_UNCACHED_ACD) \ 128 fn(REGULAR_RO_ATOMIC_UNCACHED_ACE) \ 129 fn(REGULAR_RO_NO_ATOMIC_CACHED_ACD) \ 130 fn(REGULAR_RO_NO_ATOMIC_CACHED_ACE) \ 131 fn(REGULAR_RO_NO_ATOMIC_UNCACHED_ACD) \ 132 fn(REGULAR_RO_NO_ATOMIC_UNCACHED_ACE) 133 134 #define PTE_PCF_HW_FROM_SW(name) \ 135 case (SW_MMU_PTE_PCF_##name): \ 136 { \ 137 *pPtePcfHw = NV_MMU_VER3_PTE_PCF_##name; \ 138 break; \ 139 } 140 141 #define PTE_PCF_SW_FROM_HW(name) \ 142 case (NV_MMU_VER3_PTE_PCF_##name): \ 143 { \ 144 *pPtePcfSw = SW_MMU_PTE_PCF_##name; \ 145 break; \ 146 } 147 148 // 149 // Takes a SW PTE PCF and translates to HW PTE PCF 150 // If bit patterns is not supported by HW, return NV_ERR_NOT_SUPPORTED 151 // 152 NV_STATUS 153 kgmmuTranslatePtePcfFromSw_GH100 154 ( 155 KernelGmmu *pKernelGmmu, 156 NvU32 ptePcfSw, 157 NvU32 *pPtePcfHw 158 ) 159 { 160 switch (ptePcfSw) 161 { 162 PTE_PCF_INVALID_LIST(PTE_PCF_HW_FROM_SW) 163 PTE_PCF_VALID_LIST(PTE_PCF_HW_FROM_SW) 164 165 default: 166 { 167 NV_PRINTF(LEVEL_ERROR, "Unsupported SW PTE PCF pattern requested : %x\n", ptePcfSw); 168 return NV_ERR_NOT_SUPPORTED; 169 } 170 } 171 172 return NV_OK; 173 } 174 175 NV_STATUS 176 kgmmuTranslatePtePcfFromHw_GH100 177 ( 178 KernelGmmu *pKernelGmmu, 179 NvU32 ptePcfHw, 180 NvBool bPteValid, 181 NvU32 *pPtePcfSw 182 ) 183 { 184 if (!bPteValid) 185 { 186 switch (ptePcfHw) 187 { 188 PTE_PCF_INVALID_LIST(PTE_PCF_SW_FROM_HW) 189 190 default: return NV_ERR_NOT_SUPPORTED; 191 } 192 } 193 else 194 { 195 switch (ptePcfHw) 196 { 197 PTE_PCF_VALID_LIST(PTE_PCF_SW_FROM_HW) 198 199 default: 200 { 201 NV_PRINTF(LEVEL_ERROR, "Unsupported HW PTE PCF pattern requested : %x\n", ptePcfHw); 202 return NV_ERR_NOT_SUPPORTED; 203 } 204 } 205 } 206 207 return NV_OK; 208 } 209 210 #define PDE_PCF_INVALID_LIST(fn) \ 211 fn(INVALID_ATS_ALLOWED) \ 212 fn(SPARSE_ATS_ALLOWED) \ 213 fn(INVALID_ATS_NOT_ALLOWED) \ 214 fn(SPARSE_ATS_NOT_ALLOWED) 215 216 #define PDE_PCF_VALID_LIST(fn) \ 217 fn(VALID_CACHED_ATS_ALLOWED) \ 218 fn(VALID_CACHED_ATS_NOT_ALLOWED) \ 219 fn(VALID_UNCACHED_ATS_ALLOWED) \ 220 fn(VALID_UNCACHED_ATS_NOT_ALLOWED) 221 222 #define PDE_PCF_HW_FROM_SW(name) \ 223 case (SW_MMU_PDE_PCF_##name): \ 224 { \ 225 *pPdePcfHw = NV_MMU_VER3_PDE_PCF_##name; \ 226 break; \ 227 } 228 229 #define PDE_PCF_SW_FROM_HW(name) \ 230 case (NV_MMU_VER3_PDE_PCF_##name): \ 231 { \ 232 *pPdePcfSw = SW_MMU_PDE_PCF_##name; \ 233 break; \ 234 } 235 236 // 237 // Takes a SW PDE PCF and translates to HW PDE PCF 238 // If a bit pattern is not supported by HW, return NV_ERR_NOT_SUPPORTED 239 // 240 NV_STATUS 241 kgmmuTranslatePdePcfFromSw_GH100 242 ( 243 KernelGmmu *pKernelGmmu, 244 NvU32 pdePcfSw, 245 NvU32 *pPdePcfHw 246 ) 247 { 248 switch (pdePcfSw) 249 { 250 PDE_PCF_INVALID_LIST(PDE_PCF_HW_FROM_SW) 251 PDE_PCF_VALID_LIST(PDE_PCF_HW_FROM_SW) 252 253 default: return NV_ERR_NOT_SUPPORTED; 254 } 255 256 return NV_OK; 257 } 258 259 // 260 // Takes a HW PDE PCF and translates to SW PDE PCF 261 // If a bit pattern is not supported by SW, return NV_ERR_NOT_SUPPORTED 262 // 263 NV_STATUS 264 kgmmuTranslatePdePcfFromHw_GH100 265 ( 266 KernelGmmu *pKernelGmmu, 267 NvU32 pdePcfHw, 268 GMMU_APERTURE aperture, 269 NvU32 *pPdePcfSw 270 ) 271 { 272 if (!aperture) 273 { 274 switch (pdePcfHw) 275 { 276 PDE_PCF_INVALID_LIST(PDE_PCF_SW_FROM_HW) 277 278 default: return NV_ERR_NOT_SUPPORTED; 279 } 280 } 281 else 282 { 283 switch (pdePcfHw) 284 { 285 PDE_PCF_VALID_LIST(PDE_PCF_SW_FROM_HW) 286 287 default: return NV_ERR_NOT_SUPPORTED; 288 } 289 } 290 291 return NV_OK; 292 } 293 294 /* 295 * @brief Validates fabric base address. 296 * 297 * @param pKernelGmmu 298 * @param fabricBaseAddr 299 * 300 * @returns On success, NV_OK. 301 * On failure, returns NV_ERR_XXX. 302 */ 303 NV_STATUS 304 kgmmuValidateFabricBaseAddress_GH100 305 ( 306 KernelGmmu *pKernelGmmu, 307 NvU64 fabricBaseAddr 308 ) 309 { 310 OBJGPU *pGpu = ENG_GET_GPU(pKernelGmmu); 311 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 312 NvU64 fbSizeBytes; 313 314 fbSizeBytes = pMemoryManager->Ram.fbTotalMemSizeMb << 20; 315 316 // 317 // Hopper SKUs will be paired with NVSwitches (Laguna Seca) supporting 2K 318 // mapslots that can cover 512GB each. Make sure that the fabric base 319 // address being used is valid to cover whole frame buffer. 320 // 321 322 // Check if fabric address is aligned to mapslot size. 323 if (fabricBaseAddr & (NVBIT64(39) - 1)) 324 { 325 return NV_ERR_INVALID_ARGUMENT; 326 } 327 328 // Align fbSize to mapslot size. 329 fbSizeBytes = RM_ALIGN_UP(fbSizeBytes, NVBIT64(39)); 330 331 return NV_OK; 332 } 333 334 /*! 335 * @brief Get the engine ID associated with the Graphics Engine 336 */ 337 NvU32 338 kgmmuGetGraphicsEngineId_GH100 339 ( 340 KernelGmmu *pKernelGmmu 341 ) 342 { 343 return NV_PFAULT_MMU_ENG_ID_GRAPHICS; 344 } 345 346 NV_STATUS 347 kgmmuGetFaultRegisterMappings_GH100 348 ( 349 OBJGPU *pGpu, 350 KernelGmmu *pKernelGmmu, 351 NvU32 index, 352 NvP64 *pFaultBufferGet, 353 NvP64 *pFaultBufferPut, 354 NvP64 *pFaultBufferInfo, 355 NvP64 *pHubIntr, 356 NvP64 *pHubIntrEnSet, 357 NvP64 *pHubIntrEnClear, 358 NvU32 *faultMask, 359 NvP64 *pPrefetchCtrl 360 ) 361 { 362 return kgmmuGetFaultRegisterMappings_TU102(pGpu, pKernelGmmu, index, 363 pFaultBufferGet, pFaultBufferPut, 364 pFaultBufferInfo, pHubIntr, 365 pHubIntrEnSet, pHubIntrEnClear, 366 faultMask, pPrefetchCtrl); 367 } 368 369 NV_STATUS 370 kgmmuFaultBufferAllocSharedMemory_GH100 371 ( 372 OBJGPU *pGpu, 373 KernelGmmu *pKernelGmmu, 374 FAULT_BUFFER_TYPE index 375 ) 376 { 377 return NV_OK; 378 } 379 380 void 381 kgmmuFaultBufferFreeSharedMemory_GH100 382 ( 383 OBJGPU *pGpu, 384 KernelGmmu *pKernelGmmu, 385 FAULT_BUFFER_TYPE index 386 ) 387 { 388 return; 389 } 390 391 /* 392 * @brief GSP client can use this function to initiate a replayable fault buffer flush when the 393 * HW fault buffer is owned by GSP. 394 */ 395 NV_STATUS 396 kgmmuIssueReplayableFaultBufferFlush_GH100 397 ( 398 OBJGPU *pGpu, 399 KernelGmmu *pKernelGmmu 400 ) 401 { 402 403 return NV_OK; 404 } 405