1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #define NVOC_KERN_GMMU_H_PRIVATE_ACCESS_ALLOWED 25 26 #include "gpu/mmu/kern_gmmu.h" 27 #include "gpu/mem_mgr/mem_mgr.h" 28 #include "vgpu/vgpu_events.h" 29 #include "nv_sriov_defines.h" 30 #include "kernel/gpu/intr/intr.h" 31 32 #include "mmu/gmmu_fmt.h" 33 #include "published/hopper/gh100/dev_mmu.h" 34 #include "published/hopper/gh100/dev_fault.h" 35 #include "published/hopper/gh100/dev_vm.h" 36 #include "published/hopper/gh100/dev_vm_addendum.h" 37 38 /*! 39 * Check if a specific GMMU format version is supported. 40 */ 41 NvBool 42 kgmmuFmtIsVersionSupported_GH10X(KernelGmmu *pKernelGmmu, NvU32 version) 43 { 44 return (version == GMMU_FMT_VERSION_3); 45 } 46 47 /*! 48 * Initialize the GMMU format families. 49 */ 50 NV_STATUS 51 kgmmuFmtFamiliesInit_GH100(OBJGPU *pGpu, KernelGmmu* pKernelGmmu) 52 { 53 NvU32 i; 54 NvU32 pdePcfHw = 0; 55 NvU32 pdePcfSw = 0; 56 NvU32 ptePcfHw = 0; 57 NvU32 ptePcfSw = 0; 58 59 // Initialize the sparse encoding in the PDE PCF field for V3 60 GMMU_FMT_FAMILY *pFam = pKernelGmmu->pFmtFamilies[GMMU_FMT_VERSION_3 - 1]; 61 62 if (pFam != NULL) 63 { 64 // 1.Initialize sparsePde 65 pdePcfSw |= (1 << SW_MMU_PCF_SPARSE_IDX); 66 pdePcfSw |= (1 << SW_MMU_PCF_ATS_ALLOWED_IDX); 67 NV_ASSERT_OR_RETURN((kgmmuTranslatePdePcfFromSw_HAL(pKernelGmmu, pdePcfSw, &pdePcfHw) == NV_OK), 68 NV_ERR_INVALID_ARGUMENT); 69 gmmuFieldSetAperture(&pFam->pde.fldAperture, GMMU_APERTURE_INVALID, 70 pFam->sparsePde.v8); 71 nvFieldSet32(&pFam->pde.fldPdePcf, pdePcfHw, pFam->sparsePde.v8); 72 73 // 2.Initialize sparsePdeMulti 74 for (i = 0; i < MMU_FMT_MAX_SUB_LEVELS; ++i) 75 { 76 const GMMU_FMT_PDE *pPdeFmt = &pFam->pdeMulti.subLevels[i]; 77 gmmuFieldSetAperture(&pPdeFmt->fldAperture, GMMU_APERTURE_INVALID, 78 pFam->sparsePdeMulti.v8); 79 // Set PDE PCF sparse bit only for sub-level 0 for PdeMulti 80 if (i == 0) 81 { 82 nvFieldSet32(&pPdeFmt->fldPdePcf, pdePcfHw, pFam->sparsePdeMulti.v8); 83 } 84 } 85 86 // 3.Initialize nv4kPte 87 ptePcfSw |= (1 << SW_MMU_PCF_NV4K_IDX); 88 nvFieldSetBool(&pFam->pte.fldValid, NV_FALSE, pFam->nv4kPte.v8); 89 NV_ASSERT_OR_RETURN((kgmmuTranslatePtePcfFromSw_HAL(pKernelGmmu, ptePcfSw, &ptePcfHw) == NV_OK), 90 NV_ERR_INVALID_ARGUMENT); 91 nvFieldSet32(&pFam->pte.fldPtePcf, ptePcfHw, pFam->nv4kPte.v8); 92 93 // 4.Initialize sparsePte 94 ptePcfSw = (1 << SW_MMU_PCF_SPARSE_IDX); 95 nvFieldSetBool(&pFam->pte.fldValid, NV_FALSE, pFam->sparsePte.v8); 96 NV_ASSERT_OR_RETURN((kgmmuTranslatePtePcfFromSw_HAL(pKernelGmmu, ptePcfSw, &ptePcfHw) == NV_OK), 97 NV_ERR_INVALID_ARGUMENT); 98 nvFieldSet32(&pFam->pte.fldPtePcf, ptePcfHw, pFam->sparsePte.v8); 99 } 100 101 return NV_OK; 102 } 103 104 #define PTE_PCF_INVALID_LIST(fn) \ 105 fn(INVALID) \ 106 fn(NO_VALID_4KB_PAGE) \ 107 fn(SPARSE) \ 108 fn(MAPPING_NOWHERE) 109 110 #define PTE_PCF_VALID_LIST(fn) \ 111 fn(PRIVILEGE_RW_ATOMIC_CACHED_ACD) \ 112 fn(PRIVILEGE_RW_ATOMIC_CACHED_ACE) \ 113 fn(PRIVILEGE_RW_ATOMIC_UNCACHED_ACD) \ 114 fn(PRIVILEGE_RW_ATOMIC_UNCACHED_ACE) \ 115 fn(PRIVILEGE_RW_NO_ATOMIC_UNCACHED_ACE) \ 116 fn(PRIVILEGE_RW_NO_ATOMIC_CACHED_ACE) \ 117 fn(PRIVILEGE_RO_ATOMIC_UNCACHED_ACE) \ 118 fn(PRIVILEGE_RO_NO_ATOMIC_UNCACHED_ACE) \ 119 fn(PRIVILEGE_RO_NO_ATOMIC_CACHED_ACE) \ 120 fn(REGULAR_RW_ATOMIC_CACHED_ACD) \ 121 fn(REGULAR_RW_ATOMIC_CACHED_ACE) \ 122 fn(REGULAR_RW_ATOMIC_UNCACHED_ACD) \ 123 fn(REGULAR_RW_ATOMIC_UNCACHED_ACE) \ 124 fn(REGULAR_RW_NO_ATOMIC_CACHED_ACD) \ 125 fn(REGULAR_RW_NO_ATOMIC_CACHED_ACE) \ 126 fn(REGULAR_RW_NO_ATOMIC_UNCACHED_ACD) \ 127 fn(REGULAR_RW_NO_ATOMIC_UNCACHED_ACE) \ 128 fn(REGULAR_RO_ATOMIC_CACHED_ACD) \ 129 fn(REGULAR_RO_ATOMIC_CACHED_ACE) \ 130 fn(REGULAR_RO_ATOMIC_UNCACHED_ACD) \ 131 fn(REGULAR_RO_ATOMIC_UNCACHED_ACE) \ 132 fn(REGULAR_RO_NO_ATOMIC_CACHED_ACD) \ 133 fn(REGULAR_RO_NO_ATOMIC_CACHED_ACE) \ 134 fn(REGULAR_RO_NO_ATOMIC_UNCACHED_ACD) \ 135 fn(REGULAR_RO_NO_ATOMIC_UNCACHED_ACE) 136 137 #define PTE_PCF_HW_FROM_SW(name) \ 138 case (SW_MMU_PTE_PCF_##name): \ 139 { \ 140 *pPtePcfHw = NV_MMU_VER3_PTE_PCF_##name; \ 141 break; \ 142 } 143 144 #define PTE_PCF_SW_FROM_HW(name) \ 145 case (NV_MMU_VER3_PTE_PCF_##name): \ 146 { \ 147 *pPtePcfSw = SW_MMU_PTE_PCF_##name; \ 148 break; \ 149 } 150 151 // 152 // Takes a SW PTE PCF and translates to HW PTE PCF 153 // If bit patterns is not supported by HW, return NV_ERR_NOT_SUPPORTED 154 // 155 NV_STATUS 156 kgmmuTranslatePtePcfFromSw_GH100 157 ( 158 KernelGmmu *pKernelGmmu, 159 NvU32 ptePcfSw, 160 NvU32 *pPtePcfHw 161 ) 162 { 163 switch (ptePcfSw) 164 { 165 PTE_PCF_INVALID_LIST(PTE_PCF_HW_FROM_SW) 166 PTE_PCF_VALID_LIST(PTE_PCF_HW_FROM_SW) 167 168 default: 169 { 170 NV_PRINTF(LEVEL_ERROR, "Unsupported SW PTE PCF pattern requested : %x\n", ptePcfSw); 171 return NV_ERR_NOT_SUPPORTED; 172 } 173 } 174 175 return NV_OK; 176 } 177 178 NV_STATUS 179 kgmmuTranslatePtePcfFromHw_GH100 180 ( 181 KernelGmmu *pKernelGmmu, 182 NvU32 ptePcfHw, 183 NvBool bPteValid, 184 NvU32 *pPtePcfSw 185 ) 186 { 187 if (!bPteValid) 188 { 189 switch (ptePcfHw) 190 { 191 PTE_PCF_INVALID_LIST(PTE_PCF_SW_FROM_HW) 192 193 default: return NV_ERR_NOT_SUPPORTED; 194 } 195 } 196 else 197 { 198 switch (ptePcfHw) 199 { 200 PTE_PCF_VALID_LIST(PTE_PCF_SW_FROM_HW) 201 202 default: 203 { 204 NV_PRINTF(LEVEL_ERROR, "Unsupported HW PTE PCF pattern requested : %x\n", ptePcfHw); 205 return NV_ERR_NOT_SUPPORTED; 206 } 207 } 208 } 209 210 return NV_OK; 211 } 212 213 #define PDE_PCF_INVALID_LIST(fn) \ 214 fn(INVALID_ATS_ALLOWED) \ 215 fn(SPARSE_ATS_ALLOWED) \ 216 fn(INVALID_ATS_NOT_ALLOWED) \ 217 fn(SPARSE_ATS_NOT_ALLOWED) 218 219 #define PDE_PCF_VALID_LIST(fn) \ 220 fn(VALID_CACHED_ATS_ALLOWED) \ 221 fn(VALID_CACHED_ATS_NOT_ALLOWED) \ 222 fn(VALID_UNCACHED_ATS_ALLOWED) \ 223 fn(VALID_UNCACHED_ATS_NOT_ALLOWED) 224 225 #define PDE_PCF_HW_FROM_SW(name) \ 226 case (SW_MMU_PDE_PCF_##name): \ 227 { \ 228 *pPdePcfHw = NV_MMU_VER3_PDE_PCF_##name; \ 229 break; \ 230 } 231 232 #define PDE_PCF_SW_FROM_HW(name) \ 233 case (NV_MMU_VER3_PDE_PCF_##name): \ 234 { \ 235 *pPdePcfSw = SW_MMU_PDE_PCF_##name; \ 236 break; \ 237 } 238 239 // 240 // Takes a SW PDE PCF and translates to HW PDE PCF 241 // If a bit pattern is not supported by HW, return NV_ERR_NOT_SUPPORTED 242 // 243 NV_STATUS 244 kgmmuTranslatePdePcfFromSw_GH100 245 ( 246 KernelGmmu *pKernelGmmu, 247 NvU32 pdePcfSw, 248 NvU32 *pPdePcfHw 249 ) 250 { 251 switch (pdePcfSw) 252 { 253 PDE_PCF_INVALID_LIST(PDE_PCF_HW_FROM_SW) 254 PDE_PCF_VALID_LIST(PDE_PCF_HW_FROM_SW) 255 256 default: return NV_ERR_NOT_SUPPORTED; 257 } 258 259 return NV_OK; 260 } 261 262 // 263 // Takes a HW PDE PCF and translates to SW PDE PCF 264 // If a bit pattern is not supported by SW, return NV_ERR_NOT_SUPPORTED 265 // 266 NV_STATUS 267 kgmmuTranslatePdePcfFromHw_GH100 268 ( 269 KernelGmmu *pKernelGmmu, 270 NvU32 pdePcfHw, 271 GMMU_APERTURE aperture, 272 NvU32 *pPdePcfSw 273 ) 274 { 275 if (!aperture) 276 { 277 switch (pdePcfHw) 278 { 279 PDE_PCF_INVALID_LIST(PDE_PCF_SW_FROM_HW) 280 281 default: return NV_ERR_NOT_SUPPORTED; 282 } 283 } 284 else 285 { 286 switch (pdePcfHw) 287 { 288 PDE_PCF_VALID_LIST(PDE_PCF_SW_FROM_HW) 289 290 default: return NV_ERR_NOT_SUPPORTED; 291 } 292 } 293 294 return NV_OK; 295 } 296 297 /* 298 * @brief Validates fabric base address. 299 * 300 * @param pKernelGmmu 301 * @param fabricBaseAddr 302 * 303 * @returns On success, NV_OK. 304 * On failure, returns NV_ERR_XXX. 305 */ 306 NV_STATUS 307 kgmmuValidateFabricBaseAddress_GH100 308 ( 309 KernelGmmu *pKernelGmmu, 310 NvU64 fabricBaseAddr 311 ) 312 { 313 OBJGPU *pGpu = ENG_GET_GPU(pKernelGmmu); 314 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 315 NvU64 fbSizeBytes; 316 317 fbSizeBytes = pMemoryManager->Ram.fbTotalMemSizeMb << 20; 318 319 // 320 // Hopper SKUs will be paired with NVSwitches (Laguna Seca) supporting 2K 321 // mapslots that can cover 512GB each. Make sure that the fabric base 322 // address being used is valid to cover whole frame buffer. 323 // 324 325 // Check if fabric address is aligned to mapslot size. 326 if (fabricBaseAddr & (NVBIT64(39) - 1)) 327 { 328 return NV_ERR_INVALID_ARGUMENT; 329 } 330 331 // Align fbSize to mapslot size. 332 fbSizeBytes = RM_ALIGN_UP(fbSizeBytes, NVBIT64(39)); 333 334 return NV_OK; 335 } 336 337 /*! 338 * @brief Get the engine ID associated with the Graphics Engine 339 */ 340 NvU32 341 kgmmuGetGraphicsEngineId_GH100 342 ( 343 KernelGmmu *pKernelGmmu 344 ) 345 { 346 return NV_PFAULT_MMU_ENG_ID_GRAPHICS; 347 } 348 349 NV_STATUS 350 kgmmuGetFaultRegisterMappings_GH100 351 ( 352 OBJGPU *pGpu, 353 KernelGmmu *pKernelGmmu, 354 NvU32 index, 355 NvP64 *pFaultBufferGet, 356 NvP64 *pFaultBufferPut, 357 NvP64 *pFaultBufferInfo, 358 NvP64 *pHubIntr, 359 NvP64 *pHubIntrEnSet, 360 NvP64 *pHubIntrEnClear, 361 NvU32 *faultMask, 362 NvP64 *pPrefetchCtrl 363 ) 364 { 365 DEVICE_MAPPING *pMapping = gpuGetDeviceMapping(pGpu, DEVICE_INDEX_GPU, 0); 366 NvP64 bar0Mapping = NV_PTR_TO_NvP64(pMapping->gpuNvAddr); 367 368 NV_ASSERT_OR_RETURN((index < NUM_FAULT_BUFFERS), NV_ERR_INVALID_ARGUMENT); 369 370 // 371 // If Hopper CC is not enabled or GSP doesn't entirely own the HW fault buffers 372 // use the Turing HAL 373 // 374 if (!gpuIsCCFeatureEnabled(pGpu) || !gpuIsGspOwnedFaultBuffersEnabled(pGpu)) 375 { 376 return kgmmuGetFaultRegisterMappings_TU102(pGpu, pKernelGmmu, index, 377 pFaultBufferGet, pFaultBufferPut, 378 pFaultBufferInfo, pHubIntr, 379 pHubIntrEnSet, pHubIntrEnClear, 380 faultMask, pPrefetchCtrl); 381 } 382 383 *pFaultBufferGet = 0; 384 *pFaultBufferInfo = 0; 385 *pHubIntr = 0; 386 *pHubIntrEnSet = 0; 387 *pHubIntrEnClear = 0; 388 *faultMask = 0; 389 *pPrefetchCtrl = 0; 390 391 // 392 // When Hopper CC is enabled, we repurpose the access counter registers to 393 // hold the PUT pointer of the shadow buffers. Only GSP-RM can write the 394 // PUT pointer to these PRIs. CPU has read-only access to these PRIs 395 // 396 if (index == REPLAYABLE_FAULT_BUFFER) 397 { 398 Intr *pIntr = GPU_GET_INTR(pGpu); 399 NvU32 intrVector = intrGetVectorFromEngineId(pGpu, pIntr, MC_ENGINE_IDX_REPLAYABLE_FAULT_CPU, NV_FALSE); 400 struct GMMU_FAULT_BUFFER *pFaultBuffer; 401 GMMU_CLIENT_SHADOW_FAULT_BUFFER *pClientShadowFaultBuf; 402 FAULT_BUFFER_SHARED_MEMORY *pFaultBufSharedMem; 403 NvU32 leafReg; 404 NvU32 leafBit; 405 406 leafReg = NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(intrVector); 407 leafBit = NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_BIT(intrVector); 408 409 pFaultBuffer = &pKernelGmmu->mmuFaultBuffer[GPU_GFID_PF]; 410 pClientShadowFaultBuf = 411 KERNEL_POINTER_FROM_NvP64(GMMU_CLIENT_SHADOW_FAULT_BUFFER *, 412 pFaultBuffer->pClientShadowFaultBuffer[index]); 413 414 pFaultBufSharedMem = 415 KERNEL_POINTER_FROM_NvP64(FAULT_BUFFER_SHARED_MEMORY *, 416 pClientShadowFaultBuf->pFaultBufferSharedMemoryAddress); 417 418 *pHubIntr = NvP64_PLUS_OFFSET(bar0Mapping, 419 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF(leafReg))); 420 *pHubIntrEnSet = NvP64_PLUS_OFFSET(bar0Mapping, 421 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_SET(leafReg))); 422 *pHubIntrEnClear = NvP64_PLUS_OFFSET(bar0Mapping, 423 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_CLEAR(leafReg))); 424 *faultMask = NVBIT(leafBit); 425 *pFaultBufferGet = (NvU32*) &(pFaultBufSharedMem->swGetIndex); 426 *pFaultBufferPut = NvP64_PLUS_OFFSET(bar0Mapping, 427 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_ACCESS_COUNTER_NOTIFY_BUFFER_HI)); 428 *pPrefetchCtrl = NvP64_PLUS_OFFSET(bar0Mapping, 429 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_MMU_PAGE_FAULT_CTRL)); 430 } 431 else if (index == NON_REPLAYABLE_FAULT_BUFFER) 432 { 433 *pFaultBufferPut = NvP64_PLUS_OFFSET(bar0Mapping, 434 GPU_GET_VREG_OFFSET(pGpu, NV_VIRTUAL_FUNCTION_PRIV_ACCESS_COUNTER_NOTIFY_BUFFER_LO)); 435 } 436 else 437 { 438 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_ARGUMENT); 439 } 440 441 return NV_OK; 442 } 443 444 NV_STATUS 445 kgmmuFaultBufferAllocSharedMemory_GH100 446 ( 447 OBJGPU *pGpu, 448 KernelGmmu *pKernelGmmu, 449 FAULT_BUFFER_TYPE index 450 ) 451 { 452 NV_STATUS status; 453 GMMU_CLIENT_SHADOW_FAULT_BUFFER *pClientShadowFaultBuffer; 454 MEMORY_DESCRIPTOR *pMemDesc; 455 NvU64 flags = MEMDESC_FLAGS_NONE; 456 457 if (pKernelGmmu->getProperty(pKernelGmmu, PDB_PROP_KGMMU_FAULT_BUFFER_DISABLED)) 458 { 459 NV_PRINTF(LEVEL_ERROR, "Fault-Buffer is disabled. Flush Seq memory cannot be created\n"); 460 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_STATE); 461 } 462 463 if (index != REPLAYABLE_FAULT_BUFFER) 464 { 465 return NV_OK; 466 } 467 468 if (!gpuIsCCFeatureEnabled(pGpu) || !gpuIsGspOwnedFaultBuffersEnabled(pGpu)) 469 { 470 return NV_OK; 471 } 472 473 // 474 // On systems with SEV enabled, the fault buffer flush sequence memory should be allocated 475 // in unprotected sysmem as GSP will be writing to this location to let the guest 476 // know a flush has finished. 477 // 478 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 479 480 pClientShadowFaultBuffer = &pKernelGmmu->mmuFaultBuffer[GPU_GFID_PF].clientShadowFaultBuffer[index]; 481 status = memdescCreate(&pMemDesc, pGpu, 482 sizeof(FAULT_BUFFER_SHARED_MEMORY), RM_PAGE_SIZE, 483 NV_FALSE, ADDR_SYSMEM, NV_MEMORY_UNCACHED, 484 flags); 485 if (status != NV_OK) 486 { 487 return status; 488 } 489 490 status = memdescAlloc(pMemDesc); 491 if (status != NV_OK) 492 { 493 goto destroy_memdesc; 494 } 495 496 status = memdescMap(pMemDesc, 0, 497 memdescGetSize(pMemDesc), 498 NV_TRUE, NV_PROTECT_READ_WRITE, 499 &pClientShadowFaultBuffer->pFaultBufferSharedMemoryAddress, 500 &pClientShadowFaultBuffer->pFaultBufferSharedMemoryPriv); 501 if (status != NV_OK) 502 { 503 goto free_memory; 504 } 505 506 pClientShadowFaultBuffer->pFaultBufferSharedMemDesc = pMemDesc; 507 508 return NV_OK; 509 510 free_memory: 511 memdescFree(pMemDesc); 512 513 destroy_memdesc: 514 memdescDestroy(pMemDesc); 515 516 return status; 517 } 518 519 void 520 kgmmuFaultBufferFreeSharedMemory_GH100 521 ( 522 OBJGPU *pGpu, 523 KernelGmmu *pKernelGmmu, 524 FAULT_BUFFER_TYPE index 525 ) 526 { 527 MEMORY_DESCRIPTOR *pMemDesc; 528 GMMU_CLIENT_SHADOW_FAULT_BUFFER *pClientShadowFaultBuffer; 529 530 if (index != REPLAYABLE_FAULT_BUFFER) 531 { 532 return; 533 } 534 535 if (!gpuIsCCFeatureEnabled(pGpu) || !gpuIsGspOwnedFaultBuffersEnabled(pGpu)) 536 { 537 return; 538 } 539 540 pClientShadowFaultBuffer = &pKernelGmmu->mmuFaultBuffer[GPU_GFID_PF].clientShadowFaultBuffer[index]; 541 pMemDesc = pClientShadowFaultBuffer->pFaultBufferSharedMemDesc; 542 543 memdescUnmap(pMemDesc, 544 NV_TRUE, osGetCurrentProcess(), 545 pClientShadowFaultBuffer->pFaultBufferSharedMemoryAddress, 546 pClientShadowFaultBuffer->pFaultBufferSharedMemoryPriv); 547 548 memdescFree(pMemDesc); 549 memdescDestroy(pMemDesc); 550 return; 551 } 552 553 /* 554 * @brief GSP client can use this function to initiate a replayable fault buffer flush when the 555 * HW fault buffer is owned by GSP. 556 */ 557 NV_STATUS 558 kgmmuIssueReplayableFaultBufferFlush_GH100 559 ( 560 OBJGPU *pGpu, 561 KernelGmmu *pKernelGmmu 562 ) 563 { 564 GMMU_CLIENT_SHADOW_FAULT_BUFFER *pClientShadowFaultBuffer; 565 FAULT_BUFFER_SHARED_MEMORY *pFaultBufSharedMem; 566 NvU32 gfid; 567 volatile NvU32 *pFlushSeqAddr; 568 NvU32 replayableFlushSeqValue; 569 NV_STATUS status; 570 RMTIMEOUT timeout; 571 572 if (!gpuIsCCFeatureEnabled(pGpu) || !gpuIsGspOwnedFaultBuffersEnabled(pGpu) || !IS_GSP_CLIENT(pGpu)) 573 { 574 return NV_OK; 575 } 576 577 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 578 579 pClientShadowFaultBuffer = &pKernelGmmu->mmuFaultBuffer[gfid].clientShadowFaultBuffer[REPLAYABLE_FAULT_BUFFER]; 580 pFaultBufSharedMem = KERNEL_POINTER_FROM_NvP64(FAULT_BUFFER_SHARED_MEMORY *, 581 pClientShadowFaultBuffer->pFaultBufferSharedMemoryAddress); 582 pFlushSeqAddr = (NvU32*) &(pFaultBufSharedMem->flushBufferSeqNum); 583 replayableFlushSeqValue = *pFlushSeqAddr; 584 585 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0); 586 GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_DOORBELL, NV_DOORBELL_NOTIFY_LEAF_SERVICE_REPLAYABLE_FAULT_FLUSH_HANDLE); 587 588 while (replayableFlushSeqValue + 1 != *pFlushSeqAddr) 589 { 590 status = gpuCheckTimeout(pGpu, &timeout); 591 592 if (status != NV_OK) 593 { 594 NV_PRINTF(LEVEL_ERROR, "gpuCheckTimeout failed, status = 0x%x\n", status); 595 return status; 596 } 597 osSpinLoop(); 598 } 599 600 return NV_OK; 601 } 602 603 /* 604 * @brief When Hopper Confidential Compute is enabled, the put index of the 605 * client replayable/non-replayable shadow buffers gets stored in the 606 * access counter PRIs. This function is used by Kernel RM to read the put index. 607 * 608 * @param[in] pGpu OBJGPU pointer 609 * @param[in] pKernelGmmu KernelGmmu pointer 610 * @param[in] type Replayable/Non-replayable fault buffer 611 * 612 * @returns NvU32 613 */ 614 NvU32 615 kgmmuReadShadowBufPutIndex_GH100 616 ( 617 OBJGPU *pGpu, 618 KernelGmmu *pKernelGmmu, 619 FAULT_BUFFER_TYPE type 620 ) 621 { 622 NvU32 val; 623 if (type == REPLAYABLE_FAULT_BUFFER) 624 { 625 val = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_REPLAYABLE_FAULT_SHADOW_BUFFER_PUT); 626 } 627 else 628 { 629 val = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_NON_REPLAYABLE_FAULT_SHADOW_BUFFER_PUT); 630 val = DRF_VAL(_VIRTUAL_FUNCTION_PRIV, _NON_REPLAYABLE_FAULT_SHADOW_BUFFER_PUT, _PTR, val); 631 } 632 return val; 633 } 634