1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @file 26 * @brief VOLTA specific HALs for UVM routines reside in this file 27 */ 28 29 #include "core/core.h" 30 #include "nvRmReg.h" 31 #include "gpu/gpu.h" 32 #include "gpu/mmu/kern_gmmu.h" 33 #include "gpu/uvm/uvm.h" 34 #include "os/os.h" 35 #include "gpu/mem_mgr/mem_mgr.h" 36 #include "gpu/mem_mgr/mem_desc.h" 37 #include "gpu/bus/kern_bus.h" 38 #include "rmapi/event.h" 39 40 #include "class/clc365.h" 41 #include "ctrl/ctrlc365.h" 42 #include "published/volta/gv100/dev_fb.h" 43 44 NV_STATUS 45 uvmSetupAccessCntrBuffer_GV100 46 ( 47 OBJGPU *pGpu, 48 OBJUVM *pUvm 49 ) 50 { 51 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 52 NvU64 vaddr; 53 NV_STATUS status = NV_OK; 54 55 // Return if guest RM is with no sriov 56 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 57 (IS_VIRTUAL(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 58 { 59 return NV_OK; 60 } 61 62 if (!pUvm->accessCntrBuffer.pUvmAccessCntrMemDesc) 63 { 64 return NV_ERR_INVALID_OBJECT_BUFFER; 65 } 66 67 status = kbusMapCpuInvisibleBar2Aperture_HAL(pGpu, pKernelBus, pUvm->accessCntrBuffer.pUvmAccessCntrMemDesc, 68 &vaddr, pUvm->accessCntrBuffer.pUvmAccessCntrMemDesc->Size, 0, GPU_GFID_PF); 69 if (status != NV_OK) 70 { 71 return status; 72 } 73 pUvm->accessCntrBuffer.bar2UvmAccessCntrBufferAddr = vaddr; 74 75 uvmProgramWriteAccessCntrBufferAddress_HAL(pGpu, pUvm, vaddr); 76 uvmProgramAccessCntrBufferEnabled_HAL(pGpu, pUvm, NV_FALSE); 77 78 return NV_OK; 79 } 80 81 NV_STATUS 82 uvmDisableAccessCntr_GV100 83 ( 84 OBJGPU *pGpu, 85 OBJUVM *pUvm, 86 NvBool bIsErrorRecovery 87 ) 88 { 89 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 90 RMTIMEOUT timeout; 91 NvU32 putPtr; 92 NvU32 getPtr; 93 NV_STATUS status = NV_OK; 94 95 status = gpuSanityCheckRegisterAccess(pGpu, 0, NULL); 96 if (status != NV_OK) 97 return status; 98 99 if (!bIsErrorRecovery && kgmmuTestAccessCounterWriteNak_HAL(pGpu, pKernelGmmu)) 100 { 101 NV_PRINTF(LEVEL_ERROR, 102 "Forcing bIsErrorRecovery = NV_TRUE because of WRITE_NACK.\n"); 103 bIsErrorRecovery = NV_TRUE; 104 } 105 106 uvmProgramAccessCntrBufferEnabled_HAL(pGpu, pUvm, NV_FALSE); 107 108 // 109 // Check for any pending notifications which might be pending in pipe to ensure 110 // they don't show up later when the buffer is enabled again. To ensure that HW sets the 111 // correct notifications PUSHED status in priv, perform a read to ensure that EN == FALSE. 112 // If PUSHED == TRUE, RM will check the PUT pointer and if updated, it will wait for valid 113 // bit to show up for all packets and then reset the buffer 114 // 115 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0); 116 if (!uvmIsAccessCntrBufferEnabled_HAL(pGpu, pUvm)) 117 { 118 while (!uvmIsAccessCntrBufferPushed_HAL(pGpu, pUvm)) 119 { 120 if (gpuCheckTimeout(pGpu, &timeout) == NV_ERR_TIMEOUT) 121 { 122 NV_PRINTF(LEVEL_ERROR, 123 "Timeout waiting for HW to write notification buffer.\n"); 124 DBG_BREAKPOINT(); 125 return NV_ERR_TIMEOUT; 126 } 127 } 128 129 // 130 // If called from error recovery, we can't wait for packet to show up as notification packets 131 // could be the source of error 132 // 133 if (bIsErrorRecovery) 134 goto done; 135 136 // If PUT pointer is updated, wait for VALID packets to show up and reset the packets 137 uvmReadAccessCntrBufferPutPtr_HAL(pGpu, pUvm, &putPtr); 138 uvmReadAccessCntrBufferGetPtr_HAL(pGpu, pUvm, &getPtr); 139 if (getPtr != putPtr) 140 { 141 MEMORY_DESCRIPTOR *pMemDesc = RMCFG_FEATURE_PLATFORM_GSP ? 142 pUvm->accessCntrBuffer.pUvmAccessCntrMemDesc : 143 pUvm->accessCntrBuffer.pUvmAccessCntrAllocMemDesc; 144 NvU8 *pAccessCntrBufferPage; 145 NvU32 entriesPerPage = RM_PAGE_SIZE / NVC365_NOTIFY_BUF_SIZE; 146 NvU32 pageSizeModBufSize = RM_PAGE_SIZE % NVC365_NOTIFY_BUF_SIZE; 147 NvU32 maxEntryCount = pMemDesc->Size / NVC365_NOTIFY_BUF_SIZE; 148 NvU32 inPageGetPtr; 149 NvP64 pAddr; 150 NvP64 pPriv; 151 152 NV_ASSERT_OR_RETURN(pageSizeModBufSize == 0, NV_ERR_INVALID_OPERATION); 153 154 // Map one buffer page and wait for packets to become valid 155 status = memdescMap(pMemDesc, (getPtr / entriesPerPage) * RM_PAGE_SIZE, RM_PAGE_SIZE, 156 NV_TRUE, NV_PROTECT_READ_WRITE, &pAddr, &pPriv); 157 if (status != NV_OK) 158 { 159 NV_PRINTF(LEVEL_ERROR, "Failed to map access counter buffer while disabling it: %d\n", 160 status); 161 return status; 162 } 163 164 while (getPtr != putPtr) 165 { 166 pAccessCntrBufferPage = (NvU8 *)pAddr; 167 inPageGetPtr = getPtr % entriesPerPage; 168 169 // Wait for an entry to be come valid 170 while (!DRF_VAL_MW(C365, _NOTIFY_BUF_ENTRY, _VALID, (NvU32 *)&pAccessCntrBufferPage[inPageGetPtr * NVC365_NOTIFY_BUF_SIZE])) 171 osSchedule(); 172 173 portMemSet((void *)(&pAccessCntrBufferPage[inPageGetPtr * NVC365_NOTIFY_BUF_SIZE]), 0, NVC365_NOTIFY_BUF_SIZE); 174 getPtr = ((getPtr + 1 == maxEntryCount) ? 0 : (getPtr + 1)); 175 176 // Map another page with entries to clear 177 if (getPtr % entriesPerPage == 0) 178 { 179 memdescUnmap(pMemDesc, NV_TRUE, osGetCurrentProcess(), pAddr, pPriv); 180 status = memdescMap(pMemDesc, (getPtr / entriesPerPage) * RM_PAGE_SIZE, RM_PAGE_SIZE, 181 NV_TRUE, NV_PROTECT_READ_WRITE, &pAddr, &pPriv); 182 if (status != NV_OK) 183 { 184 NV_PRINTF(LEVEL_ERROR, "Failed to map access counter buffer while disabling it: %d\n", 185 status); 186 187 // Write get progress so far, all entries in [get, put) 188 // are valid or will become valid. 189 uvmWriteAccessCntrBufferGetPtr_HAL(pGpu, pUvm, getPtr); 190 return status; 191 } 192 } 193 } 194 195 uvmWriteAccessCntrBufferGetPtr_HAL(pGpu, pUvm, getPtr); 196 197 memdescUnmap(pMemDesc, NV_TRUE, osGetCurrentProcess(), pAddr, pPriv); 198 } 199 } 200 else 201 { 202 NV_PRINTF(LEVEL_ERROR, "Failed disabling notification buffer.\n"); 203 DBG_BREAKPOINT(); 204 return NV_ERR_TIMEOUT; 205 } 206 207 done: 208 return NV_OK; 209 } 210 211 NV_STATUS 212 uvmUnloadAccessCntrBuffer_GV100(OBJGPU *pGpu, OBJUVM *pUvm) 213 { 214 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 215 216 // Return if guest RM is with no sriov 217 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 218 (IS_VIRTUAL(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 219 { 220 return NV_OK; 221 } 222 223 uvmDisableAccessCntr_HAL(pGpu, pUvm, NV_FALSE); 224 kbusUnmapCpuInvisibleBar2Aperture_HAL(pGpu, pKernelBus, pUvm->accessCntrBuffer.pUvmAccessCntrMemDesc, 225 pUvm->accessCntrBuffer.bar2UvmAccessCntrBufferAddr, GPU_GFID_PF); 226 pUvm->accessCntrBuffer.bar2UvmAccessCntrBufferAddr = 0; 227 228 return NV_OK; 229 } 230 231 NV_STATUS 232 uvmDestroyAccessCntrBuffer_GV100(OBJGPU *pGpu, OBJUVM *pUvm) 233 { 234 if(pUvm == NULL) 235 { 236 return NV_WARN_NULL_OBJECT; 237 } 238 239 // Return if guest RM is with no sriov 240 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 241 (IS_VIRTUAL(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 242 { 243 return NV_OK; 244 } 245 246 memdescFree(pUvm->accessCntrBuffer.pUvmAccessCntrAllocMemDesc); 247 memdescDestroy(pUvm->accessCntrBuffer.pUvmAccessCntrAllocMemDesc); 248 249 pUvm->accessCntrBuffer.pUvmAccessCntrAllocMemDesc = NULL; 250 pUvm->accessCntrBuffer.accessCntrBufferSize = 0; 251 252 return NV_OK; 253 } 254 255 NV_STATUS 256 uvmInitAccessCntrBuffer_GV100(OBJGPU *pGpu, OBJUVM *pUvm) 257 { 258 NV_STATUS status; 259 MEMORY_DESCRIPTOR *pUvmAccessCntrBufferDesc; 260 NvP64 pAddr; 261 NvP64 pPriv; 262 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 263 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 264 265 NvU32 accessCntrBufferAperture = 0; 266 NvU32 accessCntrBufferAttr = 0; 267 NV2080_CTRL_INTERNAL_UVM_GET_ACCESS_CNTR_BUFFER_SIZE_PARAMS getSizeParams = {0}; 268 269 // Return if guest RM is with no sriov 270 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu) || 271 (IS_VIRTUAL(pGpu) && gpuIsWarBug200577889SriovHeavyEnabled(pGpu))) 272 { 273 return NV_OK; 274 } 275 276 // Issue control to fetch buffer size from physical 277 status = pUvm->pRmApi->Control(pUvm->pRmApi, 278 pUvm->hClient, 279 pUvm->hSubdevice, 280 NV2080_CTRL_CMD_INTERNAL_UVM_GET_ACCESS_CNTR_BUFFER_SIZE, 281 &getSizeParams, 282 sizeof(getSizeParams)); 283 if (status != NV_OK) 284 { 285 return status; 286 } 287 288 pUvm->accessCntrBuffer.accessCntrBufferSize = getSizeParams.bufferSize; 289 290 accessCntrBufferAperture = ADDR_SYSMEM; 291 accessCntrBufferAttr = NV_MEMORY_CACHED; 292 memdescOverrideInstLoc(DRF_VAL(_REG_STR_RM, _INST_LOC_4, _UVM_FAULT_BUFFER_REPLAYABLE, pGpu->instLocOverrides4), 293 "UVM access counter", &accessCntrBufferAperture, &accessCntrBufferAttr); 294 295 status = memdescCreate(&pUvmAccessCntrBufferDesc, pGpu, pUvm->accessCntrBuffer.accessCntrBufferSize, 0, 296 NV_FALSE, accessCntrBufferAperture, accessCntrBufferAttr, MEMDESC_FLAGS_LOST_ON_SUSPEND); 297 if (status != NV_OK) 298 { 299 return status; 300 } 301 302 // 303 // GPU doesn't read accessCounter notification buffer memory, so if buffer is in sysmem, 304 // ensure that GpuCacheAttr is set to UNCACHED as having a vol bit set in PTEs will ensure HUB 305 // uses L2Bypass mode and it will save extra cycles to cache in L2 while MMU will write notification packets. 306 // 307 if (accessCntrBufferAperture == ADDR_SYSMEM && 308 pKernelGmmu->getProperty(pKernelGmmu, PDB_PROP_KGMMU_SYSMEM_FAULT_BUFFER_GPU_UNCACHED)) 309 { 310 memdescSetGpuCacheAttrib(pUvmAccessCntrBufferDesc, NV_MEMORY_UNCACHED); 311 } 312 313 status = memdescAlloc(pUvmAccessCntrBufferDesc); 314 if (status != NV_OK) 315 { 316 memdescDestroy(pUvmAccessCntrBufferDesc); 317 return status; 318 } 319 320 memdescSetName(pGpu, pUvmAccessCntrBufferDesc, NV_RM_SURF_NAME_ACCESS_COUNTER_BUFFER, NULL); 321 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pUvmAccessCntrBufferDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 322 323 status = memdescMap(pUvmAccessCntrBufferDesc, 0, 324 memdescGetSize(pUvmAccessCntrBufferDesc), NV_TRUE, 325 NV_PROTECT_READ_WRITE, &pAddr, &pPriv); 326 if (status != NV_OK) 327 { 328 memdescFree(pUvmAccessCntrBufferDesc); 329 memdescDestroy(pUvmAccessCntrBufferDesc); 330 return status; 331 } 332 portMemSet(NvP64_VALUE(pAddr), 0, memdescGetSize(pUvmAccessCntrBufferDesc)); 333 334 memdescUnmap(pUvmAccessCntrBufferDesc, NV_TRUE, osGetCurrentProcess(), pAddr, pPriv); 335 336 pUvm->accessCntrBuffer.pUvmAccessCntrAllocMemDesc = pUvmAccessCntrBufferDesc; 337 338 return status; 339 } 340 341 NV_STATUS 342 uvmResetAccessCntrBuffer_GV100(OBJGPU *pGpu, OBJUVM *pUvm, NvU32 counterType) 343 { 344 switch(counterType) 345 { 346 case NVC365_CTRL_ACCESS_COUNTER_TYPE_ALL: 347 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_NOTIFY_BUFFER_CLR, _ALL_COUNTERS, _CLR); 348 break; 349 case NVC365_CTRL_ACCESS_COUNTER_TYPE_MIMC: 350 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_NOTIFY_BUFFER_CLR, _MIMC, _CLR); 351 break; 352 case NVC365_CTRL_ACCESS_COUNTER_TYPE_MOMC: 353 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_NOTIFY_BUFFER_CLR, _MOMC, _CLR); 354 break; 355 default: 356 return NV_ERR_INVALID_ARGUMENT; 357 } 358 return NV_OK; 359 } 360 361 NV_STATUS 362 uvmAccessCntrSetCounterLimit_GV100(OBJGPU *pGpu, OBJUVM *pUvm, NvU32 type, NvU32 limit) 363 { 364 if (type == NVC365_CTRL_ACCESS_COUNTER_MIMC_LIMIT) 365 { 366 switch(limit) 367 { 368 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_NONE: 369 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MIMC_USE_LIMIT, _NONE); 370 break; 371 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_QTR: 372 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MIMC_USE_LIMIT, _QTR); 373 break; 374 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_HALF: 375 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MIMC_USE_LIMIT, _HALF); 376 break; 377 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_FULL: 378 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MIMC_USE_LIMIT, _FULL); 379 break; 380 default: 381 return NV_ERR_INVALID_ARGUMENT; 382 } 383 } 384 else if (type == NVC365_CTRL_ACCESS_COUNTER_MOMC_LIMIT) 385 { 386 switch(limit) 387 { 388 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_NONE: 389 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MOMC_USE_LIMIT, _NONE); 390 break; 391 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_QTR: 392 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MOMC_USE_LIMIT, _QTR); 393 break; 394 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_HALF: 395 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MOMC_USE_LIMIT, _HALF); 396 break; 397 case NVC365_CTRL_ACCESS_COUNTER_USE_LIMIT_FULL: 398 GPU_FLD_WR_DRF_DEF(pGpu, _PFB_NISO, _ACCESS_COUNTER_CONFIG, _MOMC_USE_LIMIT, _FULL); 399 break; 400 default: 401 return NV_ERR_INVALID_ARGUMENT; 402 } 403 } 404 else 405 return NV_ERR_INVALID_ARGUMENT; 406 407 return NV_OK; 408 } 409