1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/gr/kernel_graphics.h" 25 #include "kernel/gpu/mem_mgr/mem_mgr.h" 26 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 27 #include "kernel/gpu/intr/engine_idx.h" 28 #include "nvRmReg.h" 29 30 #include "ctrl/ctrl0080/ctrl0080fifo.h" 31 32 void 33 kgraphicsInitFecsRegistryOverrides_GP100 34 ( 35 OBJGPU *pGpu, 36 KernelGraphics *pKernelGraphics 37 ) 38 { 39 NvU32 data; 40 41 // init the FECS buffer attributes before allocating buffer 42 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_CTXSW_LOG, &data) == NV_OK) 43 { 44 NvBool bIntrFallback = NV_FALSE; 45 NvBool bIntr = NV_FALSE; 46 NvBool bLog = NV_FALSE; 47 48 switch (data) 49 { 50 case NV_REG_STR_RM_CTXSW_LOG_ENABLE_INTR_APC: 51 bIntrFallback = NV_TRUE; 52 // Intentional fall-through 53 case NV_REG_STR_RM_CTXSW_LOG_ENABLE_INTR: 54 bIntr = NV_TRUE; 55 // Intentional fall-through 56 case NV_REG_STR_RM_CTXSW_LOG_ENABLE: 57 bLog = NV_TRUE; 58 break; 59 default: 60 break; 61 } 62 63 kgraphicsSetBottomHalfCtxswLoggingEnabled(pGpu, pKernelGraphics, bIntrFallback); 64 kgraphicsSetIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics, bIntr); 65 kgraphicsSetCtxswLoggingSupported(pGpu, pKernelGraphics, bLog); 66 } 67 68 fecsSetRecordsPerIntr(pGpu, pKernelGraphics, NV_REG_STR_RM_CTXSW_LOG_RECORDS_PER_INTR_DEFAULT); 69 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_CTXSW_LOG_RECORDS_PER_INTR, &data) == NV_OK) 70 { 71 fecsSetRecordsPerIntr(pGpu, pKernelGraphics, data); 72 } 73 } 74 75 /*! 76 * @brief Allocate common local/global buffers that are required by the graphics context for GfxP Pool 77 * 78 * @param[in] pGpu 79 * @param[in] pKernelGraphics 80 * @param[in] gfid host or guest gfid 81 * @param[in] pKernelGraphicsContext graphics context - if valid allocate local 82 */ 83 NV_STATUS 84 kgraphicsAllocGrGlobalCtxBuffers_GP100 85 ( 86 OBJGPU *pGpu, 87 KernelGraphics *pKernelGraphics, 88 NvU32 gfid, 89 KernelGraphicsContext *pKernelGraphicsContext 90 ) 91 { 92 extern NV_STATUS kgraphicsAllocGrGlobalCtxBuffers_GM200(OBJGPU *pGpu, KernelGraphics *pKernelGraphics, NvU32 gfid, KernelGraphicsContext *pKernelGraphicsContext); 93 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 94 NvU64 allocFlags = MEMDESC_FLAGS_NONE; 95 NV_STATUS status; 96 CTX_BUF_POOL_INFO *pCtxBufPool; 97 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo; 98 99 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 100 101 pCtxBufPool = NULL; 102 if (pKernelGraphicsContext != NULL) 103 { 104 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast; 105 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 106 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast)); 107 108 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer; 109 110 // 111 // if we already have local buffers allocated, return as we may get 112 // called multiple times per-channel 113 // 114 if (pCtxBuffers->bAllocated) 115 return NV_OK; 116 117 // check for allocating local buffers in VPR memory (don't want for global memory) 118 if ( 119 pKernelGraphicsContextUnicast->bVprChannel) 120 allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED; 121 122 // If allocated per channel, ensure allocations goes into Suballocator if available 123 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 124 } 125 else 126 { 127 pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 128 NV_ASSERT_OK_OR_RETURN(ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_GR_GLOBAL, 129 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &pCtxBufPool)); 130 } 131 132 // Don't use context buffer pool for VF allocations managed by host RM. 133 if (ctxBufPoolIsSupported(pGpu) && (pCtxBufPool != NULL)) 134 { 135 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 136 } 137 138 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 139 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 140 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE); 141 142 status = kgraphicsAllocGrGlobalCtxBuffers_GM200(pGpu, pKernelGraphics, gfid, pKernelGraphicsContext); 143 144 return status; 145 } 146 147 NV_STATUS 148 kgraphicsAllocGlobalCtxBuffers_GP100 149 ( 150 OBJGPU *pGpu, 151 KernelGraphics *pKernelGraphics, 152 NvU32 gfid 153 ) 154 { 155 CTX_BUF_POOL_INFO *pCtxBufPool = NULL; 156 NvU64 allocFlags = 0; 157 NvU32 fecsBufferSize = 0; 158 NvU32 fecsBufferAlign = 0x0; 159 GR_GLOBALCTX_BUFFERS *pCtxBuffers; 160 GR_BUFFER_ATTR *pCtxAttr; 161 162 // SKIP FECS buffer allocation for Virtual context 163 if (IS_GFID_VF(gfid)) 164 { 165 return NV_OK; 166 } 167 168 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 169 170 pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid]; 171 pCtxAttr = pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr; 172 NV_ASSERT_OK_OR_RETURN( 173 ctxBufPoolGetGlobalPool(pGpu, 174 CTX_BUF_ID_GR_GLOBAL, 175 RM_ENGINE_TYPE_GR(pKernelGraphics->instance), 176 &pCtxBufPool)); 177 178 if (pCtxBufPool != NULL) 179 { 180 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 181 } 182 183 // TODO: Need this check for vGPU configs without SRIOV support? 184 if (!IS_VIRTUAL(pGpu) || IS_VIRTUAL_WITH_SRIOV(pGpu)) 185 { 186 NvU32 engineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT; 187 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 188 189 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 190 fecsBufferSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size; 191 fecsBufferAlign = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].alignment; 192 } 193 194 // Allocate the FECS buffer if the right regkey (RmCtxswLog) is enabled 195 if ((fecsBufferSize > 0) && 196 kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics)) 197 { 198 NvBool bIsFbBroken; 199 MEMORY_DESCRIPTOR **ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT]; 200 201 bIsFbBroken = pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) || 202 pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_ALL_INST_IN_SYSMEM); 203 204 if (bIsFbBroken) 205 { 206 pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList = ADDRLIST_SYSMEM_ONLY; 207 pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr = NV_MEMORY_UNCACHED; 208 } 209 210 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 211 memdescCreate(ppMemDesc, pGpu, 212 fecsBufferSize, // size 213 fecsBufferAlign, // alignment 214 NV_TRUE, // physically contiguous 215 ADDR_UNKNOWN, 216 pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr, 217 allocFlags | MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS | MEMDESC_FLAGS_GPU_PRIVILEGED)); 218 219 if ((*ppMemDesc)->_addressSpace == ADDR_FBMEM) 220 memdescSetGpuCacheAttrib(*ppMemDesc, NV_MEMORY_CACHED); 221 222 if ((allocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0) 223 { 224 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 225 226 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 227 NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool)); 228 } 229 230 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 231 memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList)); 232 } 233 234 return NV_OK; 235 } 236 237 /** 238 * @brief Services the GRn_FECS_LOG interrupts. 239 * 240 * @returns Zero, because KernelGraphics opts out of stuck interrupt detection. 241 */ 242 NvU32 243 kgraphicsServiceInterrupt_GP100 244 ( 245 OBJGPU *pGpu, 246 KernelGraphics *pKernelGraphics, 247 IntrServiceServiceInterruptArguments *pParams 248 ) 249 { 250 NvU32 grIdx = pKernelGraphics->instance; 251 252 NV_ASSERT_OR_RETURN(pParams != NULL, 0); 253 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn_FECS_LOG(grIdx), 0); 254 255 // if MIG is disabled and an access has been made to any GR engine ID > 0, fail 256 if (!IS_MIG_IN_USE(pGpu) && (grIdx != 0)) 257 { 258 NV_ASSERT_FAILED("GR[1-7]_FECS_LOG is not supported if MIG is disabled!"); 259 return 0; 260 } 261 262 if ((pGpu->fecsCtxswLogConsumerCount > 0) && 263 (kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics))) 264 { 265 if (fecsClearIntrPendingIfPending(pGpu, pKernelGraphics)) 266 { 267 nvEventBufferFecsCallback(pGpu, (void*)pKernelGraphics); 268 } 269 } 270 return 0; 271 } 272 273 /** 274 * @brief Clears the stall interrupt leaf vector and return whether to call ServiceStall. 275 * @details Normally there's no need to override this function; however, 276 * the FECS_LOG engine idxs do not have real interrupt vectors to clear. 277 * This implementation just tells INTR to continue with servicing. 278 * 279 * @returns NV_TRUE indicating the interrupt should be serviced. 280 */ 281 NvBool 282 kgraphicsClearInterrupt_GP100 283 ( 284 OBJGPU *pGpu, 285 KernelGraphics *pKernelGraphics, 286 IntrServiceClearInterruptArguments *pParams 287 ) 288 { 289 NvU32 grIdx = pKernelGraphics->instance; 290 291 NV_ASSERT_OR_RETURN(pParams != NULL, NV_FALSE); 292 NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn_FECS_LOG(grIdx), 0); 293 294 return NV_TRUE; 295 } 296 297