1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/gr/kernel_graphics.h"
25 #include "kernel/gpu/mem_mgr/mem_mgr.h"
26 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
27 #include "kernel/gpu/intr/engine_idx.h"
28 #include "nvRmReg.h"
29 
30 #include "ctrl/ctrl0080/ctrl0080fifo.h"
31 
32 void
33 kgraphicsInitFecsRegistryOverrides_GP100
34 (
35     OBJGPU *pGpu,
36     KernelGraphics *pKernelGraphics
37 )
38 {
39     NvU32 data;
40 
41     // init the FECS buffer attributes before allocating buffer
42     if (osReadRegistryDword(pGpu, NV_REG_STR_RM_CTXSW_LOG, &data) == NV_OK)
43     {
44         NvBool bIntrFallback = NV_FALSE;
45         NvBool bIntr = NV_FALSE;
46         NvBool bLog = NV_FALSE;
47 
48         switch (data)
49         {
50             case NV_REG_STR_RM_CTXSW_LOG_ENABLE_INTR_APC:
51                 bIntrFallback = NV_TRUE;
52                 // Intentional fall-through
53             case NV_REG_STR_RM_CTXSW_LOG_ENABLE_INTR:
54                 bIntr = NV_TRUE;
55                 // Intentional fall-through
56             case NV_REG_STR_RM_CTXSW_LOG_ENABLE:
57                 bLog = NV_TRUE;
58                 break;
59             default:
60                 break;
61         }
62 
63         kgraphicsSetBottomHalfCtxswLoggingEnabled(pGpu, pKernelGraphics, bIntrFallback);
64         kgraphicsSetIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics, bIntr);
65         kgraphicsSetCtxswLoggingSupported(pGpu, pKernelGraphics, bLog);
66     }
67 
68     fecsSetRecordsPerIntr(pGpu, pKernelGraphics, NV_REG_STR_RM_CTXSW_LOG_RECORDS_PER_INTR_DEFAULT);
69     if (osReadRegistryDword(pGpu, NV_REG_STR_RM_CTXSW_LOG_RECORDS_PER_INTR, &data) == NV_OK)
70     {
71         fecsSetRecordsPerIntr(pGpu, pKernelGraphics, data);
72     }
73 }
74 
75 /*!
76  * @brief Allocate common local/global buffers that are required by the graphics context for GfxP Pool
77  *
78  * @param[in] pGpu
79  * @param[in] pKernelGraphics
80  * @param[in] gfid                   host or guest gfid
81  * @param[in] pKernelGraphicsContext graphics context - if valid allocate local
82  */
83 NV_STATUS
84 kgraphicsAllocGrGlobalCtxBuffers_GP100
85 (
86     OBJGPU *pGpu,
87     KernelGraphics *pKernelGraphics,
88     NvU32 gfid,
89     KernelGraphicsContext *pKernelGraphicsContext
90 )
91 {
92     extern NV_STATUS kgraphicsAllocGrGlobalCtxBuffers_GM200(OBJGPU *pGpu, KernelGraphics *pKernelGraphics, NvU32 gfid, KernelGraphicsContext *pKernelGraphicsContext);
93     GR_GLOBALCTX_BUFFERS         *pCtxBuffers;
94     NvU64                         allocFlags = MEMDESC_FLAGS_NONE;
95     NV_STATUS                     status;
96     CTX_BUF_POOL_INFO            *pCtxBufPool;
97     const KGRAPHICS_STATIC_INFO  *pKernelGraphicsStaticInfo;
98 
99     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
100 
101     pCtxBufPool = NULL;
102     if (pKernelGraphicsContext != NULL)
103     {
104         KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
105         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
106             kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
107 
108         pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer;
109 
110         //
111         // if we already have local buffers allocated, return as we may get
112         // called multiple times per-channel
113         //
114         if (pCtxBuffers->bAllocated)
115              return NV_OK;
116 
117         // check for allocating local buffers in VPR memory (don't want for global memory)
118         if (
119             pKernelGraphicsContextUnicast->bVprChannel)
120             allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
121 
122         // If allocated per channel, ensure allocations goes into Suballocator if available
123         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
124     }
125     else
126     {
127         pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid];
128         NV_ASSERT_OK_OR_RETURN(ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_GR_GLOBAL,
129             RM_ENGINE_TYPE_GR(pKernelGraphics->instance), &pCtxBufPool));
130     }
131 
132     // Don't use context buffer pool for VF allocations managed by host RM.
133     if (ctxBufPoolIsSupported(pGpu) && (pCtxBufPool != NULL))
134     {
135         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
136     }
137 
138     pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
139     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
140     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
141 
142     status = kgraphicsAllocGrGlobalCtxBuffers_GM200(pGpu, pKernelGraphics, gfid, pKernelGraphicsContext);
143 
144     return status;
145 }
146 
147 NV_STATUS
148 kgraphicsAllocGlobalCtxBuffers_GP100
149 (
150     OBJGPU *pGpu,
151     KernelGraphics *pKernelGraphics,
152     NvU32 gfid
153 )
154 {
155     CTX_BUF_POOL_INFO *pCtxBufPool = NULL;
156     NvU64 allocFlags = 0;
157     NvU32 fecsBufferSize = 0;
158     NvU32 fecsBufferAlign = 0x0;
159     GR_GLOBALCTX_BUFFERS *pCtxBuffers;
160     GR_BUFFER_ATTR *pCtxAttr;
161 
162     // SKIP FECS buffer allocation for Virtual context
163     if (IS_GFID_VF(gfid))
164     {
165         return NV_OK;
166     }
167 
168     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
169 
170     pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid];
171     pCtxAttr = pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr;
172     NV_ASSERT_OK_OR_RETURN(
173         ctxBufPoolGetGlobalPool(pGpu,
174                                 CTX_BUF_ID_GR_GLOBAL,
175                                 RM_ENGINE_TYPE_GR(pKernelGraphics->instance),
176                                 &pCtxBufPool));
177 
178     if (pCtxBufPool != NULL)
179     {
180         allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
181     }
182 
183     // TODO: Need this check for vGPU configs without SRIOV support?
184     if (!IS_VIRTUAL(pGpu) || IS_VIRTUAL_WITH_SRIOV(pGpu))
185     {
186         NvU32 engineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT;
187         const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
188 
189         NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
190         fecsBufferSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].size;
191         fecsBufferAlign = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[engineId].alignment;
192     }
193 
194     // Allocate the FECS buffer if the right regkey (RmCtxswLog) is enabled
195     if ((fecsBufferSize > 0) &&
196         kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics))
197     {
198         NvBool bIsFbBroken;
199         MEMORY_DESCRIPTOR **ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT];
200 
201         bIsFbBroken = pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) ||
202                       pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_ALL_INST_IN_SYSMEM);
203 
204         if (bIsFbBroken)
205         {
206             pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList = ADDRLIST_SYSMEM_ONLY;
207             pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr = NV_MEMORY_UNCACHED;
208         }
209 
210         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
211             memdescCreate(ppMemDesc, pGpu,
212                           fecsBufferSize,   // size
213                           fecsBufferAlign,  // alignment
214                           NV_TRUE,          // physically contiguous
215                           ADDR_UNKNOWN,
216                           pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].cpuAttr,
217                           allocFlags | MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS | MEMDESC_FLAGS_GPU_PRIVILEGED));
218 
219         if ((*ppMemDesc)->_addressSpace == ADDR_FBMEM)
220             memdescSetGpuCacheAttrib(*ppMemDesc, NV_MEMORY_CACHED);
221 
222         if ((allocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
223         {
224             MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
225 
226             memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
227             NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
228         }
229 
230         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
231             memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_FECS_EVENT].pAllocList));
232     }
233 
234     return NV_OK;
235 }
236 
237 /**
238  * @brief Services the GRn_FECS_LOG interrupts.
239  *
240  * @returns Zero, because KernelGraphics opts out of stuck interrupt detection.
241  */
242 NvU32
243 kgraphicsServiceInterrupt_GP100
244 (
245     OBJGPU *pGpu,
246     KernelGraphics *pKernelGraphics,
247     IntrServiceServiceInterruptArguments *pParams
248 )
249 {
250     NvU32 grIdx = pKernelGraphics->instance;
251 
252     NV_ASSERT_OR_RETURN(pParams != NULL, 0);
253     NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn_FECS_LOG(grIdx), 0);
254 
255     // if MIG is disabled and an access has been made to any GR engine ID > 0, fail
256     if (!IS_MIG_IN_USE(pGpu) && (grIdx != 0))
257     {
258         NV_ASSERT_FAILED("GR[1-7]_FECS_LOG is not supported if MIG is disabled!");
259         return 0;
260     }
261 
262     if ((pGpu->fecsCtxswLogConsumerCount > 0) &&
263         (kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics)))
264     {
265         if (fecsClearIntrPendingIfPending(pGpu, pKernelGraphics))
266         {
267             nvEventBufferFecsCallback(pGpu, (void*)pKernelGraphics);
268         }
269     }
270     return 0;
271 }
272 
273 /**
274  * @brief Clears the stall interrupt leaf vector and return whether to call ServiceStall.
275  * @details Normally there's no need to override this function; however,
276  *          the FECS_LOG engine idxs do not have real interrupt vectors to clear.
277  *          This implementation just tells INTR to continue with servicing.
278  *
279  * @returns NV_TRUE indicating the interrupt should be serviced.
280  */
281 NvBool
282 kgraphicsClearInterrupt_GP100
283 (
284     OBJGPU *pGpu,
285     KernelGraphics *pKernelGraphics,
286     IntrServiceClearInterruptArguments *pParams
287 )
288 {
289     NvU32 grIdx = pKernelGraphics->instance;
290 
291     NV_ASSERT_OR_RETURN(pParams != NULL, NV_FALSE);
292     NV_ASSERT_OR_RETURN(pParams->engineIdx == MC_ENGINE_IDX_GRn_FECS_LOG(grIdx), 0);
293 
294     return NV_TRUE;
295 }
296 
297