1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #define NVOC_KERNEL_GRAPHICS_CONTEXT_H_PRIVATE_ACCESS_ALLOWED
25
26 #include "kernel/gpu/gr/kernel_graphics_context.h"
27 #include "kernel/gpu/gr/kernel_graphics_manager.h"
28 #include "kernel/gpu/gr/kernel_graphics.h"
29 #include "kernel/gpu/fifo/kernel_channel.h"
30 #include "kernel/gpu/fifo/kernel_channel_group.h"
31 #include "kernel/gpu/fifo/kernel_channel_group_api.h"
32 #include "kernel/mem_mgr/gpu_vaspace.h"
33 #include "kernel/gpu/mem_mgr/mem_mgr.h"
34 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
35 #include "kernel/core/locks.h"
36 #include "kernel/gpu/nvlink/kernel_nvlink.h"
37 #include "vgpu/rpc.h"
38 #include "gpu/device/device.h"
39 #include "kernel/gpu/subdevice/subdevice.h"
40 #include "kernel/virtualization/hypervisor/hypervisor.h"
41 #include "gpu/mem_mgr/virt_mem_allocator.h"
42 #include "gpu/mmu/kern_gmmu.h"
43 #include "platform/sli/sli.h"
44 #include "rmapi/client.h"
45
46 /*!
47 * @brief Retrieve the context object from a KernelChannel.
48 *
49 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing
50 */
51 NV_STATUS
kgrctxFromKernelChannel_IMPL(KernelChannel * pKernelChannel,KernelGraphicsContext ** ppKernelGraphicsContext)52 kgrctxFromKernelChannel_IMPL
53 (
54 KernelChannel *pKernelChannel,
55 KernelGraphicsContext **ppKernelGraphicsContext
56 )
57 {
58 RsResourceRef *pResourceRef;
59
60 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
61
62 if (pKernelChannel->hKernelGraphicsContext != NV01_NULL_OBJECT)
63 {
64 NV_ASSERT_OK_OR_RETURN(
65 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel),
66 pKernelChannel->hKernelGraphicsContext,
67 classId(KernelGraphicsContext),
68 &pResourceRef));
69 }
70 else if ((pKernelChannel->pKernelChannelGroupApi != NULL) &&
71 (pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext != NV01_NULL_OBJECT))
72 {
73 NV_ASSERT_OK_OR_RETURN(
74 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannel),
75 pKernelChannel->pKernelChannelGroupApi->hKernelGraphicsContext,
76 classId(KernelGraphicsContext),
77 &pResourceRef));
78 }
79 else
80 {
81 return NV_ERR_OBJECT_NOT_FOUND;
82 }
83
84 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext);
85 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE);
86
87 return NV_OK;
88 }
89
90 /*!
91 * @brief Retrieve the context object from a ChannelGroup.
92 *
93 * @return NV_ERR_OBJECT_NOT_FOUND if context object is missing
94 */
95 NV_STATUS
kgrctxFromKernelChannelGroupApi_IMPL(KernelChannelGroupApi * pKernelChannelGroupApi,KernelGraphicsContext ** ppKernelGraphicsContext)96 kgrctxFromKernelChannelGroupApi_IMPL
97 (
98 KernelChannelGroupApi *pKernelChannelGroupApi,
99 KernelGraphicsContext **ppKernelGraphicsContext
100 )
101 {
102 RsResourceRef *pResourceRef;
103
104 NV_ASSERT_OR_RETURN(pKernelChannelGroupApi != NULL, NV_ERR_INVALID_ARGUMENT);
105
106 if (pKernelChannelGroupApi->hKernelGraphicsContext == NV01_NULL_OBJECT)
107 return NV_ERR_OBJECT_NOT_FOUND;
108
109 NV_ASSERT_OK_OR_RETURN(
110 clientGetResourceRefByType(RES_GET_CLIENT(pKernelChannelGroupApi),
111 pKernelChannelGroupApi->hKernelGraphicsContext,
112 classId(KernelGraphicsContext),
113 &pResourceRef));
114
115 *ppKernelGraphicsContext = dynamicCast(pResourceRef->pResource, KernelGraphicsContext);
116 NV_ASSERT_OR_RETURN(*ppKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE);
117
118 return NV_OK;
119 }
120
121 /**
122 * @brief Handle NV0090 ctrl call forwarding. The current control call is
123 * dispatched to the KernelGraphicsContext object provided.
124 */
kgrctxCtrlHandle(CALL_CONTEXT * pCallContext,NvHandle hKernelGraphicsContext)125 NV_STATUS kgrctxCtrlHandle
126 (
127 CALL_CONTEXT *pCallContext,
128 NvHandle hKernelGraphicsContext
129 )
130 {
131 RsResourceRef *pResourceRef;
132
133 NV_ASSERT_OK_OR_RETURN(
134 clientGetResourceRefByType(pCallContext->pClient,
135 hKernelGraphicsContext,
136 classId(KernelGraphicsContext),
137 &pResourceRef));
138
139 return resControl(pResourceRef->pResource, pCallContext, pCallContext->pControlParams);
140 }
141
142 /**
143 * @brief Translate global ctx buffer enum to external NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID
144 */
145 NV_STATUS
kgrctxGetGlobalContextBufferExternalId_IMPL(GR_GLOBALCTX_BUFFER id,NvU32 * pExternalId)146 kgrctxGetGlobalContextBufferExternalId_IMPL
147 (
148 GR_GLOBALCTX_BUFFER id,
149 NvU32 *pExternalId
150 )
151 {
152 NV_ASSERT_OR_RETURN(pExternalId != NULL, NV_ERR_INVALID_ARGUMENT);
153 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, id), NV_ERR_INVALID_ARGUMENT);
154
155 switch (id)
156 {
157 case GR_GLOBALCTX_BUFFER_BUNDLE_CB:
158 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB;
159 break;
160 case GR_GLOBALCTX_BUFFER_PAGEPOOL:
161 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL;
162 break;
163 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB:
164 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB;
165 break;
166 case GR_GLOBALCTX_BUFFER_RTV_CB:
167 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL;
168 break;
169 case GR_GLOBALCTX_BUFFER_GFXP_POOL:
170 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL;
171 break;
172 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK:
173 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK;
174 break;
175 case GR_GLOBALCTX_BUFFER_FECS_EVENT:
176 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT;
177 break;
178 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP:
179 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP;
180 break;
181 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP:
182 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP;
183 break;
184 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP:
185 // TODO this is not valid, this is not a context buffer
186 *pExternalId = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP;
187 break;
188 //
189 // No default case: Compiler will enforce that this switch is updated if
190 // new global ctx buffers are added
191 //
192 }
193
194 return NV_OK;
195 }
196
197 /**
198 * @brief Translate NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID to global ctx buffer enum
199 */
200 NV_STATUS
kgrctxGetGlobalContextBufferInternalId_IMPL(NvU32 externalId,GR_GLOBALCTX_BUFFER * pInternalId)201 kgrctxGetGlobalContextBufferInternalId_IMPL
202 (
203 NvU32 externalId,
204 GR_GLOBALCTX_BUFFER *pInternalId
205 )
206 {
207 NV_ASSERT_OR_RETURN(pInternalId != NULL, NV_ERR_INVALID_ARGUMENT);
208
209 switch (externalId)
210 {
211 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN:
212 // fall through
213 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM:
214 // fall through
215 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH:
216 return NV_ERR_INVALID_ARGUMENT;
217 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB:
218 *pInternalId = GR_GLOBALCTX_BUFFER_BUNDLE_CB;
219 break;
220 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL:
221 *pInternalId = GR_GLOBALCTX_BUFFER_PAGEPOOL;
222 break;
223 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB:
224 *pInternalId = GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB;
225 break;
226 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL:
227 *pInternalId = GR_GLOBALCTX_BUFFER_RTV_CB;
228 break;
229 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL:
230 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_POOL;
231 break;
232 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK:
233 *pInternalId = GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK;
234 break;
235 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT:
236 *pInternalId = GR_GLOBALCTX_BUFFER_FECS_EVENT;
237 break;
238 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP:
239 *pInternalId = GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP;
240 break;
241 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP:
242 *pInternalId = GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP;
243 break;
244 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP:
245 *pInternalId = GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP;
246 break;
247 default:
248 return NV_ERR_INVALID_ARGUMENT;
249 }
250
251 return NV_OK;
252 }
253
254 /*! Translate GR_CTX_BUFFER to NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */
255 NV_STATUS
kgrctxCtxBufferToFifoEngineId_IMPL(GR_CTX_BUFFER buffer,NvU32 * pFifoEngineId)256 kgrctxCtxBufferToFifoEngineId_IMPL
257 (
258 GR_CTX_BUFFER buffer,
259 NvU32 *pFifoEngineId
260 )
261 {
262 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_CTX_BUFFER, buffer), NV_ERR_INVALID_ARGUMENT);
263
264 switch (buffer)
265 {
266 case GR_CTX_BUFFER_MAIN:
267 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS;
268 break;
269 case GR_CTX_BUFFER_ZCULL:
270 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ZCULL;
271 break;
272 case GR_CTX_BUFFER_PM:
273 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM;
274 break;
275 case GR_CTX_BUFFER_PREEMPT:
276 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PREEMPT;
277 break;
278 case GR_CTX_BUFFER_SPILL:
279 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SPILL;
280 break;
281 case GR_CTX_BUFFER_BETA_CB:
282 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BETACB;
283 break;
284 case GR_CTX_BUFFER_PAGEPOOL:
285 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL;
286 break;
287 case GR_CTX_BUFFER_RTV_CB:
288 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV;
289 break;
290 case GR_CTX_BUFFER_PATCH:
291 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH;
292 break;
293 case GR_CTX_BUFFER_SETUP:
294 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_SETUP;
295 break;
296 // No default case - Compiler enforces switch update if enum is changed
297 }
298
299 return NV_OK;
300 }
301
302 /**
303 * @brief Translate global ctx buffer enum to NV0080 FIFO engine context properties index
304 */
305 NV_STATUS
kgrctxGlobalCtxBufferToFifoEngineId_IMPL(GR_GLOBALCTX_BUFFER buffId,NvU32 * pFifoEngineId)306 kgrctxGlobalCtxBufferToFifoEngineId_IMPL
307 (
308 GR_GLOBALCTX_BUFFER buffId,
309 NvU32 *pFifoEngineId
310 )
311 {
312 NV_ASSERT_OR_RETURN(pFifoEngineId != NULL, NV_ERR_INVALID_ARGUMENT);
313 NV_ASSERT_OR_RETURN(NV_ENUM_IS(GR_GLOBALCTX_BUFFER, buffId), NV_ERR_INVALID_ARGUMENT);
314
315 switch (buffId)
316 {
317 case GR_GLOBALCTX_BUFFER_BUNDLE_CB:
318 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB;
319 break;
320 case GR_GLOBALCTX_BUFFER_PAGEPOOL:
321 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL;
322 break;
323 case GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB:
324 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB;
325 break;
326 case GR_GLOBALCTX_BUFFER_RTV_CB:
327 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_RTV_CB_GLOBAL;
328 break;
329 case GR_GLOBALCTX_BUFFER_GFXP_POOL:
330 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_POOL;
331 break;
332 case GR_GLOBALCTX_BUFFER_GFXP_CTRL_BLK:
333 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_GFXP_CTRL_BLK;
334 break;
335 case GR_GLOBALCTX_BUFFER_FECS_EVENT:
336 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_FECS_EVENT;
337 break;
338 case GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP:
339 // fall-through
340 case GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP:
341 // fall-through
342 case GR_GLOBAL_BUFFER_GLOBAL_PRIV_ACCESS_MAP:
343 *pFifoEngineId = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP;
344 break;
345 // No default case - compiler enforces update if enum changes
346 }
347
348 return NV_OK;
349 }
350
351 NV_STATUS
kgrctxGetGidInfoInPlace_IMPL(OBJGPU * pGpu,NvU8 * pUuidBuffer,NvU32 uuidBufferSize,NvU32 flags)352 kgrctxGetGidInfoInPlace_IMPL
353 (
354 OBJGPU *pGpu,
355 NvU8 *pUuidBuffer,
356 NvU32 uuidBufferSize,
357 NvU32 flags
358 )
359 {
360 NvU8 *pUuid;
361 NvU32 uuidLength;
362 NV_STATUS status;
363
364 // on success, allocates memory for uuid
365 status = gpuGetGidInfo(pGpu, &pUuid, &uuidLength, flags);
366 if (status != NV_OK)
367 {
368 return status;
369 }
370
371 if (uuidLength == uuidBufferSize)
372 {
373 portMemCopy(pUuidBuffer, uuidBufferSize, pUuid, uuidLength);
374 }
375
376 portMemFree(pUuid);
377
378 if (uuidLength != uuidBufferSize)
379 {
380 return NV_ERR_INVALID_ARGUMENT;
381 }
382
383 return NV_OK;
384 }
385
386 NV_STATUS
kgrctxFillCtxBufferInfo_IMPL(MEMORY_DESCRIPTOR * pMemDesc,NvU32 externalId,NvBool bBufferGlobal,NV2080_CTRL_GR_CTX_BUFFER_INFO * pCtxBufferInfo)387 kgrctxFillCtxBufferInfo_IMPL
388 (
389 MEMORY_DESCRIPTOR *pMemDesc,
390 NvU32 externalId,
391 NvBool bBufferGlobal,
392 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo
393 )
394 {
395 NvU64 pageSize;
396 NV_STATUS status;
397
398 MEMORY_DESCRIPTOR *pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
399
400 pCtxBufferInfo->bufferHandle = NV_PTR_TO_NvP64(pMemDesc);
401 pCtxBufferInfo->bufferType = externalId;
402 pCtxBufferInfo->bIsContigous = memdescGetContiguity(pMemDesc, AT_GPU);
403 pCtxBufferInfo->aperture = memdescGetAddressSpace(pMemDesc);
404 pCtxBufferInfo->pageCount = pMemDesc->PageCount;
405 pCtxBufferInfo->kind = memdescGetPteKindForGpu(pMemDesc, pMemDesc->pGpu);
406
407 {
408 NvU64 physAddr;
409 GMMU_APERTURE aperture = kgmmuGetExternalAllocAperture(pCtxBufferInfo->aperture);
410
411 memdescGetPhysAddrsForGpu(pMemDesc, pMemDesc->pGpu,
412 AT_GPU, 0, 0, 1,
413 &physAddr);
414
415 pCtxBufferInfo->physAddr =
416 kgmmuEncodePhysAddr(GPU_GET_KERNEL_GMMU(pMemDesc->pGpu), aperture, physAddr,
417 NVLINK_INVALID_FABRIC_ADDR);
418 }
419
420 pageSize = memdescGetPageSize(pMemDesc, AT_GPU);
421 if (pageSize == 0)
422 {
423 status = memmgrSetMemDescPageSize_HAL(pMemDesc->pGpu,
424 GPU_GET_MEMORY_MANAGER(pMemDesc->pGpu),
425 pMemDesc,
426 AT_GPU,
427 RM_ATTR_PAGE_SIZE_DEFAULT);
428 if (status != NV_OK)
429 return status;
430
431 pageSize = memdescGetPageSize(pMemDesc, AT_GPU);
432 NV_ASSERT(pageSize != 0);
433 }
434
435 //
436 // Alignment is used to adjust the mapping VA. Hence, we need to make sure
437 // that at least pageSize to make mapping calculation work correctly.
438 //
439 pCtxBufferInfo->alignment = (pMemDesc->Alignment != 0) ?
440 NV_ALIGN_UP(pMemDesc->Alignment, pageSize) : pageSize;
441
442 pCtxBufferInfo->size = pMemDesc->ActualSize;
443 pCtxBufferInfo->pageSize = pageSize;
444
445 pCtxBufferInfo->bGlobalBuffer = bBufferGlobal;
446 pCtxBufferInfo->bLocalBuffer = !bBufferGlobal;
447 pCtxBufferInfo->bDeviceDescendant = pRootMemDesc->pGpu != NULL;
448
449 if (pCtxBufferInfo->bDeviceDescendant)
450 {
451 status = kgrctxGetGidInfoInPlace(pMemDesc->pGpu, pCtxBufferInfo->uuid, sizeof(pCtxBufferInfo->uuid),
452 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _TYPE, _SHA1) |
453 DRF_DEF(2080_GPU_CMD, _GPU_GET_GID_FLAGS, _FORMAT, _BINARY));
454 if (status != NV_OK)
455 return status;
456 }
457
458 return NV_OK;
459 }
460
461 /*!
462 * @brief Construct dupable kernel graphics context.
463 */
464 NV_STATUS
kgrctxConstruct_IMPL(KernelGraphicsContext * pKernelGraphicsContext,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)465 kgrctxConstruct_IMPL
466 (
467 KernelGraphicsContext *pKernelGraphicsContext,
468 CALL_CONTEXT *pCallContext,
469 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
470 )
471 {
472 NV_STATUS status = NV_OK;
473 RsShared *pShared;
474
475 if (RS_IS_COPY_CTOR(pParams))
476 return kgrctxCopyConstruct_IMPL(pKernelGraphicsContext, pCallContext, pParams);
477
478 NV_ASSERT_OK_OR_GOTO(status,
479 serverAllocShare(&g_resServ, classInfo(KernelGraphicsContextShared), &pShared),
480 cleanup);
481
482 pKernelGraphicsContext->pShared = dynamicCast(pShared, KernelGraphicsContextShared);
483 NV_ASSERT_OK_OR_GOTO(status,
484 shrkgrctxInit(GPU_RES_GET_GPU(pKernelGraphicsContext),
485 pKernelGraphicsContext->pShared,
486 pKernelGraphicsContext),
487 cleanup);
488
489 cleanup:
490 if (status != NV_OK)
491 {
492 if (pKernelGraphicsContext->pShared != NULL)
493 serverFreeShare(&g_resServ, pShared);
494
495 }
496
497 return status;
498 }
499
500 /*!
501 * @brief Copy Construct dupable kernel graphics context.
502 */
503 NV_STATUS
kgrctxCopyConstruct_IMPL(KernelGraphicsContext * pKernelGraphicsContextDst,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)504 kgrctxCopyConstruct_IMPL
505 (
506 KernelGraphicsContext *pKernelGraphicsContextDst,
507 CALL_CONTEXT *pCallContext,
508 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
509 )
510 {
511 RsResourceRef *pSrcRef = pParams->pSrcRef;
512 KernelGraphicsContext *pKernelGraphicsContextSrc =
513 dynamicCast(pSrcRef->pResource, KernelGraphicsContext);
514
515 pKernelGraphicsContextDst->pShared = pKernelGraphicsContextSrc->pShared;
516 serverRefShare(&g_resServ, staticCast(pKernelGraphicsContextDst->pShared, RsShared));
517
518 return NV_OK;
519 }
520
521 NvHandle
kgrctxGetInternalObjectHandle_IMPL(KernelGraphicsContext * pKernelGraphicsContext)522 kgrctxGetInternalObjectHandle_IMPL(KernelGraphicsContext *pKernelGraphicsContext)
523 {
524 return NV01_NULL_OBJECT;
525 }
526
527 /*!
528 * @brief Destruct dupable kernel graphics context.
529 */
530 void
kgrctxDestruct_IMPL(KernelGraphicsContext * pKernelGraphicsContext)531 kgrctxDestruct_IMPL
532 (
533 KernelGraphicsContext *pKernelGraphicsContext
534 )
535 {
536 RsShared *pShared = staticCast(pKernelGraphicsContext->pShared, RsShared);
537 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext);
538
539 if (serverGetShareRefCount(&g_resServ, pShared) == 1)
540 {
541 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContext->pShared, pKernelGraphicsContext);
542 }
543
544 serverFreeShare(&g_resServ, pShared);
545 }
546
547 /*!
548 * @brief Retrieve unicast context state
549 */
kgrctxGetUnicast_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphicsContextUnicast ** ppKernelGraphicsContextUnicast)550 NV_STATUS kgrctxGetUnicast_IMPL
551 (
552 OBJGPU *pGpu,
553 KernelGraphicsContext *pKernelGraphicsContext,
554 KernelGraphicsContextUnicast **ppKernelGraphicsContextUnicast
555 )
556 {
557 *ppKernelGraphicsContextUnicast = NULL;
558 NV_ASSERT_OR_RETURN(pKernelGraphicsContext->pShared != NULL, NV_ERR_INVALID_STATE);
559 *ppKernelGraphicsContextUnicast = &pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast;
560 return NV_OK;
561 }
562
563 /*!
564 * @brief Query the details of MMU faults caused by this context
565 */
566 NV_STATUS
kgrctxLookupMmuFaultInfo_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS * pParams)567 kgrctxLookupMmuFaultInfo_IMPL
568 (
569 OBJGPU *pGpu,
570 KernelGraphicsContext *pKernelGraphicsContext,
571 NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS *pParams
572 )
573 {
574 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
575 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList);
576 NvU32 i;
577
578 NV_ASSERT_OK_OR_RETURN(
579 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
580
581 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.head < size, NV_ERR_INVALID_STATE);
582 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->mmuFault.tail < size, NV_ERR_INVALID_STATE);
583
584 pParams->count = 0;
585 for (i = pKernelGraphicsContextUnicast->mmuFault.tail;
586 i != pKernelGraphicsContextUnicast->mmuFault.head;
587 i = (i + 1) % size)
588 {
589 pParams->mmuFaultInfoList[pParams->count] = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[i];
590 pParams->count++;
591 }
592
593 return NV_OK;
594 }
595
596 /*!
597 * @brief Query the details of MMU faults caused by this context
598 */
599 NV_STATUS
kgrctxLookupMmuFault_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NV83DE_MMU_FAULT_INFO * pMmuFaultInfo)600 kgrctxLookupMmuFault_IMPL
601 (
602 OBJGPU *pGpu,
603 KernelGraphicsContext *pKernelGraphicsContext,
604 NV83DE_MMU_FAULT_INFO *pMmuFaultInfo
605 )
606 {
607 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
608
609 NV_ASSERT_OK_OR_RETURN(
610 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
611
612 *pMmuFaultInfo = pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo;
613 return NV_OK;
614 }
615
616 /*!
617 * @brief clear the details of MMU faults caused by this context
618 */
619 NV_STATUS
kgrctxClearMmuFault_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)620 kgrctxClearMmuFault_IMPL
621 (
622 OBJGPU *pGpu,
623 KernelGraphicsContext *pKernelGraphicsContext
624 )
625 {
626 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
627
628 NV_ASSERT_OK_OR_RETURN(
629 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
630
631 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_FALSE;
632 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = 0;
633 return NV_OK;
634 }
635
636 /*!
637 * @brief Record the details of an MMU fault caused by this context
638 */
639 void
kgrctxRecordMmuFault_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NvU32 mmuFaultInfo,NvU64 mmuFaultAddress,NvU32 mmuFaultType,NvU32 mmuFaultAccessType)640 kgrctxRecordMmuFault_IMPL
641 (
642 OBJGPU *pGpu,
643 KernelGraphicsContext *pKernelGraphicsContext,
644 NvU32 mmuFaultInfo,
645 NvU64 mmuFaultAddress,
646 NvU32 mmuFaultType,
647 NvU32 mmuFaultAccessType
648 )
649 {
650 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
651 NV_STATUS status;
652
653 NV_ASSERT_OK_OR_ELSE(status,
654 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
655 return;);
656
657 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.valid = NV_TRUE;
658 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfo.faultInfo = mmuFaultInfo;
659
660 {
661 const NvU32 size = NV_ARRAY_ELEMENTS(pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList);
662
663 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.head < size);
664 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->mmuFault.tail < size);
665
666 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultAddress = mmuFaultAddress;
667 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].faultType = mmuFaultType;
668 pKernelGraphicsContextUnicast->mmuFault.mmuFaultInfoList[pKernelGraphicsContextUnicast->mmuFault.head].accessType = mmuFaultAccessType;
669 pKernelGraphicsContextUnicast->mmuFault.head = (pKernelGraphicsContextUnicast->mmuFault.head + 1) % size;
670 if (pKernelGraphicsContextUnicast->mmuFault.head == pKernelGraphicsContextUnicast->mmuFault.tail)
671 pKernelGraphicsContextUnicast->mmuFault.tail = (pKernelGraphicsContextUnicast->mmuFault.tail + 1) % size;
672 }
673 }
674
675 /*!
676 * @brief getter for active debugger interator
677 */
678 KernelSMDebuggerSessionListIter
kgrctxGetDebuggerSessionIter_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)679 kgrctxGetDebuggerSessionIter_IMPL
680 (
681 OBJGPU *pGpu,
682 KernelGraphicsContext *pKernelGraphicsContext
683 )
684 {
685 return listIterAll(&pKernelGraphicsContext->pShared->activeDebuggers);
686 }
687
688 /*! add active debugger session */
689 NvBool
kgrctxRegisterKernelSMDebuggerSession_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelSMDebuggerSession * pKernelSMDebuggerSession)690 kgrctxRegisterKernelSMDebuggerSession_IMPL
691 (
692 OBJGPU *pGpu,
693 KernelGraphicsContext *pKernelGraphicsContext,
694 KernelSMDebuggerSession *pKernelSMDebuggerSession
695 )
696 {
697 return listAppendValue(&pKernelGraphicsContext->pShared->activeDebuggers, &pKernelSMDebuggerSession) != NULL;
698 }
699
700 /*! remove active debugger session */
701 void
kgrctxDeregisterKernelSMDebuggerSession_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelSMDebuggerSession * pKernelSMDebuggerSession)702 kgrctxDeregisterKernelSMDebuggerSession_IMPL
703 (
704 OBJGPU *pGpu,
705 KernelGraphicsContext *pKernelGraphicsContext,
706 KernelSMDebuggerSession *pKernelSMDebuggerSession
707 )
708 {
709 listRemoveFirstByValue(&pKernelGraphicsContext->pShared->activeDebuggers, &pKernelSMDebuggerSession);
710 }
711
712 /*!
713 * @brief Determine whether channels in this context are associated with GR engine
714 *
715 * @returns NV_TRUE if passed channel is allocated on GR, and is on GR runlist
716 */
717 NvBool
kgrctxIsValid_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel)718 kgrctxIsValid_IMPL
719 (
720 OBJGPU *pGpu,
721 KernelGraphicsContext *pKernelGraphicsContext,
722 KernelChannel *pKernelChannel
723 )
724 {
725 NvU32 runlistId;
726 RM_ENGINE_TYPE engineType;
727
728 // TODO remove pKernelChannel from params
729
730 if (RM_ENGINE_TYPE_IS_VALID(kchannelGetEngineType(pKernelChannel)) &&
731 !RM_ENGINE_TYPE_IS_GR(kchannelGetEngineType(pKernelChannel)))
732 {
733 return NV_FALSE;
734 }
735
736 NV_CHECK_OR_RETURN(LEVEL_INFO, kchannelIsRunlistSet(pGpu, pKernelChannel), NV_FALSE);
737
738 runlistId = kchannelGetRunlistId(pKernelChannel);
739 NV_ASSERT_OK(
740 kfifoEngineInfoXlate_HAL(pGpu, GPU_GET_KERNEL_FIFO(pGpu),
741 ENGINE_INFO_TYPE_RUNLIST, runlistId,
742 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *) &engineType));
743
744 NV_CHECK_OR_RETURN(LEVEL_INFO, RM_ENGINE_TYPE_IS_GR(engineType), NV_FALSE);
745
746 return NV_TRUE;
747 }
748
749 /*!
750 * @brief returns if the main context buffer has been allocated
751 *
752 * @return NV_TRUE if allocated, NV_FALSE otherwise
753 */
754 NvBool
kgrctxIsMainContextAllocated_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)755 kgrctxIsMainContextAllocated_IMPL
756 (
757 OBJGPU *pGpu,
758 KernelGraphicsContext *pKernelGraphicsContext
759 )
760 {
761 MEMORY_DESCRIPTOR *pMemDesc;
762 return (kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pMemDesc) == NV_OK) &&
763 (pMemDesc != NULL);
764 }
765
766 /*! Retrieve the memdesc containing the main ctx buffer */
kgrctxGetMainContextBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,MEMORY_DESCRIPTOR ** ppMemDesc)767 NV_STATUS kgrctxGetMainContextBuffer_IMPL
768 (
769 OBJGPU *pGpu,
770 KernelGraphicsContext *pKernelGraphicsContext,
771 MEMORY_DESCRIPTOR **ppMemDesc
772 )
773 {
774 ENGINE_CTX_DESCRIPTOR *pEngCtxDesc;
775 NvU32 subDevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
776 RsResourceRef *pParentRef = RES_GET_REF(pKernelGraphicsContext)->pParentRef;
777 KernelChannelGroupApi *pKernelChannelGroupApi;
778 KernelChannelGroup *pKernelChannelGroup;
779
780 *ppMemDesc = NULL;
781 pKernelChannelGroupApi = dynamicCast(pParentRef->pResource, KernelChannelGroupApi);
782 pKernelChannelGroup = pKernelChannelGroupApi->pKernelChannelGroup;
783 pEngCtxDesc = pKernelChannelGroup->ppEngCtxDesc[subDevInst];
784
785 if (pEngCtxDesc != NULL)
786 *ppMemDesc = pEngCtxDesc->pMemDesc;
787
788 return NV_OK;
789 }
790
791 /*!
792 * @brief Retrieve information about the context buffers
793 */
794 NV_STATUS
kgrctxGetCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 gfid,NvU32 bufferCount,MEMORY_DESCRIPTOR ** ppBuffers,NvU32 * pCtxBufferType,NvU32 * pBufferCountOut,NvU32 * pFirstGlobalBuffer)795 kgrctxGetCtxBuffers_IMPL
796 (
797 OBJGPU *pGpu,
798 KernelGraphicsContext *pKernelGraphicsContext,
799 KernelGraphics *pKernelGraphics,
800 NvU32 gfid,
801 NvU32 bufferCount,
802 MEMORY_DESCRIPTOR **ppBuffers,
803 NvU32 *pCtxBufferType,
804 NvU32 *pBufferCountOut,
805 NvU32 *pFirstGlobalBuffer
806 )
807 {
808 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
809 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
810 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc;
811 NvU32 bufferCountOut = 0;
812 NvU32 i;
813
814 NV_ASSERT_OK_OR_RETURN(
815 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
816
817 // Get local context buffer memdesc.
818 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
819 kgrctxGetMainContextBuffer(pGpu, pKernelGraphicsContext, &pGrCtxBufferMemDesc));
820
821 NV_CHECK_OR_RETURN(LEVEL_ERROR,
822 pGrCtxBufferMemDesc != NULL,
823 NV_ERR_INVALID_OBJECT);
824
825 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT);
826 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN;
827 ppBuffers[bufferCountOut++] = pGrCtxBufferMemDesc;
828
829 // Get context patch buffer memdesc.
830 NV_CHECK_OR_RETURN(LEVEL_SILENT,
831 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL,
832 NV_ERR_INVALID_STATE);
833
834 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT);
835 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH;
836 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc;
837
838 // Add PM ctxsw buffer if it's allocated.
839 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
840 {
841 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT);
842 pCtxBufferType[bufferCountOut] = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM;
843 ppBuffers[bufferCountOut++] = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc;
844 }
845
846 if (pFirstGlobalBuffer != NULL)
847 {
848 *pFirstGlobalBuffer = bufferCountOut;
849 }
850
851 // Add global buffers.
852 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++)
853 {
854 MEMORY_DESCRIPTOR *pMemDesc = pGlobalCtxBuffers->memDesc[i];
855 if (pMemDesc != NULL)
856 {
857 NV_CHECK_OR_RETURN(LEVEL_INFO, bufferCountOut < bufferCount, NV_ERR_INVALID_ARGUMENT);
858 NV_ASSERT_OK(
859 kgrctxGetGlobalContextBufferExternalId(i, &pCtxBufferType[bufferCountOut]));
860 ppBuffers[bufferCountOut++] = pMemDesc;
861 }
862 }
863
864 *pBufferCountOut = bufferCountOut;
865
866 return NV_OK;
867 }
868
869 /*
870 * @brief Get maximum context buffer count including global and local buffers
871 *
872 * @param[in] pGpu
873 * @param[in] pKernelGraphicsContext
874 * @param[in] pKernelGraphics
875 * @param[out] pBufferCount Max context buffer count
876 */
877 NV_STATUS
kgrctxGetBufferCount_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 * pBufferCount)878 kgrctxGetBufferCount_IMPL
879 (
880 OBJGPU *pGpu,
881 KernelGraphicsContext *pKernelGraphicsContext,
882 KernelGraphics *pKernelGraphics,
883 NvU32 *pBufferCount
884 )
885 {
886 NvU32 gfid;
887 NvBool bCallingContextPlugin;
888 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers;
889 NvU32 i;
890
891 *pBufferCount = 0;
892
893 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
894 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin));
895
896 pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
897
898 if (bCallingContextPlugin)
899 {
900 gfid = GPU_GFID_PF;
901 }
902
903 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++)
904 {
905 if (pGlobalCtxBuffers->memDesc[i] != NULL)
906 (*pBufferCount)++;
907 }
908
909 // Increment by 3 to include local ctx buffer, patch context buffer and PM Ctxsw buffer
910 *pBufferCount += 3;
911
912 return NV_OK;
913 }
914
915 /*
916 * @brief Get context buffer info like size, alignment for global and
917 * local buffers
918 *
919 * @param[in] pGpu
920 * @param[in] pKernelGraphicsContext
921 * @param[in] pKernelGraphics
922 * @param[in] gfid
923 * @param[in] bufferMaxCount Amount of space provided in pCtxBufferInfo
924 * @param[out] bufferCount Number of buffers described
925 * @param[out] pCtxBufferInfo Structure to fill buffer information
926 *
927 * @return NV_OK successfully probed all buffers
928 * @return NV_ERR_INVALID_ARGUMENT not enough space provided for buffers queried
929 */
930 NV_STATUS
kgrctxGetCtxBufferInfo_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 gfid,NvU32 bufferMaxCount,NvU32 * pBufferCount,NV2080_CTRL_GR_CTX_BUFFER_INFO * pCtxBufferInfo)931 kgrctxGetCtxBufferInfo_IMPL
932 (
933 OBJGPU *pGpu,
934 KernelGraphicsContext *pKernelGraphicsContext,
935 KernelGraphics *pKernelGraphics,
936 NvU32 gfid,
937 NvU32 bufferMaxCount,
938 NvU32 *pBufferCount,
939 NV2080_CTRL_GR_CTX_BUFFER_INFO *pCtxBufferInfo
940 )
941 {
942 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT];
943 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT];
944 NvU32 memdescCount;
945 NvU32 firstGlobalBuffer;
946 NvU32 i;
947 NvU32 j;
948
949 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
950 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics,
951 gfid,
952 NV_ARRAY_ELEMENTS(pMemDescArray),
953 pMemDescArray,
954 bufferExternalId,
955 &memdescCount,
956 &firstGlobalBuffer));
957
958 if (bufferMaxCount < memdescCount)
959 return NV_ERR_INVALID_ARGUMENT;
960
961 for (i = 0; i < memdescCount; i++)
962 {
963 NvBool bGlobalBuffer = (i >= firstGlobalBuffer);
964
965 NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
966 kgrctxFillCtxBufferInfo(pMemDescArray[i],
967 bufferExternalId[i],
968 bGlobalBuffer,
969 &pCtxBufferInfo[i]));
970 }
971
972 //
973 // Sort the buffer info in descending order using alignment so that the
974 // VA range calculation can figure out the optimal VA range size.
975 //
976 for (i = 0; i < memdescCount; i++)
977 {
978 for (j = 0; j < memdescCount - 1; j++)
979 {
980 if (pCtxBufferInfo[j].alignment < pCtxBufferInfo[j + 1].alignment)
981 {
982 NV2080_CTRL_GR_CTX_BUFFER_INFO tmp;
983
984 portMemCopy(&tmp, sizeof(tmp), &pCtxBufferInfo[j], sizeof(tmp));
985 portMemCopy(&pCtxBufferInfo[j], sizeof(tmp), &pCtxBufferInfo[j + 1], sizeof(tmp));
986 portMemCopy(&pCtxBufferInfo[j + 1], sizeof(tmp), &tmp, sizeof(tmp));
987 }
988 }
989 }
990
991 *pBufferCount = memdescCount;
992
993 return NV_OK;
994 }
995
996 /*
997 * @brief Return physical addresses of context buffer starting from its 'firstPage' page.
998 * The function is intended to be called repeatedly while advancing
999 * 'firstPage' parameter until all pages of the buffer are queried.
1000 * This condition is indicated by *pNoMorePage == NV_TRUE.
1001 * If requested buffer is contiguous, address of first page is returned
1002 * always.
1003 *
1004 * @param[in] pGpu
1005 * @param[in] pKernelGraphicsContext
1006 * @param[in] pKernelGraphics
1007 * @param[in] gfid
1008 * @param[in] bufferType Requested ctx buffer type
1009 * @param[in] firstPage First page of the buffer to be queried
1010 * @param[out] pPhysAddrs Array to be filled with page addresses
1011 * @param[in] addrsSize Number of elements of pPhysAddrs
1012 * @param[out] pNumPages Number of page addresses returned
1013 * @param[out] pbNoMorePages End of buffer reached
1014 *
1015 * @return NV_OK successfully obtained requested addresses
1016 * @return NV_ERR_INVALID_ARGUMENT pMemDesc does not specify a context buffer
1017 */
1018 NV_STATUS
kgrctxGetCtxBufferPtes_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 gfid,NvU32 bufferType,NvU32 firstPage,NvU64 * pPhysAddrs,NvU32 addrsSize,NvU32 * pNumPages,NvBool * pbNoMorePages)1019 kgrctxGetCtxBufferPtes_IMPL
1020 (
1021 OBJGPU *pGpu,
1022 KernelGraphicsContext *pKernelGraphicsContext,
1023 KernelGraphics *pKernelGraphics,
1024 NvU32 gfid,
1025 NvU32 bufferType,
1026 NvU32 firstPage,
1027 NvU64 *pPhysAddrs,
1028 NvU32 addrsSize,
1029 NvU32 *pNumPages,
1030 NvBool *pbNoMorePages
1031 )
1032 {
1033 NvU64 bufferSize;
1034 NvU64 pageSize;
1035 NvU32 numPages;
1036 NvU32 bufferCount;
1037 NvU32 i;
1038 MEMORY_DESCRIPTOR *pMemDescArray[3 + GR_GLOBALCTX_BUFFER_COUNT];
1039 NvU32 bufferExternalId[3 + GR_GLOBALCTX_BUFFER_COUNT];
1040
1041 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1042 kgrctxGetCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics,
1043 gfid,
1044 NV_ARRAY_ELEMENTS(pMemDescArray),
1045 pMemDescArray,
1046 bufferExternalId,
1047 &bufferCount,
1048 NULL));
1049
1050 for (i = 0; i < bufferCount; i++)
1051 {
1052 if (bufferExternalId[i] == bufferType)
1053 break;
1054 }
1055
1056 if (addrsSize == 0)
1057 {
1058 return NV_ERR_INVALID_ARGUMENT;
1059 }
1060
1061 NV_CHECK_OR_RETURN(LEVEL_ERROR, i != bufferCount, NV_ERR_INVALID_ARGUMENT);
1062
1063 bufferSize = memdescGetSize(pMemDescArray[i]);
1064 pageSize = memdescGetPageSize(pMemDescArray[i], AT_GPU);
1065
1066 if (pageSize == 0)
1067 {
1068 return NV_ERR_INVALID_STATE;
1069 }
1070
1071 numPages = NV_ROUNDUP(bufferSize, pageSize) / pageSize;
1072
1073 if (firstPage >= numPages)
1074 {
1075 numPages = 0;
1076 *pbNoMorePages = NV_TRUE;
1077 goto done;
1078 }
1079
1080 if (memdescGetContiguity(pMemDescArray[i], AT_GPU))
1081 {
1082 firstPage = 0;
1083 numPages = 1;
1084 *pbNoMorePages = NV_TRUE;
1085 }
1086 else
1087 {
1088 numPages -= firstPage;
1089 *pbNoMorePages = (numPages <= addrsSize);
1090 numPages = NV_MIN(numPages, addrsSize);
1091 }
1092
1093 if (numPages > 0)
1094 {
1095 memdescGetPhysAddrs(pMemDescArray[i],
1096 AT_GPU,
1097 firstPage * pageSize,
1098 pageSize,
1099 numPages,
1100 pPhysAddrs);
1101 }
1102
1103 done:
1104 *pNumPages = numPages;
1105
1106 return NV_OK;
1107 }
1108
1109 /*!
1110 * This function does following things
1111 * 1. Allocate main GR context buffer
1112 * 2. Sets up GR context memory descriptor by calling kchannelSetEngineContextMemDesc
1113 */
1114 NV_STATUS
kgrctxAllocMainCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)1115 kgrctxAllocMainCtxBuffer_IMPL
1116 (
1117 OBJGPU *pGpu,
1118 KernelGraphicsContext *pKernelGraphicsContext,
1119 KernelGraphics *pKernelGraphics,
1120 KernelChannel *pKernelChannel
1121 )
1122 {
1123 MEMORY_DESCRIPTOR *pGrCtxBufferMemDesc = NULL;
1124 NvU32 ctxSize;
1125 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1126 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1127 NvU64 allocFlags = MEMDESC_FLAGS_GPU_PRIVILEGED;
1128 CTX_BUF_POOL_INFO *pCtxBufPool = NULL;
1129 NvBool bIsContiguous = kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics);
1130 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_MAIN);
1131
1132 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
1133
1134 NV_ASSERT_OK_OR_RETURN(
1135 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1136
1137 //
1138 // Allocate space for per-subcontext headers in the context buffer.
1139 // With subcontext, per-subcontext context header is programmed in the channel instance block.
1140 // Per-subcontext headers can be seperate from the context buffer.
1141 // For the initial phase, we allocate them at the end of the context buffers for easier tracking.
1142 // This will waste some memory (256 KB), if the number of subcontexts are sparse.
1143 // Will cleanup this up later to be on-demand.
1144 //
1145 // We also need to report the updated context size to KMD for virtual context.
1146 // Bug 1764102 tracks the VC support.
1147 //
1148 NV_ASSERT_OK_OR_RETURN(
1149 kgraphicsGetMainCtxBufferSize(pGpu, pKernelGraphics, NV_TRUE, &ctxSize));
1150
1151 if (ctxBufPoolIsSupported(pGpu) &&
1152 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL)
1153 {
1154 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
1155 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool;
1156 }
1157
1158 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1159 memdescCreate(&pGrCtxBufferMemDesc, pGpu, ctxSize,
1160 RM_PAGE_SIZE, bIsContiguous, ADDR_UNKNOWN,
1161 pAttr->cpuAttr,
1162 allocFlags | MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE));
1163
1164 //
1165 // Force page size to 4KB, we can change this later when RM access method
1166 // support 64k pages
1167 //
1168 NV_ASSERT_OK_OR_RETURN(
1169 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pGrCtxBufferMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB));
1170
1171 NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(pGrCtxBufferMemDesc, pCtxBufPool));
1172
1173 NV_STATUS status;
1174 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_CONTEXT_BUFFER, pGrCtxBufferMemDesc, pAttr->pAllocList);
1175 NV_ASSERT_OK_OR_RETURN(status);
1176
1177 NV_ASSERT_OK_OR_RETURN(
1178 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel,
1179 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)),
1180 pGrCtxBufferMemDesc));
1181 pKernelGraphicsContextUnicast->pMainCtxBuffer = pGrCtxBufferMemDesc;
1182 return NV_OK;
1183 }
1184
1185 /*!
1186 * @brief Allocate and setup the GR ctx patch buffer
1187 */
1188 NV_STATUS
kgrctxAllocPatchBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)1189 kgrctxAllocPatchBuffer_IMPL
1190 (
1191 OBJGPU *pGpu,
1192 KernelGraphicsContext *pKernelGraphicsContext,
1193 KernelGraphics *pKernelGraphics,
1194 KernelChannel *pKernelChannel
1195 )
1196 {
1197 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1198 const KGRAPHICS_STATIC_INFO *pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
1199 NvU32 status = NV_OK;
1200 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1201 MEMORY_DESCRIPTOR **ppMemDesc;
1202 NvU64 size;
1203 NvU64 flags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
1204 CTX_BUF_POOL_INFO *pCtxBufPool;
1205 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PATCH);
1206
1207 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
1208
1209 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE);
1210 NV_ASSERT_OR_RETURN(pStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
1211
1212 NV_ASSERT_OK_OR_RETURN(
1213 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1214
1215 pCtxBufPool = NULL;
1216 if (ctxBufPoolIsSupported(pGpu) &&
1217 (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool) != NULL)
1218 {
1219 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
1220 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool;
1221 }
1222
1223 ppMemDesc = &pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc;
1224 size = pStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PATCH].size;
1225 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1226 memdescCreate(ppMemDesc, pGpu, size, RM_PAGE_SIZE, NV_TRUE,
1227 ADDR_UNKNOWN,
1228 pAttr->cpuAttr,
1229 flags));
1230
1231 //
1232 // Force page size to 4KB we can change this later when RM access method
1233 // support 64k pages
1234 //
1235 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
1236 NV_ASSERT_OK_OR_GOTO(status,
1237 memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool),
1238 failed);
1239
1240 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_117,
1241 (*ppMemDesc), pAttr->pAllocList);
1242 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
1243 status,
1244 failed);
1245
1246 return NV_OK;
1247
1248 failed:
1249 memdescFree(*ppMemDesc);
1250 memdescDestroy(*ppMemDesc);
1251 *ppMemDesc = NULL;
1252 return status;
1253 }
1254
1255 /*!
1256 * @brief Allocate the local ctx PM buffer
1257 */
1258 NV_STATUS
kgrctxAllocPmBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)1259 kgrctxAllocPmBuffer_IMPL
1260 (
1261 OBJGPU *pGpu,
1262 KernelGraphicsContext *pKernelGraphicsContext,
1263 KernelGraphics *pKernelGraphics,
1264 KernelChannel *pKernelChannel
1265 )
1266 {
1267 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1268 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
1269 NvU32 size;
1270 NV_STATUS status = NV_OK;
1271 NvU64 flags = MEMDESC_FLAGS_GPU_PRIVILEGED;
1272 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1273 CTX_BUF_POOL_INFO *pCtxBufPool;
1274 MEMORY_DESCRIPTOR **ppMemDesc;
1275 const GR_BUFFER_ATTR *pAttr = kgraphicsGetContextBufferAttr(pGpu, pKernelGraphics, GR_CTX_BUFFER_PM);
1276
1277 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
1278
1279 NV_ASSERT_OK_OR_RETURN(
1280 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1281
1282 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
1283 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
1284 size = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PM].size;
1285 if (size == 0)
1286 return NV_ERR_INVALID_STATE;
1287
1288 pCtxBufPool = NULL;
1289 if (ctxBufPoolIsSupported(pGpu) &&
1290 pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool != NULL)
1291 {
1292 flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
1293 pCtxBufPool = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pCtxBufPool;
1294 }
1295
1296 //
1297 // For SRIOV Heavy, the PM ctxsw buffer allocation will be redirected to
1298 // host RM subheap. Subheap is used by host RM to allocate memory
1299 // on behalf of the guest(VF) context.
1300 //
1301 if (gpuIsWarBug200577889SriovHeavyEnabled(pGpu))
1302 {
1303 flags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
1304 }
1305
1306 ppMemDesc = &pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc;
1307 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1308 memdescCreate(ppMemDesc, pGpu,
1309 size,
1310 RM_PAGE_SIZE,
1311 NV_TRUE,
1312 ADDR_UNKNOWN,
1313 pAttr->cpuAttr,
1314 flags));
1315
1316 //
1317 // Force page size to 4KB we can change this later when RM access method
1318 // support 64k pages
1319 //
1320 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
1321 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB),
1322 error);
1323
1324 NV_ASSERT_OK_OR_GOTO(status, memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool), error);
1325
1326 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_118,
1327 (*ppMemDesc), pAttr->pAllocList);
1328 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
1329 status,
1330 error);
1331
1332 return NV_OK;
1333
1334 error:
1335 memdescFree(*ppMemDesc);
1336 memdescDestroy(*ppMemDesc);
1337 *ppMemDesc = NULL;
1338
1339 return status;
1340 }
1341
1342 /*!
1343 * This function allocates and maps various GR buffers.
1344 */
1345 NV_STATUS
kgrctxAllocCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelGraphicsObject * pKernelGraphicsObject)1346 kgrctxAllocCtxBuffers_IMPL
1347 (
1348 OBJGPU *pGpu,
1349 KernelGraphicsContext *pKernelGraphicsContext,
1350 KernelGraphics *pKernelGraphics,
1351 KernelGraphicsObject *pKernelGraphicsObject
1352 )
1353 {
1354 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1355 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant);
1356 NV_STATUS status = NV_OK;
1357 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId;
1358 NvU32 objType;
1359 NvU32 gfid = kchannelGetGfid(pChannelDescendant->pKernelChannel);
1360 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel;
1361 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
1362
1363 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
1364
1365 NV_ASSERT_OR_RETURN(kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject) != NULL, NV_ERR_INVALID_STATE);
1366
1367 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1368 kgrctxGetUnicast(pGpu,
1369 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject),
1370 &pKernelGraphicsContextUnicast));
1371
1372 // Allocate the GR ctx buffer if required
1373 if (!kgrctxIsMainContextAllocated(pGpu, pKernelGraphicsContext) &&
1374 !kchannelIsCtxBufferAllocSkipped(pChannelDescendant->pKernelChannel))
1375 {
1376 if (pKernelGraphicsContextUnicast->pMainCtxBuffer == NULL)
1377 {
1378 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1379 kgrctxAllocMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel));
1380 }
1381 else
1382 {
1383 // Ctx buffer was uninstalled in FIFO but not freed. Reinstall it
1384 NV_ASSERT_OK_OR_RETURN(
1385 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel,
1386 ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)),
1387 pKernelGraphicsContextUnicast->pMainCtxBuffer));
1388 }
1389 }
1390
1391 kgrmgrGetGrObjectType(classNum, &objType);
1392
1393 //
1394 // for each channel, we need to allocate the context patch buffer, this memory region
1395 // will be used to override settings in the context after it is restored, things
1396 // like the global ctx buffer addresses, etc.
1397 //
1398 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL)
1399 {
1400 NV_ASSERT_OK_OR_RETURN(
1401 kgrctxAllocPatchBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel));
1402 }
1403
1404 // Pre-allocate the PM ctxsw buffer, if required
1405 if (kgrctxShouldPreAllocPmBuffer_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel))
1406 {
1407 NV_ASSERT_OK_OR_RETURN(
1408 kgrctxAllocPmBuffer(pGpu, kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject),
1409 pKernelGraphics,
1410 pChannelDescendant->pKernelChannel));
1411 }
1412
1413 //
1414 // Allocate Ctx Buffers that are local to this channel if required
1415 // and they have yet to be allocated.
1416 //
1417 if ((pKernelGraphicsContextUnicast->bVprChannel
1418 ) && !pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated)
1419 {
1420 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1421 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid,
1422 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject)));
1423 }
1424
1425 // Allocate global context for this gfid if they haven't been already
1426 if (!pGlobalCtxBuffers->bAllocated)
1427 {
1428 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1429 kgraphicsAllocGrGlobalCtxBuffers_HAL(pGpu, pKernelGraphics, gfid, NULL));
1430 }
1431
1432 return status;
1433 }
1434
1435 // Map common buffer to a channel's context helper function
1436 NV_STATUS
kgrctxMapGlobalCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 gfid,OBJVASPACE * pVAS,GR_GLOBALCTX_BUFFER buffId,NvBool bIsReadOnly)1437 kgrctxMapGlobalCtxBuffer_IMPL
1438 (
1439 OBJGPU *pGpu,
1440 KernelGraphicsContext *pKernelGraphicsContext,
1441 KernelGraphics *pKernelGraphics,
1442 NvU32 gfid,
1443 OBJVASPACE *pVAS,
1444 GR_GLOBALCTX_BUFFER buffId,
1445 NvBool bIsReadOnly
1446 )
1447 {
1448 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
1449 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1450 NV_STATUS status = NV_OK;
1451 NvU64 vaddr;
1452 MEMORY_DESCRIPTOR *pMemDesc;
1453
1454 NV_ASSERT_OK_OR_RETURN(
1455 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1456
1457 // if we have local buffers per-channel allocated, use them, otherwise use the global buffers
1458 pMemDesc = pKernelGraphicsContextUnicast->localCtxBuffer.memDesc[buffId];
1459 if (pMemDesc == NULL)
1460 {
1461 pMemDesc = pGlobalCtxBuffers->memDesc[buffId];
1462 }
1463
1464 if (pMemDesc == NULL)
1465 {
1466 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo;
1467 NvU32 fifoEngineId;
1468 NvU32 buffSize;
1469
1470 pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
1471 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
1472 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
1473 NV_ASSERT_OK_OR_RETURN(kgrctxGlobalCtxBufferToFifoEngineId(buffId, &fifoEngineId));
1474 buffSize = pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[fifoEngineId].size;
1475
1476 if (buffSize == 0)
1477 {
1478 NV_PRINTF(LEVEL_INFO,
1479 "Could not map %s Buffer as buffer is not supported\n",
1480 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId));
1481 return NV_OK;
1482 }
1483 else
1484 {
1485 NV_PRINTF(LEVEL_ERROR,
1486 "Could not map %s Buffer, no memory allocated for it!\n",
1487 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId));
1488 return NV_ERR_INVALID_ARGUMENT;
1489 }
1490 }
1491
1492 // Unconditionally call map for refcounting
1493 NV_ASSERT_OK_OR_ELSE(status,
1494 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pVAS,
1495 &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId],
1496 kgraphicsIsGlobalCtxBufferSizeAligned(pGpu, pKernelGraphics, buffId),
1497 bIsReadOnly),
1498 NV_PRINTF(LEVEL_ERROR, "%s Buffer could not be mapped\n",
1499 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId));
1500 return status; );
1501
1502 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status,
1503 vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr));
1504
1505 NV_PRINTF(LEVEL_INFO,
1506 "GPU:%d %s Buffer PA @ 0x%llx VA @ 0x%llx of Size 0x%llx\n",
1507 pGpu->gpuInstance, NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId),
1508 memdescGetPhysAddr(memdescGetMemDescFromGpu(pMemDesc, pGpu), AT_GPU, 0),
1509 vaddr, pMemDesc->Size);
1510
1511 return status;
1512 }
1513
1514 //
1515 // Map common buffers to a channel's context
1516 //
1517 NV_STATUS
kgrctxMapGlobalCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,NvU32 gfid,KernelChannel * pKernelChannel)1518 kgrctxMapGlobalCtxBuffers_IMPL
1519 (
1520 OBJGPU *pGpu,
1521 KernelGraphicsContext *pKernelGraphicsContext,
1522 KernelGraphics *pKernelGraphics,
1523 NvU32 gfid,
1524 KernelChannel *pKernelChannel
1525 )
1526 {
1527 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
1528 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1529
1530 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
1531
1532 NV_ASSERT_OK_OR_RETURN(
1533 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1534
1535 // if global ctx buffers were never allocated then bail out early
1536 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated &&
1537 !pGlobalCtxBuffers->bAllocated)
1538 {
1539 return NV_ERR_INVALID_STATE;
1540 }
1541
1542 kgrctxMapGlobalCtxBuffer(pGpu,
1543 pKernelGraphicsContext,
1544 pKernelGraphics,
1545 gfid,
1546 pKernelChannel->pVAS,
1547 GR_GLOBALCTX_BUFFER_BUNDLE_CB,
1548 NV_FALSE);
1549 kgrctxMapGlobalCtxBuffer(pGpu,
1550 pKernelGraphicsContext,
1551 pKernelGraphics,
1552 gfid,
1553 pKernelChannel->pVAS,
1554 GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB,
1555 NV_FALSE);
1556 kgrctxMapGlobalCtxBuffer(pGpu,
1557 pKernelGraphicsContext,
1558 pKernelGraphics,
1559 gfid,
1560 pKernelChannel->pVAS,
1561 GR_GLOBALCTX_BUFFER_PAGEPOOL,
1562 NV_FALSE);
1563
1564 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics))
1565 {
1566 kgrctxMapGlobalCtxBuffer(pGpu,
1567 pKernelGraphicsContext,
1568 pKernelGraphics,
1569 gfid,
1570 pKernelChannel->pVAS,
1571 GR_GLOBALCTX_BUFFER_RTV_CB,
1572 NV_FALSE);
1573 }
1574
1575 return NV_OK;
1576 }
1577
1578 /*!
1579 * @brief This function allocates and maps various GR buffers.
1580 */
1581 NV_STATUS
kgrctxMapCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelGraphicsObject * pKernelGraphicsObject)1582 kgrctxMapCtxBuffers_IMPL
1583 (
1584 OBJGPU *pGpu,
1585 KernelGraphicsContext *pKernelGraphicsContext,
1586 KernelGraphics *pKernelGraphics,
1587 KernelGraphicsObject *pKernelGraphicsObject
1588 )
1589 {
1590 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1591 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant);
1592 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel;
1593 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId;
1594 OBJGVASPACE *pGVAS;
1595 NvU32 objType;
1596 NvU32 gfid = kchannelGetGfid(pKernelChannel);
1597 NvBool bAcquire3d = NV_FALSE;
1598
1599 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
1600
1601 NV_ASSERT_OK_OR_RETURN(
1602 kgrctxGetUnicast(pGpu, kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject), &pKernelGraphicsContextUnicast));
1603
1604 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE);
1605 if (gvaspaceIsExternallyOwned(pGVAS))
1606 return NV_OK;
1607
1608 //
1609 // in kernel RM we skip the context buffer mapping as part of
1610 // fifoSetUpChannelDma. fifoSetUpChannelDma is not enabled in kernel RM.
1611 // So it's possible for the main ctx buffer to be unmapped by this point.
1612 //
1613 if (!kchannelIsCtxBufferAllocSkipped(pKernelChannel))
1614 {
1615 ENGINE_CTX_DESCRIPTOR *pEngCtx;
1616 MEMORY_DESCRIPTOR *pMemDesc;
1617 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
1618
1619 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst];
1620
1621 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE);
1622 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE);
1623 pMemDesc = pEngCtx->pMemDesc;
1624
1625 NV_ASSERT_OK_OR_RETURN(
1626 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pMemDesc, pKernelChannel->pVAS,
1627 &pEngCtx->vaList, NV_FALSE, NV_FALSE));
1628 }
1629
1630 kgrmgrGetGrObjectType(classNum, &objType);
1631
1632 NV_ASSERT_OK_OR_RETURN(
1633 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc, pKernelChannel->pVAS,
1634 &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, NV_FALSE, NV_FALSE));
1635
1636 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
1637 {
1638 NV_ASSERT_OK_OR_RETURN(
1639 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics, pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc, pKernelChannel->pVAS,
1640 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, NV_FALSE, NV_FALSE));
1641 }
1642
1643 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics))
1644 {
1645 kgrctxMapGlobalCtxBuffer(pGpu,
1646 pKernelGraphicsContext,
1647 pKernelGraphics,
1648 gfid,
1649 pKernelChannel->pVAS,
1650 kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel),
1651 NV_TRUE);
1652 }
1653
1654 //
1655 // Condition for acquiring 3d context buffer mappings.
1656 // For non-TSG & legacy TSG mode, always map
1657 // For subcontext TSG case, only map on 2D/3D allocations
1658 //
1659 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode)
1660 {
1661 bAcquire3d = NV_TRUE;
1662 }
1663 else
1664 {
1665 bAcquire3d = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D));
1666 }
1667
1668 if (bAcquire3d)
1669 {
1670 kgrctxMapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, gfid, pKernelChannel);
1671 }
1672
1673 return NV_OK;
1674 }
1675
1676 /*!
1677 * @brief Set parameters for promoting the PA of a ctx buffer to physical RM.
1678 */
1679 NV_STATUS
kgrctxPrepareInitializeCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel,NvU32 externalId,NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY * pEntry,NvBool * pbAddEntry)1680 kgrctxPrepareInitializeCtxBuffer_IMPL
1681 (
1682 OBJGPU *pGpu,
1683 KernelGraphicsContext *pKernelGraphicsContext,
1684 KernelGraphics *pKernelGraphics,
1685 KernelChannel *pKernelChannel,
1686 NvU32 externalId,
1687 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry,
1688 NvBool *pbAddEntry
1689 )
1690 {
1691 MEMORY_DESCRIPTOR *pMemDesc;
1692 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1693 NvU32 physAttr;
1694
1695 *pbAddEntry = NV_FALSE;
1696
1697 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_CHANNEL);
1698
1699 NV_ASSERT_OK_OR_RETURN(
1700 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1701
1702 switch (externalId)
1703 {
1704 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN:
1705 {
1706 ENGINE_CTX_DESCRIPTOR *pEngCtx;
1707 NvU32 subdevInst;
1708 if (pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized)
1709 return NV_OK;
1710
1711 // Do not try to init a KMD virtual context buffer
1712 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel))
1713 return NV_OK;
1714
1715 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
1716 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst];
1717
1718 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE);
1719 NV_ASSERT_OR_RETURN(pEngCtx->pMemDesc != NULL, NV_ERR_INVALID_STATE);
1720 pMemDesc = pEngCtx->pMemDesc;
1721 break;
1722 }
1723 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM:
1724 if (pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized)
1725 return NV_OK;
1726
1727 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc == NULL)
1728 return NV_OK;
1729
1730 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc;
1731 break;
1732 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH:
1733 if (pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized)
1734 return NV_OK;
1735
1736 NV_ASSERT(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL);
1737 pMemDesc = pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc;
1738 break;
1739 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB:
1740 // fall-through
1741 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL:
1742 // fall-through
1743 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB:
1744 // fall-through
1745 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL:
1746 // fall-through
1747 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL:
1748 // No initialization from kernel RM
1749 return NV_OK;
1750 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK:
1751 {
1752 GR_GLOBALCTX_BUFFER internalId;
1753 NvU32 gfid = kchannelGetGfid(pKernelChannel);
1754 GR_GLOBALCTX_BUFFERS *pCtxBuffers;
1755
1756 NV_ASSERT_OK_OR_RETURN(
1757 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId));
1758
1759 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated)
1760 {
1761 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer;
1762 }
1763 else
1764 {
1765 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
1766 }
1767
1768 if (pCtxBuffers->bInitialized[internalId])
1769 return NV_OK;
1770
1771 pMemDesc = pCtxBuffers->memDesc[internalId];
1772 break;
1773 }
1774 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT:
1775 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP:
1776 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP:
1777 {
1778 GR_GLOBALCTX_BUFFER internalId;
1779 NvU32 gfid = kchannelGetGfid(pKernelChannel);
1780 GR_GLOBALCTX_BUFFERS *pCtxBuffers;
1781
1782 NV_ASSERT_OK_OR_RETURN(
1783 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId));
1784
1785 pCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
1786
1787 if (pCtxBuffers->bInitialized[internalId])
1788 return NV_OK;
1789
1790 pMemDesc = pCtxBuffers->memDesc[internalId];
1791
1792 break;
1793 }
1794 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP:
1795 // No initialization from kernel RM
1796 return NV_OK;
1797 default:
1798 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT);
1799 }
1800
1801 // If these buffers are not supported or not allocated, no need to init
1802 if (pMemDesc == NULL)
1803 return NV_OK;
1804
1805 physAttr = 0x0;
1806 switch (memdescGetAddressSpace(pMemDesc))
1807 {
1808 case ADDR_FBMEM:
1809 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX,
1810 _APERTURE, _VIDMEM, physAttr);
1811 break;
1812
1813 case ADDR_SYSMEM:
1814 if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED)
1815 {
1816 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX,
1817 _APERTURE, _COH_SYS, physAttr);
1818 }
1819 else if (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED)
1820 {
1821 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX,
1822 _APERTURE, _NCOH_SYS, physAttr);
1823 }
1824 else
1825 {
1826 return NV_ERR_INVALID_STATE;
1827 }
1828 break;
1829
1830 default:
1831 return NV_ERR_INVALID_STATE;
1832 }
1833
1834 physAttr = FLD_SET_DRF(2080, _CTRL_GPU_INITIALIZE_CTX, _GPU_CACHEABLE, _NO, physAttr);
1835
1836 pEntry->gpuPhysAddr = memdescGetPhysAddr(pMemDesc, AT_GPU, 0);
1837 pEntry->size = pMemDesc->Size;
1838 pEntry->physAttr = physAttr;
1839 pEntry->bufferId = externalId;
1840 pEntry->bInitialize = NV_TRUE;
1841 pEntry->bNonmapped = NV_TRUE;
1842
1843 *pbAddEntry = NV_TRUE;
1844
1845 return NV_OK;
1846 }
1847
1848 /*!
1849 * @brief Set parameters for promoting the VA of a ctx buffer to physical RM.
1850 */
1851 NV_STATUS
kgrctxPreparePromoteCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel,NvU32 externalId,NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY * pEntry,NvBool * pbAddEntry)1852 kgrctxPreparePromoteCtxBuffer_IMPL
1853 (
1854 OBJGPU *pGpu,
1855 KernelGraphicsContext *pKernelGraphicsContext,
1856 KernelChannel *pKernelChannel,
1857 NvU32 externalId,
1858 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *pEntry,
1859 NvBool *pbAddEntry
1860 )
1861 {
1862 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1863 VA_LIST *pVaList;
1864 NvU64 vaddr;
1865 NvU64 refCount;
1866 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE);
1867 NV_STATUS status;
1868
1869 NV_ASSERT_OK_OR_RETURN(
1870 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
1871
1872 *pbAddEntry = NV_FALSE;
1873
1874 // RM is not responsible for promoting the buffers when UVM is enabled
1875 if (gvaspaceIsExternallyOwned(pGVAS))
1876 return NV_OK;
1877
1878 switch (externalId)
1879 {
1880 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN:
1881 {
1882 ENGINE_CTX_DESCRIPTOR *pEngCtx;
1883 NvU32 subdevInst;
1884
1885 // Do not try to promote a KMD virtual context buffer
1886 if (kchannelIsCtxBufferAllocSkipped(pKernelChannel))
1887 return NV_OK;
1888
1889 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
1890 pEngCtx = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst];
1891
1892 NV_ASSERT_OR_RETURN(pEngCtx != NULL, NV_ERR_INVALID_STATE);
1893 pVaList = &pEngCtx->vaList;
1894 break;
1895 }
1896 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM:
1897 pVaList = &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList;
1898 break;
1899 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH:
1900 pVaList = &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList;
1901 break;
1902 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB:
1903 // fall-through
1904 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL:
1905 // fall-through
1906 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB:
1907 // fall-through
1908 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL:
1909 // fall-through
1910 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL:
1911 // fall-through
1912 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK:
1913 // fall-through
1914 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT:
1915 // fall-through
1916 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP:
1917 // fall-through
1918 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP:
1919 // fall-through
1920 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP:
1921 {
1922 GR_GLOBALCTX_BUFFER internalId;
1923
1924 NV_ASSERT_OK_OR_RETURN(
1925 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId));
1926
1927 pVaList = &pKernelGraphicsContextUnicast->globalCtxBufferVaList[internalId];
1928 break;
1929 }
1930 default:
1931 NV_ASSERT_OR_RETURN(!"Unrecognized promote ctx enum", NV_ERR_INVALID_ARGUMENT);
1932 }
1933
1934 // If the buffer isn't currently mapped or was already promoted, nothing to do
1935 status = vaListGetRefCount(pVaList, pKernelChannel->pVAS, &refCount);
1936 if ((status != NV_OK) || (refCount > 1))
1937 return NV_OK;
1938
1939 NV_ASSERT_OK_OR_RETURN(vaListFindVa(pVaList, pKernelChannel->pVAS, &vaddr));
1940
1941 pEntry->bufferId = externalId;
1942 pEntry->gpuVirtAddr = vaddr;
1943 pEntry->bNonmapped = NV_FALSE;
1944
1945 *pbAddEntry = NV_TRUE;
1946 return NV_OK;
1947 }
1948
1949 /*! Mark the context buffer as initialized, prevent further calls to init */
1950 void
kgrctxMarkCtxBufferInitialized_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel,NvU32 externalId)1951 kgrctxMarkCtxBufferInitialized_IMPL
1952 (
1953 OBJGPU *pGpu,
1954 KernelGraphicsContext *pKernelGraphicsContext,
1955 KernelGraphics *pKernelGraphics,
1956 KernelChannel *pKernelChannel,
1957 NvU32 externalId
1958 )
1959 {
1960 NV_STATUS status;
1961 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
1962
1963 NV_ASSERT_OK_OR_ELSE(status,
1964 kgrctxGetUnicast(pGpu,
1965 pKernelGraphicsContext,
1966 &pKernelGraphicsContextUnicast),
1967 return;);
1968
1969 switch (externalId)
1970 {
1971 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN:
1972 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_TRUE;
1973 break;
1974 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM:
1975 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_TRUE;
1976 break;
1977 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PATCH:
1978 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_TRUE;
1979 break;
1980 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_BUFFER_BUNDLE_CB:
1981 // fall-through
1982 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PAGEPOOL:
1983 // fall-through
1984 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB:
1985 // fall-through
1986 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_RTV_CB_GLOBAL:
1987 // fall-through
1988 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_POOL:
1989 // fall-through
1990 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GFXP_CTRL_BLK:
1991 {
1992 // If "local" global ctx buffers are allocated, check those first
1993 if (pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated)
1994 {
1995 GR_GLOBALCTX_BUFFER internalId;
1996 GR_GLOBALCTX_BUFFERS *pKCtxBuffers;
1997
1998 NV_ASSERT_OK_OR_ELSE(status,
1999 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId),
2000 return;);
2001
2002 pKCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer;
2003 pKCtxBuffers->bInitialized[internalId] = NV_TRUE;
2004 break;
2005 }
2006 // fall-through
2007 }
2008 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_FECS_EVENT:
2009 // fall-through
2010 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP:
2011 // fall-through
2012 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP:
2013 // fall-through
2014 case NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_GLOBAL_PRIV_ACCESS_MAP:
2015 {
2016 GR_GLOBALCTX_BUFFER internalId;
2017 NvU32 gfid = kchannelGetGfid(pKernelChannel);
2018 GR_GLOBALCTX_BUFFERS *pKCtxBuffers;
2019
2020 NV_ASSERT_OK_OR_ELSE(status,
2021 kgrctxGetGlobalContextBufferInternalId(externalId, &internalId),
2022 return;);
2023
2024 // TODO XXX Make context buffers available from KGRCTX alone
2025 pKCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
2026
2027 pKCtxBuffers->bInitialized[internalId] = NV_TRUE;
2028 break;
2029 }
2030 default:
2031 NV_ASSERT(!"Unrecognized promote ctx enum");
2032 }
2033 }
2034
2035 /*! Non-UVM late bind PM ctx buffer */
2036 NV_STATUS
kgrctxSetupDeferredPmBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)2037 kgrctxSetupDeferredPmBuffer_IMPL
2038 (
2039 OBJGPU *pGpu,
2040 KernelGraphicsContext *pKernelGraphicsContext,
2041 KernelGraphics *pKernelGraphics,
2042 KernelChannel *pKernelChannel
2043 )
2044 {
2045 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2046 OBJGVASPACE *pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE);
2047 NV_STATUS status = NV_OK;
2048 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannel);
2049 Subdevice *pSubdevice;
2050
2051 NV_ASSERT_OK_OR_RETURN(
2052 subdeviceGetByInstance(
2053 RES_GET_CLIENT(pKernelChannel),
2054 RES_GET_HANDLE(pDevice),
2055 gpumgrGetSubDeviceInstanceFromGpu(pGpu),
2056 &pSubdevice));
2057
2058 GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
2059
2060 NV_ASSERT_OK_OR_RETURN(
2061 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
2062
2063 if (gvaspaceIsExternallyOwned(pGVAS))
2064 {
2065 //
2066 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers
2067 // This is done as a WAR for the issue tracked under bug 1760699
2068 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the
2069 // buffers being allocated at channel allocation time and are not going to work for a buffer
2070 // created later with an rm ctrl
2071 //
2072 NV_ASSERT_OR_RETURN(pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL, NV_ERR_INVALID_STATE);
2073 return NV_OK;
2074 }
2075 else if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
2076 {
2077 return NV_OK;
2078 }
2079
2080 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
2081 kgrctxAllocPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel));
2082
2083 //
2084 // !!!
2085 // From this point on, use `goto failed` to exit
2086 // !!!
2087 //
2088
2089 {
2090 RS_ORDERED_ITERATOR it;
2091 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel);
2092
2093 // Iterate over all channels in this TSG and map the new buffer
2094 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm &&
2095 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics))
2096 {
2097 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi);
2098 }
2099
2100 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef);
2101 while (clientRefOrderedIterNext(it.pClient, &it))
2102 {
2103 NvU64 vaddr;
2104 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel);
2105
2106 NV_ASSERT_OR_ELSE(pLoopKernelChannel != NULL,
2107 status = NV_ERR_INVALID_STATE;
2108 goto failed;);
2109
2110 if ((vaListFindVa(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK) &&
2111 (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) != NV_OK))
2112 {
2113 NvU64 refCount;
2114 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS params;
2115 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
2116 NvBool bInitialize;
2117 NvBool bPromote;
2118
2119 NV_ASSERT_OK_OR_GOTO(status,
2120 kgraphicsMapCtxBuffer(pGpu, pKernelGraphics,
2121 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc,
2122 pLoopKernelChannel->pVAS,
2123 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList,
2124 NV_FALSE,
2125 NV_FALSE),
2126 failed);
2127
2128 // Promote the vaddr for each unique VAS
2129 portMemSet(¶ms, 0, sizeof(params));
2130
2131 // Setup parameters to initialize the PA if necessary
2132 NV_ASSERT_OK_OR_GOTO(status,
2133 kgrctxPrepareInitializeCtxBuffer(pGpu,
2134 pKernelGraphicsContext,
2135 pKernelGraphics,
2136 pKernelChannel,
2137 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM,
2138 ¶ms.promoteEntry[0],
2139 &bInitialize),
2140 failed);
2141
2142 // Setup parameters to promote the VA if necessary
2143 NV_ASSERT_OK_OR_GOTO(status,
2144 kgrctxPreparePromoteCtxBuffer(pGpu,
2145 pKernelGraphicsContext,
2146 pKernelChannel,
2147 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PM,
2148 ¶ms.promoteEntry[0],
2149 &bPromote),
2150 failed);
2151
2152 NV_ASSERT_OR_ELSE(bInitialize || bPromote,
2153 status = NV_ERR_INVALID_STATE;
2154 goto failed;);
2155
2156 params.engineType = NV2080_ENGINE_TYPE_GR(kgraphicsGetInstance(pGpu, pKernelGraphics));
2157 params.hChanClient = RES_GET_CLIENT_HANDLE(pKernelChannel);
2158 params.hObject = RES_GET_HANDLE(pKernelChannel);
2159 params.entryCount = 1;
2160
2161 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
2162 pRmApi->Control(pRmApi,
2163 RES_GET_CLIENT_HANDLE(pSubdevice),
2164 RES_GET_HANDLE(pSubdevice),
2165 NV2080_CTRL_CMD_GPU_PROMOTE_CTX,
2166 ¶ms,
2167 sizeof(params)),
2168 failed);
2169
2170 //
2171 // If we successfully promoted the PA, flip a flag to ensure we don't
2172 // try to promote it again. The VA_LIST should already track this for
2173 // VA, but we can't rely on it for PA due to UVM.
2174 //
2175 if (params.promoteEntry[0].bInitialize)
2176 {
2177 kgrctxMarkCtxBufferInitialized(pGpu,
2178 pKernelGraphicsContext,
2179 pKernelGraphics,
2180 pKernelChannel,
2181 params.promoteEntry[0].bufferId);
2182 }
2183
2184 // Update the refcount for this buffer now that we've promoted it
2185 NV_ASSERT_OK_OR_GOTO(status,
2186 vaListGetRefCount(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList, pLoopKernelChannel->pVAS, &refCount),
2187 failed);
2188 NV_ASSERT_OK_OR_GOTO(status,
2189 vaListSetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, refCount),
2190 failed);
2191 }
2192 }
2193 }
2194
2195 return NV_OK;
2196
2197 failed:
2198
2199 if (kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics))
2200 {
2201 RS_ORDERED_ITERATOR it;
2202 RsResourceRef *pScopeRef = RES_GET_REF(pKernelChannel);
2203
2204 // Iterate over all channels in this TSG and try to unmap the PM ctx buffer
2205 if (!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm &&
2206 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics))
2207 {
2208 pScopeRef = RES_GET_REF(pKernelChannel->pKernelChannelGroupApi);
2209 }
2210
2211 it = kchannelGetIter(RES_GET_CLIENT(pKernelChannel), pScopeRef);
2212 while (clientRefOrderedIterNext(it.pClient, &it))
2213 {
2214 NvU64 vaddr;
2215 KernelChannel *pLoopKernelChannel = dynamicCast(it.pResourceRef->pResource, KernelChannel);
2216 if (pLoopKernelChannel == NULL)
2217 continue;
2218
2219 while (vaListFindVa(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pLoopKernelChannel->pVAS, &vaddr) == NV_OK)
2220 {
2221 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics,
2222 pLoopKernelChannel->pVAS,
2223 &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList);
2224 }
2225 }
2226 }
2227
2228 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext);
2229
2230 return status;
2231 }
2232
2233 /**
2234 * @brief unmap the memory for the pm context buffer associated with a channel
2235 */
2236 void
kgrctxUnmapCtxPmBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,OBJVASPACE * pVAS)2237 kgrctxUnmapCtxPmBuffer_IMPL
2238 (
2239 OBJGPU *pGpu,
2240 KernelGraphicsContext *pKernelGraphicsContext,
2241 KernelGraphics *pKernelGraphics,
2242 OBJVASPACE *pVAS
2243 )
2244 {
2245 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2246 NV_STATUS status;
2247
2248 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2249
2250 NV_ASSERT_OK_OR_ELSE(status,
2251 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2252 return;);
2253
2254 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList);
2255 }
2256
2257 /**
2258 * @brief unmap the memory for the zcull context buffer
2259 */
2260 void
kgrctxUnmapCtxZcullBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,OBJVASPACE * pVAS)2261 kgrctxUnmapCtxZcullBuffer_IMPL
2262 (
2263 OBJGPU *pGpu,
2264 KernelGraphicsContext *pKernelGraphicsContext,
2265 KernelGraphics *pKernelGraphics,
2266 OBJVASPACE *pVAS
2267 )
2268 {
2269 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2270 MEMORY_DESCRIPTOR *pMemDesc;
2271 VA_LIST *pVaList;
2272 NvU64 vaddr;
2273 NV_STATUS status;
2274
2275 NV_ASSERT_OK_OR_ELSE(status,
2276 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2277 return;);
2278
2279 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2280
2281 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc;
2282 pVaList = &pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList;
2283 if (pMemDesc != NULL)
2284 {
2285 //
2286 // This func assumes that the buffer was not allocated per subdevice,
2287 // and will leak any mappings performed by the secondaries.
2288 //
2289 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2290
2291 if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK)
2292 {
2293 dmaUnmapBuffer_HAL(pGpu, GPU_GET_DMA(pGpu), pVAS, vaddr);
2294 vaListRemoveVa(pVaList, pVAS);
2295 }
2296 }
2297 else if (vaListFindVa(pVaList, pVAS, &vaddr) == NV_OK)
2298 {
2299 // Zcull buffer mapped by client. Remove the VA here
2300 vaListRemoveVa(pVaList, pVAS);
2301 }
2302 }
2303
2304 /**
2305 * @brief unmap the memory for the preemption context buffers
2306 */
2307 void
kgrctxUnmapCtxPreemptionBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,OBJVASPACE * pVAS)2308 kgrctxUnmapCtxPreemptionBuffers_IMPL
2309 (
2310 OBJGPU *pGpu,
2311 KernelGraphicsContext *pKernelGraphicsContext,
2312 KernelGraphics *pKernelGraphics,
2313 OBJVASPACE *pVAS
2314 )
2315 {
2316 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2317 NV_STATUS status;
2318
2319 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2320
2321 NV_ASSERT_OK_OR_ELSE(status,
2322 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2323 return;);
2324
2325 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList);
2326
2327 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList);
2328
2329 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList);
2330
2331 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList);
2332
2333 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList);
2334 }
2335
2336 /**
2337 * @brief Unmap associated ctx buffers (main, patch, global buffers etc).
2338 *
2339 * This function is called on every channel free and therefore can't assume any
2340 * graphics objects actually exist on the given channel.
2341 * TODO: Bug 3164256
2342 * This function also unmaps and frees zcull and preempt buffers. Ideally we want to decouple
2343 * unmap and free for all buffers and move free of buffers to kgrctxFreeAssociatedCtxBuffers
2344 */
2345 void
kgrctxUnmapAssociatedCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)2346 kgrctxUnmapAssociatedCtxBuffers_IMPL
2347 (
2348 OBJGPU *pGpu,
2349 KernelGraphicsContext *pKernelGraphicsContext,
2350 KernelGraphics *pKernelGraphics,
2351 KernelChannel *pKernelChannel
2352 )
2353 {
2354 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2355 NvBool bRelease3d = NV_FALSE;
2356 NvU32 gfid = kchannelGetGfid(pKernelChannel);
2357 NvU32 status;
2358 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pKernelChannel);
2359
2360 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2361
2362 NV_ASSERT_OK_OR_ELSE(status,
2363 kgrctxGetUnicast(pGpu,
2364 pKernelGraphicsContext,
2365 &pKernelGraphicsContextUnicast),
2366 return;);
2367
2368 //
2369 // Unmap 3D context buffers on the last 3D channel free
2370 //
2371 // Condition for unmapping 3d context buffer mappings:
2372 // For non-TSG & legacy TSG mode, always unmap 3d context buffers.
2373 // For subcontext TSG case, unmap on the last graphics channel free.
2374 //
2375 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode)
2376 {
2377 bRelease3d = pKernelGraphicsContextUnicast->channelObjects == 0;
2378 }
2379 else
2380 {
2381 bRelease3d = ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] == 0) &&
2382 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] == 0));
2383 }
2384
2385 if (bRelease3d)
2386 {
2387 // Unmap Circular buffer from the current channel's address space
2388 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid);
2389 }
2390
2391 //
2392 // If multiple channels sharing the same VAS exist, it is possible both
2393 // channels could be using these mappings, and we must wait for both
2394 // channels to be detached before we remove them.
2395 //
2396 if ((!pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bAllocatedByRm) &&
2397 (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList != NULL))
2398 {
2399 CHANNEL_NODE *pChanNode;
2400 CHANNEL_LIST *pChanList;
2401
2402 pChanList = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->pChanList;
2403
2404 for (pChanNode = pChanList->pHead; pChanNode; pChanNode = pChanNode->pNext)
2405 {
2406 // Skip the channel we are looking to unmap
2407 if (kchannelGetDebugTag(pKernelChannel) == kchannelGetDebugTag(pChanNode->pKernelChannel))
2408 continue;
2409
2410 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, pChanNode->pKernelChannel->pVAS != pKernelChannel->pVAS);
2411 }
2412 }
2413
2414 // Only unmap once the last channel using this VAS has gone
2415 if (kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL)
2416 {
2417 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT);
2418 }
2419
2420 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
2421 {
2422 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS);
2423 }
2424
2425 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics))
2426 {
2427 kgrctxUnmapGlobalCtxBuffer(pGpu,
2428 pKernelGraphicsContext,
2429 pKernelGraphics,
2430 pKernelChannel->pVAS,
2431 registerMapBufferId);
2432 }
2433
2434 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL)
2435 {
2436 //
2437 // Delay freeing the patch buffer until the last channel free.
2438 // these buffers are accessed even after the last GR object is freed.
2439 //
2440 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList);
2441 }
2442
2443 kgrctxUnmapCtxZcullBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS);
2444
2445 // Release all preemption buffers if they were allocated
2446 kgrctxUnmapCtxPreemptionBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS);
2447
2448 //
2449 // Cleanup lingering main ctx buffer mappings for this VAS
2450 // TODO fix main ctx buffer refcounting
2451 // TODO move setEngineContextMemDesc to FreeMainCtxBuffer, move this loop
2452 // inside UnmapMainCtxBuffer
2453 //
2454 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] != NULL)
2455 {
2456 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
2457 VA_LIST *pVAddrList = &pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->ppEngCtxDesc[subdevInst]->vaList;
2458 NvU64 vaddr;
2459
2460 while (vaListFindVa(pVAddrList, pKernelChannel->pVAS, &vaddr) == NV_OK)
2461 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, pVAddrList);
2462 }
2463
2464 //
2465 // When sharing contexts across channels we need to defer this until all
2466 // objects have been freed.
2467 //
2468 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT,
2469 pKernelGraphicsContextUnicast->channelObjects == 0);
2470
2471 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel);
2472 }
2473
2474 /*!
2475 * @brief is this object responsible for any cleanup tasks i.e. buffer unmapping?
2476 */
kgrctxShouldCleanup_KERNEL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2477 NvBool kgrctxShouldCleanup_KERNEL
2478 (
2479 OBJGPU *pGpu,
2480 KernelGraphicsContext *pKernelGraphicsContext
2481 )
2482 {
2483 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu);
2484 };
2485
2486 /*!
2487 * This function returns whether PM ctxsw buffer should be pre-allocated or not.
2488 */
kgrctxShouldPreAllocPmBuffer_PF(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel)2489 NvBool kgrctxShouldPreAllocPmBuffer_PF
2490 (
2491 OBJGPU *pGpu,
2492 KernelGraphicsContext *pKernelGraphicsContext,
2493 KernelChannel *pKernelChannel
2494 )
2495 {
2496 OBJGVASPACE *pGVAS = NULL;
2497 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2498
2499 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2500 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE);
2501 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE);
2502
2503 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
2504 kgrctxGetUnicast(pGpu,
2505 pKernelGraphicsContext,
2506 &pKernelGraphicsContextUnicast));
2507
2508 pGVAS = dynamicCast(pKernelChannel->pVAS, OBJGVASPACE);
2509
2510 // Do not allocate the buffer, if already allocated
2511 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
2512 {
2513 return NV_FALSE;
2514 }
2515
2516 //
2517 // The PM Ctxsw buffer is now pre-allocated along with the other context buffers
2518 // This is done as a WAR for the issue tracked under bug 1760699
2519 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the
2520 // buffers being allocated at channel allocation time and are not going to work for a buffer
2521 // created later with an rm ctrl
2522 //
2523 return gvaspaceIsExternallyOwned(pGVAS);
2524 }
2525
2526 /*!
2527 * This function returns whether PM ctxsw buffer should be pre-allocated
2528 * or not.
2529 */
2530 NvBool
kgrctxShouldPreAllocPmBuffer_VF(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel)2531 kgrctxShouldPreAllocPmBuffer_VF
2532 (
2533 OBJGPU *pGpu,
2534 KernelGraphicsContext *pKernelGraphicsContext,
2535 KernelChannel *pKernelChannel
2536 )
2537 {
2538 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast = NULL;
2539
2540 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2541 NV_ASSERT_OR_RETURN(NULL != pKernelChannel, NV_ERR_INVALID_STATE);
2542 NV_ASSERT_OR_RETURN(pKernelGraphicsContext != NULL, NV_ERR_INVALID_STATE);
2543
2544 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
2545 kgrctxGetUnicast(pGpu,
2546 pKernelGraphicsContext,
2547 &pKernelGraphicsContextUnicast));
2548
2549 // Do not allocate the buffer, if already allocated
2550 if (pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL)
2551 {
2552 return NV_FALSE;
2553 }
2554
2555 //
2556 // Pre-allocate the PM Ctxsw buffer along with the other context buffers
2557 // in below scenarios:
2558 // 1. For externally owned VA spaces. This is done as a WAR for the issue
2559 // tracked under bug 1760699.
2560 // Reason: With the enablement of UVM8 by default, the UVM channel registration relies on the
2561 // buffers being allocated at channel allocation time and are not going to work for a buffer
2562 // created later with an rm ctrl.
2563 // 2. For full SRIOV vGPU guests with Profiling capability enabled
2564 //
2565 if (IS_VIRTUAL_WITH_SRIOV(pGpu) && !gpuIsWarBug200577889SriovHeavyEnabled(pGpu))
2566 {
2567 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu);
2568 NvBool bProfilingEnabledVgpuGuest = NV_FALSE;
2569
2570 if (pVSI != NULL)
2571 {
2572 bProfilingEnabledVgpuGuest = pVSI->vgpuStaticProperties.bProfilingTracingEnabled;
2573 if (bProfilingEnabledVgpuGuest)
2574 {
2575 return NV_TRUE;
2576 }
2577 }
2578 }
2579
2580 return kgrctxShouldPreAllocPmBuffer_PF(pGpu, pKernelGraphicsContext, pKernelChannel);
2581 }
2582
2583 /*!
2584 * @brief should this layer manage the ctx buffers?
2585 */
2586 NvBool
kgrctxShouldManageCtxBuffers_KERNEL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NvU32 gfid)2587 kgrctxShouldManageCtxBuffers_KERNEL
2588 (
2589 OBJGPU *pGpu,
2590 KernelGraphicsContext *pKernelGraphicsContext,
2591 NvU32 gfid
2592 )
2593 {
2594 return gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) && !IS_GFID_VF(gfid);
2595 }
2596
2597 /*!
2598 * @brief should this layer manage the ctx buffers?
2599 * If client RM is managing the ctx buffers but the channel is a plugin
2600 * channel, we should still manage them.
2601 */
2602 NvBool
kgrctxShouldManageCtxBuffers_PHYSICAL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NvU32 gfid)2603 kgrctxShouldManageCtxBuffers_PHYSICAL
2604 (
2605 OBJGPU *pGpu,
2606 KernelGraphicsContext *pKernelGraphicsContext,
2607 NvU32 gfid
2608 )
2609 {
2610 return !gpuIsClientRmAllocatedCtxBufferEnabled(pGpu) || (gpuIsSriovEnabled(pGpu) && IS_GFID_PF(gfid));
2611 }
2612
2613 /**
2614 * @brief Unmaps buffers associated with this context.
2615 */
kgrctxUnmapBuffers_KERNEL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphicsContextUnicast * pKernelGraphicsContextUnicast,KernelChannel * pKernelChannel)2616 void kgrctxUnmapBuffers_KERNEL
2617 (
2618 OBJGPU *pGpu,
2619 KernelGraphicsContext *pKernelGraphicsContext,
2620 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast,
2621 KernelChannel *pKernelChannel
2622 )
2623 {
2624 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
2625 NV_STATUS status;
2626 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo;
2627 KernelGraphics *pKernelGraphics;
2628 Device *pDevice = GPU_RES_GET_DEVICE(pKernelGraphicsContext);
2629 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext);
2630
2631 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo));
2632 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo);
2633 NV_ASSERT_OK_OR_ELSE(status,
2634 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics),
2635 return; );
2636
2637 kgrctxUnmapAssociatedCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel);
2638 }
2639
2640 /*!
2641 * @brief Unmap main GR ctx buffer
2642 *
2643 * @param[in] pGpu
2644 * @param[in] pKernelGraphicsContext
2645 * @param[in] pKernelGraphics
2646 * @param[in] pKernelChannel
2647 */
2648 void
kgrctxUnmapMainCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelChannel * pKernelChannel)2649 kgrctxUnmapMainCtxBuffer_IMPL
2650 (
2651 OBJGPU *pGpu,
2652 KernelGraphicsContext *pKernelGraphicsContext,
2653 KernelGraphics *pKernelGraphics,
2654 KernelChannel *pKernelChannel
2655 )
2656 {
2657 MEMORY_DESCRIPTOR *pCtxBufferMemDesc;
2658 NV_STATUS status;
2659
2660 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2661
2662 status = kchangrpGetEngineContextMemDesc(pGpu, pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup, &pCtxBufferMemDesc);
2663 if ((status == NV_OK) && (pCtxBufferMemDesc != NULL))
2664 {
2665 // TODO remove Channel, ENG_GR dependencies
2666 kchannelUnmapEngineCtxBuf(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)));
2667 NV_ASSERT_OR_RETURN_VOID(
2668 kchannelSetEngineContextMemDesc(pGpu, pKernelChannel, ENG_GR(kgraphicsGetInstance(pGpu, pKernelGraphics)), NULL) == NV_OK);
2669 }
2670 }
2671
2672 /*!
2673 * @brief Unmap all global ctx buffers from this context
2674 */
2675 void
kgrctxUnmapGlobalCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,OBJVASPACE * pVAS,NvU32 gfid)2676 kgrctxUnmapGlobalCtxBuffers_IMPL
2677 (
2678 OBJGPU *pGpu,
2679 KernelGraphicsContext *pKernelGraphicsContext,
2680 KernelGraphics *pKernelGraphics,
2681 OBJVASPACE *pVAS,
2682 NvU32 gfid
2683 )
2684 {
2685 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid);
2686 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2687 NV_STATUS status;
2688
2689 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2690
2691 NV_ASSERT_OK_OR_ELSE(status,
2692 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2693 return;);
2694
2695 // if global ctx buffers were never allocated then bail out early
2696 if (!pKernelGraphicsContextUnicast->localCtxBuffer.bAllocated &&
2697 !pGlobalCtxBuffers->bAllocated)
2698 {
2699 return;
2700 }
2701
2702 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_BUNDLE_CB);
2703 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB);
2704 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_PAGEPOOL);
2705
2706 if (kgraphicsIsRtvCbSupported(pGpu, pKernelGraphics))
2707 {
2708 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pVAS, GR_GLOBALCTX_BUFFER_RTV_CB);
2709 }
2710
2711 }
2712
2713 /*!
2714 * @brief Unmap specified global ctx buffer from the given VAS
2715 */
2716 void
kgrctxUnmapGlobalCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,OBJVASPACE * pVAS,GR_GLOBALCTX_BUFFER buffId)2717 kgrctxUnmapGlobalCtxBuffer_IMPL
2718 (
2719 OBJGPU *pGpu,
2720 KernelGraphicsContext *pKernelGraphicsContext,
2721 KernelGraphics *pKernelGraphics,
2722 OBJVASPACE *pVAS,
2723 GR_GLOBALCTX_BUFFER buffId
2724 )
2725 {
2726 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2727 NV_STATUS status;
2728 NvU64 vaddr;
2729
2730 NV_ASSERT_OK_OR_ELSE(status,
2731 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2732 return;);
2733
2734 status = vaListFindVa(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId], pVAS, &vaddr);
2735 if (status == NV_OK)
2736 {
2737 NV_PRINTF(LEVEL_INFO, "Unmapping %s from VA @ 0x%llx\n",
2738 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId), vaddr);
2739
2740 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pVAS, &pKernelGraphicsContextUnicast->globalCtxBufferVaList[buffId]);
2741 }
2742 else
2743 {
2744 NV_PRINTF(LEVEL_INFO, "Buffer for %s already unmapped\n",
2745 NV_ENUM_TO_STRING(GR_GLOBALCTX_BUFFER, buffId));
2746 }
2747 }
2748
2749 /*!
2750 * @brief Free main GR ctx buffer
2751 */
2752 void
kgrctxFreeMainCtxBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2753 kgrctxFreeMainCtxBuffer_IMPL
2754 (
2755 OBJGPU *pGpu,
2756 KernelGraphicsContext *pKernelGraphicsContext
2757 )
2758 {
2759 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2760 NV_STATUS status;
2761
2762 NV_ASSERT_OK_OR_ELSE(status,
2763 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2764 return;);
2765
2766 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2767 if (pKernelGraphicsContextUnicast->pMainCtxBuffer != NULL)
2768 {
2769 memdescFree(pKernelGraphicsContextUnicast->pMainCtxBuffer);
2770 memdescDestroy(pKernelGraphicsContextUnicast->pMainCtxBuffer);
2771 pKernelGraphicsContextUnicast->pMainCtxBuffer = NULL;
2772 pKernelGraphicsContextUnicast->bKGrMainCtxBufferInitialized = NV_FALSE;
2773 }
2774 }
2775
2776 /**
2777 * @brief free the memory for the zcull context buffer
2778 */
2779 void
kgrctxFreeZcullBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2780 kgrctxFreeZcullBuffer_IMPL
2781 (
2782 OBJGPU *pGpu,
2783 KernelGraphicsContext *pKernelGraphicsContext
2784 )
2785 {
2786 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2787 MEMORY_DESCRIPTOR *pMemDesc;
2788 NV_STATUS status;
2789
2790 NV_ASSERT_OK_OR_ELSE(status,
2791 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2792 return;);
2793
2794 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2795
2796 pMemDesc = pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc;
2797 if (pMemDesc == NULL)
2798 {
2799 NV_PRINTF(LEVEL_INFO,
2800 "call to free zcull ctx buffer not RM managed, skipped!\n");
2801 }
2802
2803 // buffer can be shared and refcounted -- released on final free
2804 memdescFree(pMemDesc);
2805 memdescDestroy(pMemDesc);
2806 pKernelGraphicsContextUnicast->zcullCtxswBuffer.pMemDesc = NULL;
2807 }
2808
2809 /**
2810 * @brief free the memory for the preemption context buffers
2811 */
2812 void
kgrctxFreeCtxPreemptionBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2813 kgrctxFreeCtxPreemptionBuffers_IMPL
2814 (
2815 OBJGPU *pGpu,
2816 KernelGraphicsContext *pKernelGraphicsContext
2817 )
2818 {
2819 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2820 NV_STATUS status;
2821
2822 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2823
2824 NV_ASSERT_OK_OR_ELSE(status,
2825 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2826 return;);
2827
2828 memdescFree(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc);
2829 memdescDestroy(pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc);
2830 pKernelGraphicsContextUnicast->preemptCtxswBuffer.pMemDesc = NULL;
2831
2832 memdescFree(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc);
2833 memdescDestroy(pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc);
2834 pKernelGraphicsContextUnicast->spillCtxswBuffer.pMemDesc = NULL;
2835
2836 memdescFree(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc);
2837 memdescDestroy(pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc);
2838 pKernelGraphicsContextUnicast->betaCBCtxswBuffer.pMemDesc = NULL;
2839
2840 memdescFree(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc);
2841 memdescDestroy(pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc);
2842 pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.pMemDesc = NULL;
2843
2844 memdescFree(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc);
2845 memdescDestroy(pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc);
2846 pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.pMemDesc = NULL;
2847 }
2848
2849 /*!
2850 * @brief Free patch GR ctx buffer
2851 */
2852 void
kgrctxFreePatchBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2853 kgrctxFreePatchBuffer_IMPL
2854 (
2855 OBJGPU *pGpu,
2856 KernelGraphicsContext *pKernelGraphicsContext
2857 )
2858 {
2859 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2860 NV_STATUS status;
2861
2862 NV_ASSERT_OK_OR_ELSE(status,
2863 kgrctxGetUnicast(pGpu,
2864 pKernelGraphicsContext,
2865 &pKernelGraphicsContextUnicast),
2866 return;);
2867
2868 if (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc == NULL)
2869 {
2870 NV_PRINTF(LEVEL_INFO,
2871 "Attempt to free null ctx patch buffer pointer, skipped!\n");
2872 return;
2873 }
2874
2875 memdescFree(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc);
2876 memdescDestroy(pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc);
2877
2878 pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc = NULL;
2879 pKernelGraphicsContextUnicast->bKGrPatchCtxBufferInitialized = NV_FALSE;
2880 }
2881
2882 /**
2883 * @brief Free the memory for the pm context buffer associated with a channel
2884 */
2885 void
kgrctxFreePmBuffer_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2886 kgrctxFreePmBuffer_IMPL
2887 (
2888 OBJGPU *pGpu,
2889 KernelGraphicsContext *pKernelGraphicsContext
2890 )
2891 {
2892 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2893 MEMORY_DESCRIPTOR *pMemDesc;
2894 NV_STATUS status;
2895
2896 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2897
2898 NV_ASSERT_OK_OR_ELSE(status,
2899 kgrctxGetUnicast(pGpu,
2900 pKernelGraphicsContext,
2901 &pKernelGraphicsContextUnicast),
2902 return;);
2903
2904 pMemDesc = pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc;
2905 if (pMemDesc != NULL)
2906 {
2907 //
2908 // This func assumes that the buffer was not allocated per subdevice,
2909 // and will leak any mappings performed by the secondaries.
2910 //
2911 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2912
2913 memdescFree(pMemDesc);
2914 memdescDestroy(pMemDesc);
2915 }
2916 else
2917 {
2918 NV_PRINTF(LEVEL_INFO,
2919 "Attempt to free null pm ctx buffer pointer??\n");
2920 }
2921
2922 pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc = NULL;
2923 pKernelGraphicsContextUnicast->bKGrPmCtxBufferInitialized = NV_FALSE;
2924 }
2925
2926 /*!
2927 * @brief Free "local" global context buffers
2928 */
kgrctxFreeLocalGlobalCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2929 void kgrctxFreeLocalGlobalCtxBuffers_IMPL
2930 (
2931 OBJGPU *pGpu,
2932 KernelGraphicsContext *pKernelGraphicsContext
2933 )
2934 {
2935 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2936 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
2937 MEMORY_DESCRIPTOR *pMemDesc;
2938 GR_GLOBALCTX_BUFFERS *pCtxBuffers;
2939 GR_GLOBALCTX_BUFFER buff;
2940 NV_STATUS status;
2941
2942 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2943
2944 NV_ASSERT_OK_OR_ELSE(status,
2945 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
2946 return;);
2947
2948 pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer;
2949
2950 // no ctx buffers allocated, so get out early
2951 if (!pCtxBuffers->bAllocated)
2952 return;
2953
2954 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff)
2955 {
2956 pMemDesc = pCtxBuffers->memDesc[buff];
2957 memdescFree(pMemDesc);
2958 memdescDestroy(pMemDesc);
2959 pCtxBuffers->memDesc[buff] = NULL;
2960 }
2961 FOR_EACH_IN_ENUM_END;
2962
2963 pCtxBuffers->bAllocated = NV_FALSE;
2964
2965 // make sure all L2 cache lines using CB buffers are clear after we free them
2966 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, NULL, FB_CACHE_VIDEO_MEMORY, FB_CACHE_EVICT));
2967 }
2968
2969 /**
2970 * @brief Free all associated ctx buffers (main, patch, PM and private global buffers for cases like VPR).
2971 *
2972 * This function is called on every channel free and thefore can't assume any
2973 * graphics objects actually exist on the given channel.
2974 */
kgrctxFreeAssociatedCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext)2975 void kgrctxFreeAssociatedCtxBuffers_IMPL
2976 (
2977 OBJGPU *pGpu,
2978 KernelGraphicsContext *pKernelGraphicsContext
2979 )
2980 {
2981 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
2982 NV_STATUS status;
2983
2984 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
2985
2986 NV_ASSERT_OK_OR_ELSE(status,
2987 kgrctxGetUnicast(pGpu,
2988 pKernelGraphicsContext,
2989 &pKernelGraphicsContextUnicast),
2990 return;);
2991
2992 //
2993 // When sharing contexts across channels we need to defer this until all
2994 // objects have been freed.
2995 //
2996 NV_CHECK_OR_RETURN_VOID(LEVEL_SILENT, pKernelGraphicsContextUnicast->channelObjects == 0);
2997
2998 // if we have a context patch buffer, this will free it up
2999 kgrctxFreePatchBuffer(pGpu, pKernelGraphicsContext);
3000
3001 // if we have a PM ctxsw buffer, this will free it up
3002 kgrctxFreePmBuffer(pGpu, pKernelGraphicsContext);
3003
3004 // if we have a zcull buffer, this will free it up
3005 kgrctxFreeZcullBuffer(pGpu, pKernelGraphicsContext);
3006
3007 // If we have preemption buffers, this will free them up
3008 kgrctxFreeCtxPreemptionBuffers(pGpu, pKernelGraphicsContext);
3009
3010 // Release all common buffers used as part of the gr context.
3011 kgrctxFreeLocalGlobalCtxBuffers(pGpu, pKernelGraphicsContext);
3012
3013 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext);
3014 }
3015
3016 /*!
3017 * This function unmaps various GR buffers.
3018 */
3019 NV_STATUS
kgrctxUnmapCtxBuffers_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphicsObject * pKernelGraphicsObject,KernelGraphics * pKernelGraphics,NvBool bDestructor)3020 kgrctxUnmapCtxBuffers_IMPL
3021 (
3022 OBJGPU *pGpu,
3023 KernelGraphicsContext *pKernelGraphicsContext,
3024 KernelGraphicsObject *pKernelGraphicsObject,
3025 KernelGraphics *pKernelGraphics,
3026 NvBool bDestructor
3027 )
3028 {
3029 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3030 ChannelDescendant *pChannelDescendant = staticCast(pKernelGraphicsObject, ChannelDescendant);
3031 KernelChannel *pKernelChannel = pChannelDescendant->pKernelChannel;
3032 NvU32 classNum = pChannelDescendant->resourceDesc.externalClassId;
3033 NvBool bRelease3d = NV_FALSE;
3034 NvU32 objType;
3035 NvU32 gfid;
3036 NvU64 refCount;
3037 GR_GLOBALCTX_BUFFER registerMapBufferId = kgrctxGetRegisterAccessMapId_HAL(pGpu, pKernelGraphicsContext, pChannelDescendant->pKernelChannel);
3038
3039 NV_PRINTF(LEVEL_INFO, "gpu:%d isBC=%d\n", pGpu->gpuInstance,
3040 gpumgrGetBcEnabledStatus(pGpu));
3041
3042 gfid = kchannelGetGfid(pKernelChannel);
3043
3044 NV_ASSERT_OK_OR_RETURN(
3045 kgrctxGetUnicast(pGpu,
3046 kgrobjGetKernelGraphicsContext(pGpu, pKernelGraphicsObject),
3047 &pKernelGraphicsContextUnicast));
3048
3049 kgrmgrGetGrObjectType(classNum, &objType);
3050
3051 if (!bDestructor)
3052 {
3053 //
3054 // If we are cleaning up from the constructor, then we know the main ctx
3055 // buffer is not being used yet, and no other context buffers need to be
3056 // cleaned up, so we can return early here. If we are coming from the
3057 // destructor, the context buffers are not freed until channel
3058 // destruction.
3059 // TODO move this up one stack frame
3060 //
3061 kgrctxUnmapMainCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pChannelDescendant->pKernelChannel);
3062 kgrctxFreeMainCtxBuffer(pGpu, pKernelGraphicsContext);
3063 return NV_OK;
3064 }
3065
3066 if ((pKernelGraphicsContextUnicast->channelObjects != 0) &&
3067 (pKernelGraphicsContextUnicast->ctxPatchBuffer.pMemDesc != NULL))
3068 {
3069 //
3070 // Delay freeing the patch buffer until the last channel free.
3071 // these buffers are accessed even after the last GR object is freed.
3072 //
3073 kgraphicsUnmapCtxBuffer(pGpu, pKernelGraphics, pKernelChannel->pVAS, &pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList);
3074 }
3075
3076 // Defer releasing mapping if this would cause the buffer to be unmapped
3077 if ((pKernelGraphicsContextUnicast->pmCtxswBuffer.pMemDesc != NULL) &&
3078 (vaListGetRefCount(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList, pKernelChannel->pVAS, &refCount) == NV_OK) &&
3079 (refCount > 1))
3080 {
3081 kgrctxUnmapCtxPmBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS);
3082 }
3083
3084 // Defer releasing mapping if this would cause the buffer to be unmapped
3085 if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics) &&
3086 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[registerMapBufferId], pKernelChannel->pVAS, &refCount) == NV_OK) &&
3087 (refCount > 1))
3088 {
3089 kgrctxUnmapGlobalCtxBuffer(pGpu,
3090 pKernelGraphicsContext,
3091 pKernelGraphics,
3092 pKernelChannel->pVAS,
3093 registerMapBufferId);
3094 }
3095
3096 // Defer releasing mapping if this would cause the buffer to be unmapped
3097 if ((kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, gfid)->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT] != NULL) &&
3098 (vaListGetRefCount(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[GR_GLOBALCTX_BUFFER_FECS_EVENT], pKernelChannel->pVAS, &refCount) == NV_OK) &&
3099 (refCount > 1))
3100 {
3101 kgrctxUnmapGlobalCtxBuffer(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, GR_GLOBALCTX_BUFFER_FECS_EVENT);
3102 }
3103
3104 //
3105 // Condition for releasing 3d context buffer mappings:
3106 // For non-TSG & legacy TSG mode, always unmap as long as its not the last
3107 // object in TSG
3108 // If it is the last object, then unmap during channel free as these buffers
3109 // are accessed even after the last 3D object is freed.
3110 // For subcontext TSG case, only unmap on 2D/3D object free as long its not
3111 // the last 2D/3D object
3112 // For compute object with subcontext we never map so we don't need to call
3113 // unmap.
3114 // If we free on last object then buffer gets actually unmapped (as opposed
3115 // to decrement in refcount) and we start hitting page faults
3116 //
3117 if (pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup->bLegacyMode)
3118 {
3119 bRelease3d = (pKernelGraphicsContextUnicast->channelObjects > 0);
3120 }
3121 else
3122 {
3123 NvBool bIs3dBuffer = ((objType == GR_OBJECT_TYPE_2D) || (objType == GR_OBJECT_TYPE_3D));
3124 bRelease3d = (bIs3dBuffer && ((pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D] > 0) ||
3125 (pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D] > 0)));
3126 }
3127
3128 if (bRelease3d)
3129 {
3130 kgrctxUnmapGlobalCtxBuffers(pGpu, pKernelGraphicsContext, pKernelGraphics, pKernelChannel->pVAS, gfid);
3131 }
3132
3133 return NV_OK;
3134 }
3135
3136 /*!
3137 * Function to increment the GR channel object count
3138 */
3139 void
kgrctxIncObjectCount_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NvU32 classNum)3140 kgrctxIncObjectCount_IMPL
3141 (
3142 OBJGPU *pGpu,
3143 KernelGraphicsContext *pKernelGraphicsContext,
3144 NvU32 classNum
3145 )
3146 {
3147 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3148 NvU32 objType;
3149 NV_STATUS status;
3150
3151 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
3152
3153 kgrmgrGetGrObjectType(classNum, &objType);
3154
3155 NV_ASSERT_OK_OR_ELSE(status,
3156 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
3157 return;);
3158
3159 switch (objType)
3160 {
3161 case GR_OBJECT_TYPE_COMPUTE:
3162 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_COMPUTE]++;
3163 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_INCREMENT);
3164 break;
3165
3166 case GR_OBJECT_TYPE_3D:
3167 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_3D]++;
3168 break;
3169
3170 case GR_OBJECT_TYPE_2D:
3171 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_2D]++;
3172 break;
3173
3174 case GR_OBJECT_TYPE_MEM:
3175 pKernelGraphicsContextUnicast->objectCounts[GR_OBJECT_TYPE_MEM]++;
3176 break;
3177
3178 default:
3179 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n",
3180 classNum);
3181 DBG_BREAKPOINT();
3182 return;
3183 }
3184
3185 pKernelGraphicsContextUnicast->channelObjects++;
3186 NV_PRINTF(LEVEL_INFO,
3187 "Class 0x%x allocated. %d objects allocated\n",
3188 classNum,
3189 pKernelGraphicsContextUnicast->channelObjects);
3190 }
3191
3192 /*!
3193 * Function to decrement the GR channel object count of the given class
3194 */
3195 void
kgrctxDecObjectCount_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,NvU32 classNum)3196 kgrctxDecObjectCount_IMPL
3197 (
3198 OBJGPU *pGpu,
3199 KernelGraphicsContext *pKernelGraphicsContext,
3200 NvU32 classNum
3201 )
3202 {
3203 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3204 NvU32 objType;
3205 NvU32 countIdx;
3206 NV_STATUS status;
3207
3208 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
3209
3210 kgrmgrGetGrObjectType(classNum, &objType);
3211
3212 NV_ASSERT_OK_OR_ELSE(status,
3213 kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast),
3214 return;);
3215
3216 NV_ASSERT_OR_ELSE(pKernelGraphicsContextUnicast->channelObjects != 0,
3217 NV_PRINTF(LEVEL_ERROR, "No active GR objects to free for Class 0x%x\n", classNum));
3218
3219 //
3220 // Handle 2D and Compute class reference counting
3221 //
3222 switch (objType)
3223 {
3224 case GR_OBJECT_TYPE_COMPUTE:
3225 gpuChangeComputeModeRefCount(pGpu, NV_GPU_COMPUTE_REFCOUNT_COMMAND_DECREMENT);
3226 countIdx = GR_OBJECT_TYPE_COMPUTE;
3227 break;
3228
3229 case GR_OBJECT_TYPE_3D:
3230 countIdx = GR_OBJECT_TYPE_3D;
3231 break;
3232
3233 case GR_OBJECT_TYPE_2D:
3234 countIdx = GR_OBJECT_TYPE_2D;
3235 break;
3236
3237 case GR_OBJECT_TYPE_MEM:
3238 countIdx = GR_OBJECT_TYPE_MEM;
3239 break;
3240
3241 default:
3242 NV_PRINTF(LEVEL_ERROR, "Unrecognized graphics class 0x%x\n",
3243 classNum);
3244 DBG_BREAKPOINT();
3245 return;
3246 }
3247
3248 NV_ASSERT_OR_RETURN_VOID(pKernelGraphicsContextUnicast->objectCounts[countIdx] > 0);
3249 pKernelGraphicsContextUnicast->objectCounts[countIdx]--;
3250 pKernelGraphicsContextUnicast->channelObjects--;
3251 }
3252
3253 /*!
3254 * @brief Return the register access buffer used for a given Channel's permission
3255 *
3256 * The base register access map comes from the architecture team. The user version
3257 * of the buffer removes access to PM regisers.
3258 *
3259 * The policy currently depends on administrator access on the system, except on
3260 * one VGPU configuration.
3261 */
3262 GR_GLOBALCTX_BUFFER
kgrctxGetRegisterAccessMapId_IMPL(OBJGPU * pGpu,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel)3263 kgrctxGetRegisterAccessMapId_IMPL
3264 (
3265 OBJGPU *pGpu,
3266 KernelGraphicsContext *pKernelGraphicsContext,
3267 KernelChannel *pKernelChannel
3268 )
3269 {
3270 // Using cached privilege because this function is called at a raised IRQL.
3271 if (kchannelCheckIsAdmin(pKernelChannel)
3272 && !hypervisorIsVgxHyper() && IS_GFID_PF(kchannelGetGfid(pKernelChannel)))
3273 {
3274 return GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP;
3275 }
3276
3277 return GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP;
3278 }
3279
3280 NV_STATUS
kgrctxCtrlGetTpcPartitionMode_IMPL(KernelGraphicsContext * pKernelGraphicsContext,NV0090_CTRL_TPC_PARTITION_MODE_PARAMS * pParams)3281 kgrctxCtrlGetTpcPartitionMode_IMPL
3282 (
3283 KernelGraphicsContext *pKernelGraphicsContext,
3284 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams
3285 )
3286 {
3287 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext);
3288
3289 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
3290
3291 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
3292 {
3293 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
3294 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
3295 NV_STATUS status = NV_OK;
3296
3297 NV_RM_RPC_CONTROL(pGpu,
3298 pRmCtrlParams->hClient,
3299 pRmCtrlParams->hObject,
3300 pRmCtrlParams->cmd,
3301 pRmCtrlParams->pParams,
3302 pRmCtrlParams->paramsSize,
3303 status);
3304
3305 return status;
3306 }
3307
3308 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource),
3309 NV0090_CTRL_CMD_INTERNAL_GET_TPC_PARTITION_MODE,
3310 pParams,
3311 sizeof(*pParams));
3312 }
3313
3314 NV_STATUS
kgrctxCtrlSetTpcPartitionMode_IMPL(KernelGraphicsContext * pKernelGraphicsContext,NV0090_CTRL_TPC_PARTITION_MODE_PARAMS * pParams)3315 kgrctxCtrlSetTpcPartitionMode_IMPL
3316 (
3317 KernelGraphicsContext *pKernelGraphicsContext,
3318 NV0090_CTRL_TPC_PARTITION_MODE_PARAMS *pParams
3319 )
3320 {
3321 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext);
3322
3323 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
3324
3325 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
3326 {
3327 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
3328 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
3329 NV_STATUS status = NV_OK;
3330
3331 NV_RM_RPC_CONTROL(pGpu,
3332 pRmCtrlParams->hClient,
3333 pRmCtrlParams->hObject,
3334 pRmCtrlParams->cmd,
3335 pRmCtrlParams->pParams,
3336 pRmCtrlParams->paramsSize,
3337 status);
3338
3339 return status;
3340 }
3341
3342 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource),
3343 NV0090_CTRL_CMD_INTERNAL_SET_TPC_PARTITION_MODE,
3344 pParams,
3345 sizeof(*pParams));
3346 }
3347
3348 NV_STATUS
kgrctxCtrlGetMMUDebugMode_IMPL(KernelGraphicsContext * pKernelGraphicsContext,NV0090_CTRL_GET_MMU_DEBUG_MODE_PARAMS * pParams)3349 kgrctxCtrlGetMMUDebugMode_IMPL
3350 (
3351 KernelGraphicsContext *pKernelGraphicsContext,
3352 NV0090_CTRL_GET_MMU_DEBUG_MODE_PARAMS *pParams
3353 )
3354 {
3355 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext);
3356
3357 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
3358
3359 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
3360 {
3361 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
3362 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
3363 NV_STATUS status = NV_OK;
3364
3365 NV_RM_RPC_CONTROL(pGpu,
3366 pRmCtrlParams->hClient,
3367 pRmCtrlParams->hObject,
3368 pRmCtrlParams->cmd,
3369 pRmCtrlParams->pParams,
3370 pRmCtrlParams->paramsSize,
3371 status);
3372
3373 return status;
3374 }
3375
3376 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource),
3377 NV0090_CTRL_CMD_INTERNAL_GET_MMU_DEBUG_MODE,
3378 pParams,
3379 sizeof(*pParams));
3380 }
3381
3382 NV_STATUS
kgrctxCtrlProgramVidmemPromote_IMPL(KernelGraphicsContext * pKernelGraphicsContext,NV0090_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS * pParams)3383 kgrctxCtrlProgramVidmemPromote_IMPL
3384 (
3385 KernelGraphicsContext *pKernelGraphicsContext,
3386 NV0090_CTRL_PROGRAM_VIDMEM_PROMOTE_PARAMS *pParams
3387 )
3388 {
3389 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelGraphicsContext);
3390
3391 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
3392
3393 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))
3394 {
3395 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
3396 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
3397 NV_STATUS status = NV_OK;
3398
3399 NV_RM_RPC_CONTROL(pGpu,
3400 pRmCtrlParams->hClient,
3401 pRmCtrlParams->hObject,
3402 pRmCtrlParams->cmd,
3403 pRmCtrlParams->pParams,
3404 pRmCtrlParams->paramsSize,
3405 status);
3406
3407 return status;
3408 }
3409
3410 return gpuresInternalControlForward_IMPL(staticCast(pKernelGraphicsContext, GpuResource),
3411 NV0090_CTRL_CMD_INTERNAL_PROGRAM_VIDMEM_PROMOTE,
3412 pParams,
3413 sizeof(*pParams));
3414 }
3415
3416 /*!
3417 * @brief Construct shared kernel graphics context. (Does nothing)
3418 */
3419 NV_STATUS
shrkgrctxConstruct_IMPL(KernelGraphicsContextShared * pKernelGraphicsContextShared)3420 shrkgrctxConstruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared)
3421 {
3422 return NV_OK;
3423 }
3424
3425 /*!
3426 * @brief Construct shared kernel graphics context
3427 *
3428 * @param[in] pGpu
3429 * @param[in] pKernelGraphicsContextShared
3430 * @param[in] pKernelGraphicsContext
3431 */
3432 NV_STATUS
shrkgrctxInit_IMPL(OBJGPU * pGpu,KernelGraphicsContextShared * pKernelGraphicsContextShared,KernelGraphicsContext * pKernelGraphicsContext)3433 shrkgrctxInit_IMPL
3434 (
3435 OBJGPU *pGpu,
3436 KernelGraphicsContextShared *pKernelGraphicsContextShared,
3437 KernelGraphicsContext *pKernelGraphicsContext
3438 )
3439 {
3440 NV_STATUS status = NV_OK;
3441
3442 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
3443 {
3444 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3445 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
3446 KernelGraphics *pKernelGraphics;
3447 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo;
3448 Device *pDevice = GPU_RES_GET_DEVICE(pKernelGraphicsContext);
3449 NvHandle hParent = RES_GET_PARENT_HANDLE(pKernelGraphicsContext);
3450
3451 portMemSet(&grRouteInfo, 0, sizeof(grRouteInfo));
3452 kgrmgrCtrlSetChannelHandle(hParent, &grRouteInfo);
3453
3454 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR,
3455 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics));
3456
3457 if (status != NV_OK)
3458 SLI_LOOP_BREAK;
3459
3460 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK)
3461 {
3462 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR,
3463 shrkgrctxConstructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphics, pKernelGraphicsContextUnicast));
3464
3465 if (status != NV_OK)
3466 SLI_LOOP_BREAK;
3467 }
3468 }
3469 SLI_LOOP_END;
3470
3471 listInit(&pKernelGraphicsContextShared->activeDebuggers, portMemAllocatorGetGlobalNonPaged());
3472
3473 if (status != NV_OK)
3474 shrkgrctxTeardown_IMPL(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext);
3475
3476 return status;
3477 }
3478
3479 /*!
3480 * @brief Construct unicast state for this context
3481 *
3482 * @param[in] pGpu
3483 * @param[in] pKernelGraphicsContextShared
3484 * @param[in] pKernelGraphicsContext
3485 * @param[in] pKernelGraphics
3486 * @param[in] pKernelGraphicsContextUnicast
3487 */
3488 NV_STATUS
shrkgrctxConstructUnicast_IMPL(OBJGPU * pGpu,KernelGraphicsContextShared * pKernelGraphicsContextShared,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphics * pKernelGraphics,KernelGraphicsContextUnicast * pKernelGraphicsContextUnicast)3489 shrkgrctxConstructUnicast_IMPL
3490 (
3491 OBJGPU *pGpu,
3492 KernelGraphicsContextShared *pKernelGraphicsContextShared,
3493 KernelGraphicsContext *pKernelGraphicsContext,
3494 KernelGraphics *pKernelGraphics,
3495 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast
3496 )
3497 {
3498 NvU32 i;
3499
3500 for (i = 0; i < GR_GLOBALCTX_BUFFER_COUNT; i++)
3501 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[i]));
3502
3503 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList));
3504 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList));
3505 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList));
3506
3507 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList));
3508 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList));
3509 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList));
3510 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList));
3511 NV_ASSERT_OK_OR_RETURN(vaListInit(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList));
3512
3513 pKernelGraphicsContextUnicast->bSupportsPerSubctxHeader =
3514 kgraphicsIsPerSubcontextContextHeaderSupported(pGpu, pKernelGraphics);
3515
3516 return NV_OK;
3517 }
3518
3519 /*!
3520 * @brief Destruct shared kernel graphics context.
3521 */
3522 void
shrkgrctxDestruct_IMPL(KernelGraphicsContextShared * pKernelGraphicsContextShared)3523 shrkgrctxDestruct_IMPL(KernelGraphicsContextShared *pKernelGraphicsContextShared)
3524 {
3525 NV_ASSERT(listCount(&pKernelGraphicsContextShared->activeDebuggers) == 0);
3526 listDestroy(&pKernelGraphicsContextShared->activeDebuggers);
3527 }
3528
3529 /*!
3530 * @brief Destroy the shared context state
3531 */
3532 void
shrkgrctxTeardown_IMPL(OBJGPU * pGpu,KernelGraphicsContextShared * pKernelGraphicsContextShared,KernelGraphicsContext * pKernelGraphicsContext)3533 shrkgrctxTeardown_IMPL
3534 (
3535 OBJGPU *pGpu,
3536 KernelGraphicsContextShared *pKernelGraphicsContextShared,
3537 KernelGraphicsContext *pKernelGraphicsContext
3538 )
3539 {
3540
3541 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
3542 {
3543 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3544
3545 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK)
3546 shrkgrctxDestructUnicast(pGpu, pKernelGraphicsContextShared, pKernelGraphicsContext, pKernelGraphicsContextUnicast);
3547 }
3548 SLI_LOOP_END;
3549
3550 portMemSet(&pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast, 0x0, sizeof(pKernelGraphicsContext->pShared->kernelGraphicsContextUnicast));
3551 }
3552
3553 /*!
3554 * Destroy the unicast context state
3555 */
3556 void
shrkgrctxDestructUnicast_IMPL(OBJGPU * pGpu,KernelGraphicsContextShared * pKernelGraphicsContextShared,KernelGraphicsContext * pKernelGraphicsContext,KernelGraphicsContextUnicast * pKernelGraphicsContextUnicast)3557 shrkgrctxDestructUnicast_IMPL
3558 (
3559 OBJGPU *pGpu,
3560 KernelGraphicsContextShared *pKernelGraphicsContextShared,
3561 KernelGraphicsContext *pKernelGraphicsContext,
3562 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast
3563 )
3564 {
3565 GR_GLOBALCTX_BUFFER buff;
3566
3567 if (kgrctxShouldCleanup_HAL(pGpu, pKernelGraphicsContext))
3568 kgrctxFreeAssociatedCtxBuffers(pGpu, pKernelGraphicsContext);
3569
3570 FOR_EACH_IN_ENUM(GR_GLOBALCTX_BUFFER, buff)
3571 {
3572 vaListDestroy(&pKernelGraphicsContextUnicast->globalCtxBufferVaList[buff]);
3573 }
3574 FOR_EACH_IN_ENUM_END;
3575 vaListDestroy(&pKernelGraphicsContextUnicast->ctxPatchBuffer.vAddrList);
3576 vaListDestroy(&pKernelGraphicsContextUnicast->pmCtxswBuffer.vAddrList);
3577 vaListDestroy(&pKernelGraphicsContextUnicast->zcullCtxswBuffer.vAddrList);
3578
3579 vaListDestroy(&pKernelGraphicsContextUnicast->preemptCtxswBuffer.vAddrList);
3580 vaListDestroy(&pKernelGraphicsContextUnicast->spillCtxswBuffer.vAddrList);
3581 vaListDestroy(&pKernelGraphicsContextUnicast->betaCBCtxswBuffer.vAddrList);
3582 vaListDestroy(&pKernelGraphicsContextUnicast->pagepoolCtxswBuffer.vAddrList);
3583 vaListDestroy(&pKernelGraphicsContextUnicast->rtvCbCtxswBuffer.vAddrList);
3584 }
3585
3586 /*!
3587 * @brief Perform cleanup tasks run on channel removal from context
3588 */
shrkgrctxDetach_IMPL(OBJGPU * pGpu,KernelGraphicsContextShared * pKernelGraphicsContextShared,KernelGraphicsContext * pKernelGraphicsContext,KernelChannel * pKernelChannel)3589 void shrkgrctxDetach_IMPL
3590 (
3591 OBJGPU *pGpu,
3592 KernelGraphicsContextShared *pKernelGraphicsContextShared,
3593 KernelGraphicsContext *pKernelGraphicsContext,
3594 KernelChannel *pKernelChannel
3595 )
3596 {
3597 RM_ENGINE_TYPE rmEngineType = kchannelGetEngineType(pKernelChannel);
3598
3599 // pre-Ampere chips can have NULL engine types. Find the engine type based on runlistId if set
3600 if (!RM_ENGINE_TYPE_IS_VALID(rmEngineType))
3601 {
3602 if (kchannelIsRunlistSet(pGpu, pKernelChannel))
3603 {
3604 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu);
3605
3606 NV_ASSERT_OK(
3607 kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo,
3608 ENGINE_INFO_TYPE_RUNLIST, kchannelGetRunlistId(pKernelChannel),
3609 ENGINE_INFO_TYPE_RM_ENGINE_TYPE, (NvU32 *)&rmEngineType));
3610 }
3611 else
3612 {
3613 NV_PRINTF(LEVEL_INFO, "Channel destroyed but never bound, scheduled, or had a descendant object created\n");
3614 }
3615 }
3616
3617 if (RM_ENGINE_TYPE_IS_GR(rmEngineType))
3618 {
3619 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelChannel);
3620 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu);
3621 Device *pDevice = GPU_RES_GET_DEVICE(pKernelChannel);
3622 NV2080_CTRL_GR_ROUTE_INFO grRouteInfo = {0};
3623 KernelGraphics *pKernelGraphics;
3624 NV_STATUS status;
3625
3626 kgrmgrCtrlSetChannelHandle(RES_GET_HANDLE(pKernelChannel), &grRouteInfo);
3627 NV_ASSERT_OK_OR_ELSE(status,
3628 kgrmgrCtrlRouteKGRWithDevice(pGpu, pKernelGraphicsManager, pDevice, &grRouteInfo, &pKernelGraphics),
3629 return;);
3630
3631 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
3632 {
3633 // Last chance to process FECS ctxsw log before channel goes away
3634 nvEventBufferFecsCallback(pGpu, pKernelGraphics);
3635 }
3636 SLI_LOOP_END
3637 }
3638
3639 //
3640 // If PDB_PROP_GPU_CLIENT_RM_ALLOCATED_CTX_BUFFER is set, then kernel RM
3641 // is responsible for unmapping the context buffers, otherwise it is left to
3642 // physical RM.
3643 //
3644 if (!kgrctxShouldCleanup(pGpu, pKernelGraphicsContext))
3645 return;
3646
3647 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
3648 {
3649 KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
3650
3651 if (kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast) == NV_OK)
3652 kgrctxUnmapBuffers_HAL(pGpu, pKernelGraphicsContext, pKernelGraphicsContextUnicast, pKernelChannel);
3653 }
3654 SLI_LOOP_END;
3655 }
3656
3657