1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "kernel/gpu/gr/kernel_graphics.h"
25 #include "kernel/gpu/mem_mgr/mem_mgr.h"
26 
27 #include "ctrl/ctrl0080/ctrl0080fifo.h"
28 
29 /*!
30  * @brief Allocate common buffers that are required by the graphics context
31  */
32 NV_STATUS
33 kgraphicsAllocGrGlobalCtxBuffers_GM200
34 (
35     OBJGPU *pGpu,
36     KernelGraphics *pKernelGraphics,
37     NvU32 gfid,
38     KernelGraphicsContext *pKernelGraphicsContext
39 )
40 {
41     MEMORY_DESCRIPTOR           **ppMemDesc;
42     GR_GLOBALCTX_BUFFERS         *pCtxBuffers;
43     GR_BUFFER_ATTR               *pCtxAttr;
44     NvU64                         cbAllocFlags;
45     NvBool                        bPhysicallyContiguous;
46     NvU64                         flags = MEMDESC_FLAGS_NONE;
47     NvU32                         circularBufferSize;
48     NvU32                         circularBufferAlign;
49     NvU32                         pagepoolBufferSize;
50     NvU32                         pagepoolBufferAlign;
51     NvU32                         attribBufferSize;
52     NvU32                         attribBufferAlign;
53     NvU32                         privMapBufferSize;
54     NvU32                         privMapBufferAlign;
55     NvU32                         unresPrivMapBufferSize;
56     NvU32                         unresPrivMapBufferAlign;
57     MemoryManager                *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
58     CTX_BUF_POOL_INFO            *pCtxBufPool;
59     const KGRAPHICS_STATIC_INFO  *pKernelGraphicsStaticInfo;
60 
61     NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE);
62 
63     pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics);
64     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE);
65     NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pContextBuffersInfo != NULL, NV_ERR_INVALID_STATE);
66 
67     circularBufferSize =
68         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB].size;
69     circularBufferAlign =
70         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_BUNDLE_CB].alignment;
71 
72     pagepoolBufferSize =
73         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL].size;
74     pagepoolBufferAlign =
75         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PAGEPOOL_GLOBAL].alignment;
76 
77     attribBufferSize =
78         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB].size;
79     attribBufferAlign =
80         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_ATTRIBUTE_CB].alignment;
81 
82     privMapBufferSize =
83         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP].size;
84     privMapBufferAlign =
85         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP].alignment;
86 
87     unresPrivMapBufferSize =
88         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP].size;
89     unresPrivMapBufferAlign =
90         pKernelGraphicsStaticInfo->pContextBuffersInfo->engine[NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_GRAPHICS_PRIV_ACCESS_MAP].alignment;
91 
92     // Setup the Circular Buffer DB
93     cbAllocFlags = MEMDESC_FLAGS_LOST_ON_SUSPEND;
94 
95     if (kgraphicsShouldSetContextBuffersGPUPrivileged(pGpu, pKernelGraphics))
96     {
97         cbAllocFlags |= MEMDESC_FLAGS_GPU_PRIVILEGED;
98     }
99 
100     pCtxBufPool = NULL;
101     if (pKernelGraphicsContext != NULL)
102     {
103         KernelGraphicsContextUnicast *pKernelGraphicsContextUnicast;
104 
105         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
106             kgrctxGetUnicast(pGpu, pKernelGraphicsContext, &pKernelGraphicsContextUnicast));
107 
108         pCtxBuffers = &pKernelGraphicsContextUnicast->localCtxBuffer;
109         pCtxAttr = pKernelGraphics->globalCtxBuffersInfo.localCtxAttr;
110 
111         //
112         // if we already have local buffers allocated, return as we may
113         // get called multiple times per-channel
114         //
115         if (pCtxBuffers->bAllocated)
116             return NV_OK;
117 
118         // check for allocating local buffers in VPR memory (don't want for global memory)
119         if (
120             pKernelGraphicsContextUnicast->bVprChannel)
121             cbAllocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED;
122 
123         // If allocated per channel, ensure allocations goes into Suballocator if available
124         cbAllocFlags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
125     }
126     else
127     {
128         pCtxBuffers = &pKernelGraphics->globalCtxBuffersInfo.pGlobalCtxBuffers[gfid];
129         pCtxAttr = pKernelGraphics->globalCtxBuffersInfo.globalCtxAttr;
130         NV_ASSERT_OK_OR_RETURN(
131             ctxBufPoolGetGlobalPool(pGpu, CTX_BUF_ID_GR_GLOBAL,
132                                     RM_ENGINE_TYPE_GR(pKernelGraphics->instance),
133                                     &pCtxBufPool));
134     }
135 
136     // Handle VF - must use VF attributes and flags for both global and local buffers
137     if (IS_GFID_VF(gfid))
138     {
139         pCtxAttr = pKernelGraphics->globalCtxBuffersInfo.vfGlobalCtxAttr;
140 
141         cbAllocFlags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
142         flags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
143     }
144 
145     // Don't use context buffer pool for VF allocations managed by host RM.
146     if (ctxBufPoolIsSupported(pGpu) && (pCtxBufPool != NULL))
147     {
148         cbAllocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
149         flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
150     }
151 
152     // Circular Buffer
153     if (circularBufferSize > 0)
154     {
155         ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_BUNDLE_CB];
156         bPhysicallyContiguous = pCtxAttr[GR_GLOBALCTX_BUFFER_BUNDLE_CB].pAllocList == ADDRLIST_FBMEM_ONLY;
157 
158         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
159             memdescCreate(ppMemDesc, pGpu,
160                           circularBufferSize,
161                           circularBufferAlign,
162                           bPhysicallyContiguous,
163                           ADDR_UNKNOWN,
164                           pCtxAttr[GR_GLOBALCTX_BUFFER_BUNDLE_CB].cpuAttr,
165                           cbAllocFlags | MEMDESC_FLAGS_GPU_PRIVILEGED | MEMDESC_FLAGS_HIGH_PRIORITY));
166 
167         memdescSetGpuCacheAttrib(*ppMemDesc, NV_MEMORY_CACHED);
168         if ((cbAllocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
169         {
170             memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
171             NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
172         }
173         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
174             memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_BUNDLE_CB].pAllocList));
175     }
176 
177     // Page Pool
178     if (pagepoolBufferSize > 0)
179     {
180         ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_PAGEPOOL];
181         bPhysicallyContiguous = pCtxAttr[GR_GLOBALCTX_BUFFER_PAGEPOOL].pAllocList == ADDRLIST_FBMEM_ONLY;
182 
183         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
184             memdescCreate(ppMemDesc, pGpu,
185                           pagepoolBufferSize,
186                           pagepoolBufferAlign,
187                           bPhysicallyContiguous,
188                           ADDR_UNKNOWN,
189                           pCtxAttr[GR_GLOBALCTX_BUFFER_PAGEPOOL].cpuAttr,
190                           cbAllocFlags | MEMDESC_FLAGS_GPU_PRIVILEGED));
191 
192         memdescSetGpuCacheAttrib(*ppMemDesc, NV_MEMORY_CACHED);
193         if ((cbAllocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
194         {
195             memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
196             NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
197         }
198         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
199             memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_PAGEPOOL].pAllocList));
200     }
201 
202     // Attribute Buffer
203     if (attribBufferSize > 0)
204     {
205         ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB];
206         bPhysicallyContiguous = pCtxAttr[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB].pAllocList == ADDRLIST_FBMEM_ONLY;
207 
208         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
209             memdescCreate(ppMemDesc, pGpu,
210                           attribBufferSize,
211                           attribBufferAlign,
212                           bPhysicallyContiguous,
213                           ADDR_UNKNOWN,
214                           pCtxAttr[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB].cpuAttr,
215                           cbAllocFlags | MEMDESC_FLAGS_HIGH_PRIORITY));
216 
217         memdescSetGpuCacheAttrib(*ppMemDesc, NV_MEMORY_CACHED);
218         if ((cbAllocFlags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
219         {
220             memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
221             NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
222         }
223         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
224             memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_ATTRIBUTE_CB].pAllocList));
225         memdescSetName(pGpu, *ppMemDesc, NV_RM_SURF_NAME_GR_CIRCULAR_BUFFER, NULL);
226     }
227 
228     // we do not want/need a priv access map allocated per-channel, so skip allocating
229     if (pKernelGraphicsContext == NULL)
230     {
231         if (kgraphicsDoesUcodeSupportPrivAccessMap(pGpu, pKernelGraphics))
232         {
233             NvBool bIsContiguous = kgraphicsShouldForceMainCtxContiguity_HAL(pGpu, pKernelGraphics) && gpuIsClientRmAllocatedCtxBufferEnabled(pGpu);
234 
235             // PRIV access map
236             if (privMapBufferSize > 0)
237             {
238                 ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP];
239 
240                 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
241                     memdescCreate(ppMemDesc, pGpu,
242                                   privMapBufferSize,
243                                   privMapBufferAlign,
244                                   bIsContiguous,
245                                   ADDR_UNKNOWN,
246                                   pCtxAttr[GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP].cpuAttr,
247                                   flags));
248 
249                 if ((flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
250                 {
251                     //
252                     // Force page size to 4KB, we can change this later when RM
253                     // access method support 64k pages
254                     //
255                     memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
256                     NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
257                 }
258                 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
259                     memdescAllocList(*ppMemDesc, pCtxAttr[GR_GLOBALCTX_BUFFER_PRIV_ACCESS_MAP].pAllocList));
260             }
261 
262             //
263             // vGPU does not support unrestricted priv access map buffer. Hence, avoid
264             // allocating it on vGPU configuration.
265             //
266             if ((unresPrivMapBufferSize > 0) && kgraphicsIsUnrestrictedAccessMapSupported_HAL(pGpu, pKernelGraphics))
267             {
268                 // Unrestricted PRIV access map
269                 ppMemDesc = &pCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP];
270 
271                 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
272                     memdescCreate(ppMemDesc, pGpu,
273                                   unresPrivMapBufferSize,
274                                   unresPrivMapBufferAlign,
275                                   bIsContiguous,
276                                   ADDR_UNKNOWN,
277                                   pCtxAttr[GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP].cpuAttr,
278                                   flags));
279 
280                 if ((flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL) != 0)
281                 {
282                     //
283                     // Force page size to 4KB, we can change this later when RM
284                     // access method support 64k pages
285                     //
286                     memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB);
287                     NV_ASSERT_OK_OR_RETURN(memdescSetCtxBufPool(*ppMemDesc, pCtxBufPool));
288                 }
289 
290                 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
291                     memdescAllocList(*ppMemDesc,
292                                      pCtxAttr[GR_GLOBALCTX_BUFFER_UNRESTRICTED_PRIV_ACCESS_MAP].pAllocList));
293             }
294         }
295     }
296 
297     pCtxBuffers->bAllocated = NV_TRUE;
298 
299     return NV_OK;
300 }
301 
302