1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "nvtypes.h"
27 #include "os/os.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_desc.h"
30 #include "gpu/bus/kern_bus.h"
31 #include "kernel/gpu/intr/intr.h"
32 #include "nverror.h"
33 
34 #include "published/hopper/gh100/dev_fb.h"
35 #include "published/hopper/gh100/dev_fbpa.h"
36 #include "published/hopper/gh100/dev_vm.h"
37 #include "published/hopper/gh100/pri_nv_xal_ep.h"
38 #include "published/hopper/gh100/dev_nv_xal_addendum.h"
39 #include "published/hopper/gh100/dev_nv_xpl.h"
40 #include "published/hopper/gh100/dev_xtl_ep_pri.h"
41 #include "published/hopper/gh100/hwproject.h"
42 #include "published/ampere/ga100/dev_fb.h"
43 
44 NV_STATUS
kmemsysDoCacheOp_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 reg,NvU32 regValue,NvU32 pollMask,PRMTIMEOUT pTimeout)45 kmemsysDoCacheOp_GH100
46 (
47     OBJGPU             *pGpu,
48     KernelMemorySystem *pKernelMemorySystem,
49     NvU32               reg,
50     NvU32               regValue,
51     NvU32               pollMask,
52     PRMTIMEOUT          pTimeout
53 )
54 {
55     NV_STATUS  rmStatus = NV_OK;
56     NvU32      cnt = 0;
57     NV_STATUS  timeoutStatus = NV_OK;
58     NvU32      tokenRangeMask = 0;
59     NvU32      startToken = 0;
60     NvU32      completedToken = 0;
61     NvBool     bMemopBusy = NV_TRUE;
62 
63     if (!API_GPU_ATTACHED_SANITY_CHECK(pGpu))
64     {
65         //
66         // When the GPU is lost we cannot expect to successfully do cache
67         // maintenance (see Bug 1557278).
68         //
69         return rmStatus;
70     }
71 
72     if (IS_VIRTUAL(pGpu))
73     {
74         switch (reg)
75         {
76             case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
77             case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
78                 break;
79             case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
80                 return NV_OK;
81             default:
82                 return NV_ERR_NOT_SUPPORTED;
83         }
84     }
85 
86     switch (reg)
87     {
88         case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
89             tokenRangeMask = DRF_MASK(NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY_TOKEN);
90 
91             startToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY);
92             startToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_FLUSH_DIRTY, _TOKEN, startToken);
93             break;
94 
95         case NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS:
96             tokenRangeMask = DRF_MASK(NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS_TOKEN);
97 
98             startToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS);
99             startToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS, _TOKEN, startToken);
100             break;
101 
102         case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
103             tokenRangeMask = DRF_MASK(NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE_TOKEN);
104 
105             startToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE);
106             startToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE, _TOKEN, startToken);
107             break;
108 
109         case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
110             tokenRangeMask = DRF_MASK(NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE_TOKEN);
111 
112             startToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE);
113             startToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE, _TOKEN, startToken);
114             break;
115 
116         default:
117             return NV_ERR_NOT_SUPPORTED;
118     }
119 
120     while(1)
121     {
122         switch (reg)
123         {
124             case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
125                 completedToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY_COMPLETED);
126 
127                 bMemopBusy = FLD_TEST_DRF(_XAL_EP_UFLUSH, _L2_FLUSH_DIRTY_COMPLETED, _STATUS, _BUSY, completedToken)
128                              ? NV_TRUE : NV_FALSE;
129 
130                 completedToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_FLUSH_DIRTY_COMPLETED, _TOKEN, completedToken);
131                 break;
132 
133             case NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS:
134                 completedToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS_COMPLETED);
135 
136                 bMemopBusy = FLD_TEST_DRF(_XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS_COMPLETED, _STATUS, _BUSY, completedToken)
137                              ? NV_TRUE : NV_FALSE;
138 
139                 completedToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS_COMPLETED, _TOKEN, completedToken);
140                 break;
141 
142             case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
143                 completedToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED);
144 
145                 bMemopBusy = FLD_TEST_DRF( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED, _STATUS, _BUSY, completedToken)
146                              ? NV_TRUE : NV_FALSE;
147 
148                 completedToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED, _TOKEN, completedToken);
149                 break;
150 
151             case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
152                 completedToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED);
153 
154                 bMemopBusy = FLD_TEST_DRF( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED, _STATUS, _BUSY, completedToken)
155                              ? NV_TRUE : NV_FALSE;
156 
157                 completedToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED, _TOKEN, completedToken);
158                 break;
159         }
160 
161         if (bMemopBusy == NV_FALSE)
162         {
163             break;
164         }
165 
166         //
167         // When completedToken > startToken(including the wrapping around case), due to the nature
168         // of unsigned number, the value of "(startToken - completeToken) & tokenRangeMask" will be
169         // at top of token range which will be bigger than NV_XAL_EP_MEMOP_MAX_OUTSTANDING. So it
170         // will break out from the loop.
171         //
172         // The loop will wait only when completedToken in the range of
173         // [startToken-NV_XAL_EP_MEMOP_MAX_OUTSTANDING, startToken].
174         //
175         if (((startToken - completedToken) & tokenRangeMask) > NV_XAL_EP_MEMOP_MAX_OUTSTANDING)
176         {
177             break;
178         }
179 
180         if (timeoutStatus == NV_ERR_TIMEOUT)
181         {
182             //
183             // This should not timeout, except for a HW bug.  Famous last words.
184             // On !DEBUG we just keep trucking, it's the best we can do.
185             //
186             NV_PRINTF(LEVEL_ERROR,
187                       "- timeout error waiting for reg 0x%x update cnt=%d\n",
188                       reg, cnt);
189             rmStatus = NV_ERR_TIMEOUT;
190             DBG_BREAKPOINT();
191             break;
192         }
193         else if ( API_GPU_IN_RESET_SANITY_CHECK(pGpu) ||
194                  !API_GPU_ATTACHED_SANITY_CHECK(pGpu))
195         {
196             //
197             // The GPU is in full chip reset, or has fallen off the bus
198             // Just return
199             //
200             return NV_OK;
201         }
202 
203         timeoutStatus = gpuCheckTimeout(pGpu, pTimeout);
204         osSpinLoop();
205         cnt++;
206     }
207 
208 #ifdef DEBUG
209     if (cnt > 1)
210     {
211         NV_ASSERT(kmemsysAssertFbAckTimeoutPending_HAL(pGpu, pKernelMemorySystem) == NV_FALSE);
212     }
213 #endif // DEBUG
214 
215     return rmStatus;
216 }
217 
218 /**
219  * @brief L2 Cache management OPs
220  *
221  * GK20A/T124 have a writeback L2 cache, so the cache ops are slightly
222  * different than those of Fermi/Kepler.  Specifically, we can write back dirty
223  * lines to system memory.
224  *
225  * @param pMemDesc
226  * @param targetMem
227  * @param cacheOp
228  */
229 NV_STATUS
kmemsysCacheOp_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,PMEMORY_DESCRIPTOR pMemDesc,FB_CACHE_MEMTYPE targetMem,FB_CACHE_OP cacheOp)230 kmemsysCacheOp_GH100
231 (
232     OBJGPU             *pGpu,
233     KernelMemorySystem *pKernelMemorySystem,
234     PMEMORY_DESCRIPTOR  pMemDesc,
235     FB_CACHE_MEMTYPE    targetMem,
236     FB_CACHE_OP         cacheOp
237 )
238 {
239     NV_STATUS status  = NV_OK;
240     RMTIMEOUT timeout = {0, };
241     NvU32     reg = 0;
242 
243     if ((targetMem == FB_CACHE_MEM_UNDEFINED) && pMemDesc)
244     {
245         targetMem = (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) ?
246                         FB_CACHE_VIDEO_MEMORY : FB_CACHE_SYSTEM_MEMORY;
247     }
248 
249     if ((cacheOp == FB_CACHE_OP_UNDEFINED) || (targetMem == FB_CACHE_MEM_UNDEFINED))
250     {
251         NV_PRINTF(LEVEL_ERROR, "called with null %s\n",
252                   cacheOp ? "cache operation" : "memory target");
253         DBG_BREAKPOINT();
254         return status;  // return NV_OK
255     }
256 
257     // For GK20A, an explicit sysmembar flush is needed before L2 cache flush operation.
258     // Refer GK20A LTC IAS (section 5.5)
259     kbusSendSysmembarSingle_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu));
260 
261     // Wait for the flush to flow through
262     gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
263 
264     switch (targetMem)
265     {
266         case FB_CACHE_SYSTEM_MEMORY:
267             if (cacheOp == FB_CACHE_INVALIDATE)
268             {
269                 NV_PRINTF(LEVEL_INFO,
270                           "Invalidate not supported, promoting to an evict (writeback + "
271                           "invalidate clean lines).\n");
272                 cacheOp = FB_CACHE_EVICT;
273             }
274 
275             if (cacheOp == FB_CACHE_WRITEBACK || cacheOp == FB_CACHE_EVICT)
276             {
277                 reg =  NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY;
278                 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
279             }
280 
281             if (cacheOp == FB_CACHE_EVICT)
282             {
283                 //
284                 // Ideally we should use NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES.
285                 // But NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES is not defined on desktop GPUs.
286                 // NV_UFLUSH_L2_SYSMEM_INVALIDATE is same as NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES, and is defined in all chips.
287                 // For Hopper, it is changed to NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE
288                 //
289                 reg =  NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE;
290                 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
291             }
292             break;
293 
294         case FB_CACHE_VIDEO_MEMORY:
295             // Fbmem cache ops are not supported from VF -- force NV_OK for MODS
296             if (IS_VIRTUAL(pGpu))
297                 return NV_OK;
298 
299             if (cacheOp == FB_CACHE_EVICT)
300             {
301                 NvU32 flags = NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_ALL |
302                               NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_CLEAN;
303                 status = kmemsysSendL2InvalidateEvict(pGpu, pKernelMemorySystem, flags);
304             }
305             else if (cacheOp == FB_CACHE_INVALIDATE)
306             {
307                 NvU32 flags = NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_ALL;
308                 status = kmemsysSendL2InvalidateEvict(pGpu, pKernelMemorySystem, flags);
309             }
310             else
311             {
312                 status = NV_ERR_NOT_SUPPORTED;
313             }
314             break;
315 
316         case FB_CACHE_PEER_MEMORY:
317             //
318             // for GF100 - for sysmem cache only invalidate operation is supported
319             // evict = writeback+invalidate is reduced to invalidate for GF100
320             //
321             if((cacheOp != FB_CACHE_INVALIDATE) && (cacheOp != FB_CACHE_EVICT))
322             {
323                 return NV_OK;
324             }
325 
326             reg =  NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE;
327             status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
328             break;
329 
330         case FB_CACHE_COMPTAG_MEMORY:
331             if(cacheOp != FB_CACHE_WRITEBACK && cacheOp != FB_CACHE_EVICT)
332             {
333                 return NV_OK;
334             }
335 
336             //
337             // Beware of HW bug 545242. Graphics should be idle and flushed here
338             // or comp tag cache could be corrupted.  When mods uses this call
339             // during verif, this should already be the case.
340             //
341             reg =  NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS;
342             status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
343             break;
344 
345         case FB_CACHE_DIRTY:
346             if(cacheOp != FB_CACHE_WRITEBACK && cacheOp != FB_CACHE_EVICT)
347             {
348                 return NV_OK;
349             }
350 
351             reg =  NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY;
352             status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
353             break;
354 
355         case FB_CACHE_DIRTY_ALL:
356             if(cacheOp != FB_CACHE_EVICT)
357             {
358                 return NV_OK;
359             }
360 
361             status = kmemsysSendFlushL2AllRamsAndCaches(pGpu, pKernelMemorySystem);
362             break;
363 
364         default:
365             // return OK for other memory targets
366             status = NV_OK;
367             break;
368     }
369 
370     return status;
371 }
372 
373 /*!
374  * @brief Write the sysmemFlushBuffer val into the NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR register
375  *
376  * @param[in] pGpu                OBJGPU pointer
377  * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
378  *
379  * @returns void
380  */
381 void
kmemsysProgramSysmemFlushBuffer_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)382 kmemsysProgramSysmemFlushBuffer_GH100
383 (
384     OBJGPU *pGpu,
385     KernelMemorySystem *pKernelMemorySystem
386 )
387 {
388     NvU32                  alignedSysmemFlushBufferAddr   = 0x0;
389     NvU32                  alignedSysmemFlushBufferAddrHi = 0x0;
390 
391     NV_ASSERT(pKernelMemorySystem->sysmemFlushBuffer != 0);
392 
393     // alignedSysmemFlushBufferAddr will have the lower 32 bits of the buffer address
394     alignedSysmemFlushBufferAddr = NvU64_LO32(pKernelMemorySystem->sysmemFlushBuffer);
395 
396     // alignedSysmemFlushBufferAddrHi will have the upper 32 bits of the buffer address
397     alignedSysmemFlushBufferAddrHi = NvU64_HI32(pKernelMemorySystem->sysmemFlushBuffer);
398 
399     // Assert when Sysmem Flush buffer has more than 52-bit address
400     NV_ASSERT((alignedSysmemFlushBufferAddrHi & (~NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI_ADR_MASK)) == 0);
401 
402     alignedSysmemFlushBufferAddrHi &= NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI_ADR_MASK;
403 
404     GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI, _ADR, alignedSysmemFlushBufferAddrHi);
405     GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO, _ADR, alignedSysmemFlushBufferAddr);
406 }
407 
408 /*!
409  * @brief Validate the sysmemFlushBuffer val and assert
410  *
411  * @param[in] pGpu                OBJGPU pointer
412  * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
413  *
414  * @returns void
415  */
416 void
kmemsysAssertSysmemFlushBufferValid_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)417 kmemsysAssertSysmemFlushBufferValid_GH100
418 (
419     OBJGPU *pGpu,
420     KernelMemorySystem *pKernelMemorySystem
421 )
422 {
423     NV_ASSERT((GPU_REG_RD_DRF(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO, _ADR) != 0) ||
424               (GPU_REG_RD_DRF(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI, _ADR) != 0));
425 }
426 
427 /*!
428  * @brief Add GPU memory as a NUMA node.
429  *
430  * Add GPU memory as a NUMA node to the OS kernel in platforms where
431  * GPU is coherently connected to the CPU.
432  *
433  * @param[in]  pGPU                OBJGPU pointer
434  * @param[in]  pKernelMemorySystem KernelMemorySystem pointer
435  * @param[in]  swizzId             swizzId of the MIG GPU instance, 0 for full GPU instance/non-MIG.
436  * @param[in]  offset              start offset of the GPU instance within FB
437  * @param[in]  size                size of the GPU instance
438  * @param[out] numaNodeId          NUMA node id corresponding to the added @swizzId partition memory
439  *                                 when NV_OK is returned.
440  *
441  * @returns NV_OK if all is okay.  Otherwise an error-specific value.
442  *
443  */
444 NV_STATUS
kmemsysNumaAddMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU64 offset,NvU64 size,NvS32 * numaNodeId)445 kmemsysNumaAddMemory_GH100
446 (
447     OBJGPU             *pGpu,
448     KernelMemorySystem *pKernelMemorySystem,
449     NvU32               swizzId,
450     NvU64               offset,
451     NvU64               size,
452     NvS32              *numaNodeId
453 )
454 {
455     NV_STATUS status;
456     NvU64     memblockSize   = 0;
457     NvU32     lNumaNodeId;
458 
459     NV_ASSERT_OK_OR_RETURN(osNumaMemblockSize(&memblockSize));
460     NV_ASSERT_OR_RETURN(NV_IS_ALIGNED(size, memblockSize), NV_ERR_INVALID_STATE);
461 
462     if (pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse)
463     {
464         NV_PRINTF(LEVEL_ERROR, "Memory partition: %u is already in use!\n", swizzId);
465         return NV_ERR_IN_USE;
466     }
467 
468     status = osNumaAddGpuMemory(pGpu->pOsGpuInfo, offset, size, &lNumaNodeId);
469     if (status == NV_OK)
470     {
471         pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse = NV_TRUE;
472         pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset = offset;
473         pKernelMemorySystem->memPartitionNumaInfo[swizzId].size = size;
474         pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId = lNumaNodeId;
475         *numaNodeId = lNumaNodeId;
476 
477         pKernelMemorySystem->bNumaNodesAdded = NV_TRUE;
478 
479         NV_PRINTF(LEVEL_INFO, "Memory partition: %u added successfully!"
480                   " numa id: %u offset: 0x%llx size: 0x%llx\n",
481                   swizzId, lNumaNodeId, offset, size);
482     }
483 
484     return status;
485 }
486 
487 /*!
488  * @brief Remove a particular MIG GPU instance GPU memory from OS kernel.
489  *
490  * Remove GPU memory from the OS kernel that is earlier added as a NUMA node
491  * to the kernel in platforms where GPU is coherently connected to the CPU.
492  *
493  * @param[in]  pGPU                OBJGPU pointer
494  * @param[in]  pKernelMemorySystem KernelMemorySystem pointer
495  * @param[in]  swizzId             swizzId of the MIG GPU instance, 0 for full partition/non-MIG.
496  */
497 void
kmemsysNumaRemoveMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId)498 kmemsysNumaRemoveMemory_GH100
499 (
500     OBJGPU             *pGpu,
501     KernelMemorySystem *pKernelMemorySystem,
502     NvU32               swizzId
503 )
504 {
505     if (pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse == NV_FALSE)
506     {
507         return;
508     }
509 
510     osNumaRemoveGpuMemory(pGpu->pOsGpuInfo,
511                           pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset,
512                           pKernelMemorySystem->memPartitionNumaInfo[swizzId].size,
513                           pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId);
514     pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse = NV_FALSE;
515     pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset = 0;
516     pKernelMemorySystem->memPartitionNumaInfo[swizzId].size = 0;
517     pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId = NV_U32_MAX;
518 
519     NV_PRINTF(LEVEL_INFO, "memory partition: %u removed successfully!\n",
520               swizzId);
521     return;
522 }
523 
524 /*!
525  * @brief Remove all GPU memory from OS kernel.
526  *
527  * Remove all MIG GPU instances GPU memory from the OS kernel that is earlier added
528  * as a NUMA node  to the kernel in platforms where GPU is coherently
529  * connected to the CPU.
530  *
531  * @param[in]  pGPU                OBJGPU pointer
532  * @param[in]  pKernelMemorySystem KernelMemorySystem pointer
533  *
534  */
535 void
kmemsysNumaRemoveAllMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)536 kmemsysNumaRemoveAllMemory_GH100
537 (
538     OBJGPU             *pGpu,
539     KernelMemorySystem *pKernelMemorySystem
540 )
541 {
542     NvU32 swizzId;
543 
544     for (swizzId = 0; swizzId < KMIGMGR_MAX_GPU_SWIZZID; swizzId++)
545     {
546         kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, swizzId);
547     }
548 
549     return;
550 }
551 
552 /*
553  * @brief   Function to map swizzId to VMMU Segments
554  */
555 NV_STATUS
kmemsysSwizzIdToVmmuSegmentsRange_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU32 vmmuSegmentSize,NvU32 totalVmmuSegments)556 kmemsysSwizzIdToVmmuSegmentsRange_GH100
557 (
558     OBJGPU *pGpu,
559     KernelMemorySystem *pKernelMemorySystem,
560     NvU32 swizzId,
561     NvU32 vmmuSegmentSize,
562     NvU32 totalVmmuSegments
563 )
564 {
565     KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
566     const KERNEL_MIG_MANAGER_STATIC_INFO *pStaticInfo = kmigmgrGetStaticInfo(pGpu, pKernelMIGManager);
567     NvU64 startingVmmuSegment;
568     NvU64 memSizeInVmmuSegment;
569 
570     NV_ASSERT_OR_RETURN(swizzId < KMIGMGR_MAX_GPU_SWIZZID, NV_ERR_INVALID_ARGUMENT);
571     NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE);
572     NV_ASSERT_OR_RETURN(pStaticInfo->pSwizzIdFbMemPageRanges != NULL, NV_ERR_INVALID_STATE);
573 
574     startingVmmuSegment = pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].lo;
575     memSizeInVmmuSegment = (pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].hi -
576                             pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].lo + 1);
577 
578     if (memSizeInVmmuSegment > totalVmmuSegments)
579     {
580         //
581         // SwizzID-0 should cover only partitionable range, however for AMAP,
582         // there is no difference between swizzID-0 and no MIG which can result in
583         // AMAP returning an additional vmmuSegment for swizzID-0
584         //
585         NV_ASSERT_OR_RETURN((swizzId == 0), NV_ERR_INVALID_STATE);
586     }
587 
588     NV_ASSERT_OK_OR_RETURN(
589         kmemsysInitMIGGPUInstanceMemConfigForSwizzId(pGpu, pKernelMemorySystem, swizzId, startingVmmuSegment, memSizeInVmmuSegment));
590 
591     return NV_OK;
592 }
593 
594 /*
595  * @brief   Function to check if an FB ACK timeout occured. Used only for Debug.
596  */
597 NvBool
kmemsysAssertFbAckTimeoutPending_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)598 kmemsysAssertFbAckTimeoutPending_GH100
599 (
600     OBJGPU *pGpu,
601     KernelMemorySystem *pKernelMemorySystem
602 )
603 {
604 #ifdef DEBUG
605     NvU32 intr0 = 0;
606     intr0 = GPU_REG_RD32(pGpu, NV_XAL_EP_INTR_0);
607     return DRF_VAL(_XAL_EP, _INTR_0, _FB_ACK_TIMEOUT, intr0) == NV_XAL_EP_INTR_0_FB_ACK_TIMEOUT_PENDING;
608 #else
609     return NV_FALSE;
610 #endif
611 }
612 
613 NvU32
kmemsysGetEccDedCountSize_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)614 kmemsysGetEccDedCountSize_GH100
615 (
616     OBJGPU             *pGpu,
617     KernelMemorySystem *pKernelMemorySystem
618 )
619 {
620     return NV_PFB_FBPA_0_ECC_DED_COUNT__SIZE_1;
621 }
622 
623 NvU32
kmemsysGetEccDedCountRegAddr_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 fbpa,NvU32 subp)624 kmemsysGetEccDedCountRegAddr_GH100
625 (
626     OBJGPU             *pGpu,
627     KernelMemorySystem *pKernelMemorySystem,
628     NvU32               fbpa,
629     NvU32               subp
630 )
631 {
632     return NV_PFB_FBPA_0_ECC_DED_COUNT(fbpa) + (subp * NV_FBPA_PRI_STRIDE);
633 }
634