1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "nvtypes.h"
27 #include "os/os.h"
28 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
29 #include "gpu/mem_mgr/mem_desc.h"
30 #include "gpu/bus/kern_bus.h"
31 #include "kernel/gpu/intr/intr.h"
32 #include "nverror.h"
33
34 #include "published/hopper/gh100/dev_fb.h"
35 #include "published/hopper/gh100/dev_fbpa.h"
36 #include "published/hopper/gh100/dev_vm.h"
37 #include "published/hopper/gh100/pri_nv_xal_ep.h"
38 #include "published/hopper/gh100/dev_nv_xal_addendum.h"
39 #include "published/hopper/gh100/dev_nv_xpl.h"
40 #include "published/hopper/gh100/dev_xtl_ep_pri.h"
41 #include "published/hopper/gh100/hwproject.h"
42 #include "published/ampere/ga100/dev_fb.h"
43
44 NV_STATUS
kmemsysDoCacheOp_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 reg,NvU32 regValue,NvU32 pollMask,PRMTIMEOUT pTimeout)45 kmemsysDoCacheOp_GH100
46 (
47 OBJGPU *pGpu,
48 KernelMemorySystem *pKernelMemorySystem,
49 NvU32 reg,
50 NvU32 regValue,
51 NvU32 pollMask,
52 PRMTIMEOUT pTimeout
53 )
54 {
55 NV_STATUS rmStatus = NV_OK;
56 NvU32 cnt = 0;
57 NV_STATUS timeoutStatus = NV_OK;
58 NvU32 tokenRangeMask = 0;
59 NvU32 startToken = 0;
60 NvU32 completedToken = 0;
61 NvBool bMemopBusy = NV_TRUE;
62
63 if (!API_GPU_ATTACHED_SANITY_CHECK(pGpu))
64 {
65 //
66 // When the GPU is lost we cannot expect to successfully do cache
67 // maintenance (see Bug 1557278).
68 //
69 return rmStatus;
70 }
71
72 if (IS_VIRTUAL(pGpu))
73 {
74 switch (reg)
75 {
76 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
77 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
78 break;
79 case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
80 return NV_OK;
81 default:
82 return NV_ERR_NOT_SUPPORTED;
83 }
84 }
85
86 switch (reg)
87 {
88 case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
89 tokenRangeMask = DRF_MASK(NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY_TOKEN);
90
91 startToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY);
92 startToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_FLUSH_DIRTY, _TOKEN, startToken);
93 break;
94
95 case NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS:
96 tokenRangeMask = DRF_MASK(NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS_TOKEN);
97
98 startToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS);
99 startToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS, _TOKEN, startToken);
100 break;
101
102 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
103 tokenRangeMask = DRF_MASK(NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE_TOKEN);
104
105 startToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE);
106 startToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE, _TOKEN, startToken);
107 break;
108
109 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
110 tokenRangeMask = DRF_MASK(NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE_TOKEN);
111
112 startToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE);
113 startToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE, _TOKEN, startToken);
114 break;
115
116 default:
117 return NV_ERR_NOT_SUPPORTED;
118 }
119
120 while(1)
121 {
122 switch (reg)
123 {
124 case NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY:
125 completedToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY_COMPLETED);
126
127 bMemopBusy = FLD_TEST_DRF(_XAL_EP_UFLUSH, _L2_FLUSH_DIRTY_COMPLETED, _STATUS, _BUSY, completedToken)
128 ? NV_TRUE : NV_FALSE;
129
130 completedToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_FLUSH_DIRTY_COMPLETED, _TOKEN, completedToken);
131 break;
132
133 case NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS:
134 completedToken = GPU_REG_RD32(pGpu, NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS_COMPLETED);
135
136 bMemopBusy = FLD_TEST_DRF(_XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS_COMPLETED, _STATUS, _BUSY, completedToken)
137 ? NV_TRUE : NV_FALSE;
138
139 completedToken = DRF_VAL( _XAL_EP_UFLUSH, _L2_CLEAN_COMPTAGS_COMPLETED, _TOKEN, completedToken);
140 break;
141
142 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE:
143 completedToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED);
144
145 bMemopBusy = FLD_TEST_DRF( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED, _STATUS, _BUSY, completedToken)
146 ? NV_TRUE : NV_FALSE;
147
148 completedToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_SYSMEM_INVALIDATE_COMPLETED, _TOKEN, completedToken);
149 break;
150
151 case NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE:
152 completedToken = GPU_VREG_RD32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED);
153
154 bMemopBusy = FLD_TEST_DRF( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED, _STATUS, _BUSY, completedToken)
155 ? NV_TRUE : NV_FALSE;
156
157 completedToken = DRF_VAL( _VIRTUAL_FUNCTION, _PRIV_FUNC_L2_PEERMEM_INVALIDATE_COMPLETED, _TOKEN, completedToken);
158 break;
159 }
160
161 if (bMemopBusy == NV_FALSE)
162 {
163 break;
164 }
165
166 //
167 // When completedToken > startToken(including the wrapping around case), due to the nature
168 // of unsigned number, the value of "(startToken - completeToken) & tokenRangeMask" will be
169 // at top of token range which will be bigger than NV_XAL_EP_MEMOP_MAX_OUTSTANDING. So it
170 // will break out from the loop.
171 //
172 // The loop will wait only when completedToken in the range of
173 // [startToken-NV_XAL_EP_MEMOP_MAX_OUTSTANDING, startToken].
174 //
175 if (((startToken - completedToken) & tokenRangeMask) > NV_XAL_EP_MEMOP_MAX_OUTSTANDING)
176 {
177 break;
178 }
179
180 if (timeoutStatus == NV_ERR_TIMEOUT)
181 {
182 //
183 // This should not timeout, except for a HW bug. Famous last words.
184 // On !DEBUG we just keep trucking, it's the best we can do.
185 //
186 NV_PRINTF(LEVEL_ERROR,
187 "- timeout error waiting for reg 0x%x update cnt=%d\n",
188 reg, cnt);
189 rmStatus = NV_ERR_TIMEOUT;
190 DBG_BREAKPOINT();
191 break;
192 }
193 else if ( API_GPU_IN_RESET_SANITY_CHECK(pGpu) ||
194 !API_GPU_ATTACHED_SANITY_CHECK(pGpu))
195 {
196 //
197 // The GPU is in full chip reset, or has fallen off the bus
198 // Just return
199 //
200 return NV_OK;
201 }
202
203 timeoutStatus = gpuCheckTimeout(pGpu, pTimeout);
204 osSpinLoop();
205 cnt++;
206 }
207
208 #ifdef DEBUG
209 if (cnt > 1)
210 {
211 NV_ASSERT(kmemsysAssertFbAckTimeoutPending_HAL(pGpu, pKernelMemorySystem) == NV_FALSE);
212 }
213 #endif // DEBUG
214
215 return rmStatus;
216 }
217
218 /**
219 * @brief L2 Cache management OPs
220 *
221 * GK20A/T124 have a writeback L2 cache, so the cache ops are slightly
222 * different than those of Fermi/Kepler. Specifically, we can write back dirty
223 * lines to system memory.
224 *
225 * @param pMemDesc
226 * @param targetMem
227 * @param cacheOp
228 */
229 NV_STATUS
kmemsysCacheOp_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,PMEMORY_DESCRIPTOR pMemDesc,FB_CACHE_MEMTYPE targetMem,FB_CACHE_OP cacheOp)230 kmemsysCacheOp_GH100
231 (
232 OBJGPU *pGpu,
233 KernelMemorySystem *pKernelMemorySystem,
234 PMEMORY_DESCRIPTOR pMemDesc,
235 FB_CACHE_MEMTYPE targetMem,
236 FB_CACHE_OP cacheOp
237 )
238 {
239 NV_STATUS status = NV_OK;
240 RMTIMEOUT timeout = {0, };
241 NvU32 reg = 0;
242
243 if ((targetMem == FB_CACHE_MEM_UNDEFINED) && pMemDesc)
244 {
245 targetMem = (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM) ?
246 FB_CACHE_VIDEO_MEMORY : FB_CACHE_SYSTEM_MEMORY;
247 }
248
249 if ((cacheOp == FB_CACHE_OP_UNDEFINED) || (targetMem == FB_CACHE_MEM_UNDEFINED))
250 {
251 NV_PRINTF(LEVEL_ERROR, "called with null %s\n",
252 cacheOp ? "cache operation" : "memory target");
253 DBG_BREAKPOINT();
254 return status; // return NV_OK
255 }
256
257 // For GK20A, an explicit sysmembar flush is needed before L2 cache flush operation.
258 // Refer GK20A LTC IAS (section 5.5)
259 kbusSendSysmembarSingle_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu));
260
261 // Wait for the flush to flow through
262 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
263
264 switch (targetMem)
265 {
266 case FB_CACHE_SYSTEM_MEMORY:
267 if (cacheOp == FB_CACHE_INVALIDATE)
268 {
269 NV_PRINTF(LEVEL_INFO,
270 "Invalidate not supported, promoting to an evict (writeback + "
271 "invalidate clean lines).\n");
272 cacheOp = FB_CACHE_EVICT;
273 }
274
275 if (cacheOp == FB_CACHE_WRITEBACK || cacheOp == FB_CACHE_EVICT)
276 {
277 reg = NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY;
278 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
279 }
280
281 if (cacheOp == FB_CACHE_EVICT)
282 {
283 //
284 // Ideally we should use NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES.
285 // But NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES is not defined on desktop GPUs.
286 // NV_UFLUSH_L2_SYSMEM_INVALIDATE is same as NV_UFLUSH_L2_INVALIDATE_CLEAN_LINES, and is defined in all chips.
287 // For Hopper, it is changed to NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE
288 //
289 reg = NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_SYSMEM_INVALIDATE;
290 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
291 }
292 break;
293
294 case FB_CACHE_VIDEO_MEMORY:
295 // Fbmem cache ops are not supported from VF -- force NV_OK for MODS
296 if (IS_VIRTUAL(pGpu))
297 return NV_OK;
298
299 if (cacheOp == FB_CACHE_EVICT)
300 {
301 NvU32 flags = NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_ALL |
302 NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_CLEAN;
303 status = kmemsysSendL2InvalidateEvict(pGpu, pKernelMemorySystem, flags);
304 }
305 else if (cacheOp == FB_CACHE_INVALIDATE)
306 {
307 NvU32 flags = NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_FLAGS_ALL;
308 status = kmemsysSendL2InvalidateEvict(pGpu, pKernelMemorySystem, flags);
309 }
310 else
311 {
312 status = NV_ERR_NOT_SUPPORTED;
313 }
314 break;
315
316 case FB_CACHE_PEER_MEMORY:
317 //
318 // for GF100 - for sysmem cache only invalidate operation is supported
319 // evict = writeback+invalidate is reduced to invalidate for GF100
320 //
321 if((cacheOp != FB_CACHE_INVALIDATE) && (cacheOp != FB_CACHE_EVICT))
322 {
323 return NV_OK;
324 }
325
326 reg = NV_VIRTUAL_FUNCTION_PRIV_FUNC_L2_PEERMEM_INVALIDATE;
327 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
328 break;
329
330 case FB_CACHE_COMPTAG_MEMORY:
331 if(cacheOp != FB_CACHE_WRITEBACK && cacheOp != FB_CACHE_EVICT)
332 {
333 return NV_OK;
334 }
335
336 //
337 // Beware of HW bug 545242. Graphics should be idle and flushed here
338 // or comp tag cache could be corrupted. When mods uses this call
339 // during verif, this should already be the case.
340 //
341 reg = NV_XAL_EP_UFLUSH_L2_CLEAN_COMPTAGS;
342 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
343 break;
344
345 case FB_CACHE_DIRTY:
346 if(cacheOp != FB_CACHE_WRITEBACK && cacheOp != FB_CACHE_EVICT)
347 {
348 return NV_OK;
349 }
350
351 reg = NV_XAL_EP_UFLUSH_L2_FLUSH_DIRTY;
352 status = kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, reg, 0, 0, &timeout);
353 break;
354
355 case FB_CACHE_DIRTY_ALL:
356 if(cacheOp != FB_CACHE_EVICT)
357 {
358 return NV_OK;
359 }
360
361 status = kmemsysSendFlushL2AllRamsAndCaches(pGpu, pKernelMemorySystem);
362 break;
363
364 default:
365 // return OK for other memory targets
366 status = NV_OK;
367 break;
368 }
369
370 return status;
371 }
372
373 /*!
374 * @brief Write the sysmemFlushBuffer val into the NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR register
375 *
376 * @param[in] pGpu OBJGPU pointer
377 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
378 *
379 * @returns void
380 */
381 void
kmemsysProgramSysmemFlushBuffer_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)382 kmemsysProgramSysmemFlushBuffer_GH100
383 (
384 OBJGPU *pGpu,
385 KernelMemorySystem *pKernelMemorySystem
386 )
387 {
388 NvU32 alignedSysmemFlushBufferAddr = 0x0;
389 NvU32 alignedSysmemFlushBufferAddrHi = 0x0;
390
391 NV_ASSERT(pKernelMemorySystem->sysmemFlushBuffer != 0);
392
393 // alignedSysmemFlushBufferAddr will have the lower 32 bits of the buffer address
394 alignedSysmemFlushBufferAddr = NvU64_LO32(pKernelMemorySystem->sysmemFlushBuffer);
395
396 // alignedSysmemFlushBufferAddrHi will have the upper 32 bits of the buffer address
397 alignedSysmemFlushBufferAddrHi = NvU64_HI32(pKernelMemorySystem->sysmemFlushBuffer);
398
399 // Assert when Sysmem Flush buffer has more than 52-bit address
400 NV_ASSERT((alignedSysmemFlushBufferAddrHi & (~NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI_ADR_MASK)) == 0);
401
402 alignedSysmemFlushBufferAddrHi &= NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI_ADR_MASK;
403
404 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI, _ADR, alignedSysmemFlushBufferAddrHi);
405 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO, _ADR, alignedSysmemFlushBufferAddr);
406 }
407
408 /*!
409 * @brief Validate the sysmemFlushBuffer val and assert
410 *
411 * @param[in] pGpu OBJGPU pointer
412 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
413 *
414 * @returns void
415 */
416 void
kmemsysAssertSysmemFlushBufferValid_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)417 kmemsysAssertSysmemFlushBufferValid_GH100
418 (
419 OBJGPU *pGpu,
420 KernelMemorySystem *pKernelMemorySystem
421 )
422 {
423 NV_ASSERT((GPU_REG_RD_DRF(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO, _ADR) != 0) ||
424 (GPU_REG_RD_DRF(pGpu, _PFB, _FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI, _ADR) != 0));
425 }
426
427 /*!
428 * @brief Add GPU memory as a NUMA node.
429 *
430 * Add GPU memory as a NUMA node to the OS kernel in platforms where
431 * GPU is coherently connected to the CPU.
432 *
433 * @param[in] pGPU OBJGPU pointer
434 * @param[in] pKernelMemorySystem KernelMemorySystem pointer
435 * @param[in] swizzId swizzId of the MIG GPU instance, 0 for full GPU instance/non-MIG.
436 * @param[in] offset start offset of the GPU instance within FB
437 * @param[in] size size of the GPU instance
438 * @param[out] numaNodeId NUMA node id corresponding to the added @swizzId partition memory
439 * when NV_OK is returned.
440 *
441 * @returns NV_OK if all is okay. Otherwise an error-specific value.
442 *
443 */
444 NV_STATUS
kmemsysNumaAddMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU64 offset,NvU64 size,NvS32 * numaNodeId)445 kmemsysNumaAddMemory_GH100
446 (
447 OBJGPU *pGpu,
448 KernelMemorySystem *pKernelMemorySystem,
449 NvU32 swizzId,
450 NvU64 offset,
451 NvU64 size,
452 NvS32 *numaNodeId
453 )
454 {
455 NV_STATUS status;
456 NvU64 memblockSize = 0;
457 NvU32 lNumaNodeId;
458
459 NV_ASSERT_OK_OR_RETURN(osNumaMemblockSize(&memblockSize));
460 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED(size, memblockSize), NV_ERR_INVALID_STATE);
461
462 if (pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse)
463 {
464 NV_PRINTF(LEVEL_ERROR, "Memory partition: %u is already in use!\n", swizzId);
465 return NV_ERR_IN_USE;
466 }
467
468 status = osNumaAddGpuMemory(pGpu->pOsGpuInfo, offset, size, &lNumaNodeId);
469 if (status == NV_OK)
470 {
471 pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse = NV_TRUE;
472 pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset = offset;
473 pKernelMemorySystem->memPartitionNumaInfo[swizzId].size = size;
474 pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId = lNumaNodeId;
475 *numaNodeId = lNumaNodeId;
476
477 pKernelMemorySystem->bNumaNodesAdded = NV_TRUE;
478
479 NV_PRINTF(LEVEL_INFO, "Memory partition: %u added successfully!"
480 " numa id: %u offset: 0x%llx size: 0x%llx\n",
481 swizzId, lNumaNodeId, offset, size);
482 }
483
484 return status;
485 }
486
487 /*!
488 * @brief Remove a particular MIG GPU instance GPU memory from OS kernel.
489 *
490 * Remove GPU memory from the OS kernel that is earlier added as a NUMA node
491 * to the kernel in platforms where GPU is coherently connected to the CPU.
492 *
493 * @param[in] pGPU OBJGPU pointer
494 * @param[in] pKernelMemorySystem KernelMemorySystem pointer
495 * @param[in] swizzId swizzId of the MIG GPU instance, 0 for full partition/non-MIG.
496 */
497 void
kmemsysNumaRemoveMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId)498 kmemsysNumaRemoveMemory_GH100
499 (
500 OBJGPU *pGpu,
501 KernelMemorySystem *pKernelMemorySystem,
502 NvU32 swizzId
503 )
504 {
505 if (pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse == NV_FALSE)
506 {
507 return;
508 }
509
510 osNumaRemoveGpuMemory(pGpu->pOsGpuInfo,
511 pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset,
512 pKernelMemorySystem->memPartitionNumaInfo[swizzId].size,
513 pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId);
514 pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse = NV_FALSE;
515 pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset = 0;
516 pKernelMemorySystem->memPartitionNumaInfo[swizzId].size = 0;
517 pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId = NV_U32_MAX;
518
519 NV_PRINTF(LEVEL_INFO, "memory partition: %u removed successfully!\n",
520 swizzId);
521 return;
522 }
523
524 /*!
525 * @brief Remove all GPU memory from OS kernel.
526 *
527 * Remove all MIG GPU instances GPU memory from the OS kernel that is earlier added
528 * as a NUMA node to the kernel in platforms where GPU is coherently
529 * connected to the CPU.
530 *
531 * @param[in] pGPU OBJGPU pointer
532 * @param[in] pKernelMemorySystem KernelMemorySystem pointer
533 *
534 */
535 void
kmemsysNumaRemoveAllMemory_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)536 kmemsysNumaRemoveAllMemory_GH100
537 (
538 OBJGPU *pGpu,
539 KernelMemorySystem *pKernelMemorySystem
540 )
541 {
542 NvU32 swizzId;
543
544 for (swizzId = 0; swizzId < KMIGMGR_MAX_GPU_SWIZZID; swizzId++)
545 {
546 kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, swizzId);
547 }
548
549 return;
550 }
551
552 /*
553 * @brief Function to map swizzId to VMMU Segments
554 */
555 NV_STATUS
kmemsysSwizzIdToVmmuSegmentsRange_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU32 vmmuSegmentSize,NvU32 totalVmmuSegments)556 kmemsysSwizzIdToVmmuSegmentsRange_GH100
557 (
558 OBJGPU *pGpu,
559 KernelMemorySystem *pKernelMemorySystem,
560 NvU32 swizzId,
561 NvU32 vmmuSegmentSize,
562 NvU32 totalVmmuSegments
563 )
564 {
565 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
566 const KERNEL_MIG_MANAGER_STATIC_INFO *pStaticInfo = kmigmgrGetStaticInfo(pGpu, pKernelMIGManager);
567 NvU64 startingVmmuSegment;
568 NvU64 memSizeInVmmuSegment;
569
570 NV_ASSERT_OR_RETURN(swizzId < KMIGMGR_MAX_GPU_SWIZZID, NV_ERR_INVALID_ARGUMENT);
571 NV_ASSERT_OR_RETURN(pStaticInfo != NULL, NV_ERR_INVALID_STATE);
572 NV_ASSERT_OR_RETURN(pStaticInfo->pSwizzIdFbMemPageRanges != NULL, NV_ERR_INVALID_STATE);
573
574 startingVmmuSegment = pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].lo;
575 memSizeInVmmuSegment = (pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].hi -
576 pStaticInfo->pSwizzIdFbMemPageRanges->fbMemPageRanges[swizzId].lo + 1);
577
578 if (memSizeInVmmuSegment > totalVmmuSegments)
579 {
580 //
581 // SwizzID-0 should cover only partitionable range, however for AMAP,
582 // there is no difference between swizzID-0 and no MIG which can result in
583 // AMAP returning an additional vmmuSegment for swizzID-0
584 //
585 NV_ASSERT_OR_RETURN((swizzId == 0), NV_ERR_INVALID_STATE);
586 }
587
588 NV_ASSERT_OK_OR_RETURN(
589 kmemsysInitMIGGPUInstanceMemConfigForSwizzId(pGpu, pKernelMemorySystem, swizzId, startingVmmuSegment, memSizeInVmmuSegment));
590
591 return NV_OK;
592 }
593
594 /*
595 * @brief Function to check if an FB ACK timeout occured. Used only for Debug.
596 */
597 NvBool
kmemsysAssertFbAckTimeoutPending_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)598 kmemsysAssertFbAckTimeoutPending_GH100
599 (
600 OBJGPU *pGpu,
601 KernelMemorySystem *pKernelMemorySystem
602 )
603 {
604 #ifdef DEBUG
605 NvU32 intr0 = 0;
606 intr0 = GPU_REG_RD32(pGpu, NV_XAL_EP_INTR_0);
607 return DRF_VAL(_XAL_EP, _INTR_0, _FB_ACK_TIMEOUT, intr0) == NV_XAL_EP_INTR_0_FB_ACK_TIMEOUT_PENDING;
608 #else
609 return NV_FALSE;
610 #endif
611 }
612
613 NvU32
kmemsysGetEccDedCountSize_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)614 kmemsysGetEccDedCountSize_GH100
615 (
616 OBJGPU *pGpu,
617 KernelMemorySystem *pKernelMemorySystem
618 )
619 {
620 return NV_PFB_FBPA_0_ECC_DED_COUNT__SIZE_1;
621 }
622
623 NvU32
kmemsysGetEccDedCountRegAddr_GH100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 fbpa,NvU32 subp)624 kmemsysGetEccDedCountRegAddr_GH100
625 (
626 OBJGPU *pGpu,
627 KernelMemorySystem *pKernelMemorySystem,
628 NvU32 fbpa,
629 NvU32 subp
630 )
631 {
632 return NV_PFB_FBPA_0_ECC_DED_COUNT(fbpa) + (subp * NV_FBPA_PRI_STRIDE);
633 }
634