1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "os/os.h"
25 #include "gpu/mem_sys/kern_mem_sys.h"
26 #include "gpu/mem_mgr/mem_mgr.h"
27 #include "virtualization/hypervisor/hypervisor.h"
28 #include "vgpu/vgpu_events.h"
29 #include "objrpc.h"
30 #include "gpu/bif/kernel_bif.h"
31 #include "gpu/bus/kern_bus.h"
32 #include "os/os.h"
33 #include "platform/sli/sli.h"
34 #include "nvRmReg.h"
35 #include "gpu/gsp/gsp_static_config.h"
36
37 static void
kmemsysInitRegistryOverrides(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)38 kmemsysInitRegistryOverrides
39 (
40 OBJGPU *pGpu,
41 KernelMemorySystem *pKernelMemorySystem
42 )
43 {
44 NvU32 data32;
45
46 //
47 // Bug 1032432. Check regkey for FB pull
48 //
49 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_L2_CLEAN_FB_PULL, &data32) == NV_OK)
50 {
51 if (data32 == NV_REG_STR_RM_L2_CLEAN_FB_PULL_DISABLED)
52 pKernelMemorySystem->bL2CleanFbPull = NV_FALSE;
53 }
54
55 if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_OVERRIDE_TO_GMK, &data32) == NV_OK) &&
56 (data32 != NV_REG_STR_RM_OVERRIDE_TO_GMK_DISABLED))
57 {
58 pKernelMemorySystem->overrideToGMK = data32;
59 }
60
61 }
62
63 NV_STATUS
kmemsysConstructEngine_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,ENGDESCRIPTOR engDesc)64 kmemsysConstructEngine_IMPL
65 (
66 OBJGPU *pGpu,
67 KernelMemorySystem *pKernelMemorySystem,
68 ENGDESCRIPTOR engDesc
69 )
70 {
71 pKernelMemorySystem->memPartitionNumaInfo = NULL;
72
73 kmemsysInitRegistryOverrides(pGpu, pKernelMemorySystem);
74
75 if (IS_GSP_CLIENT(pGpu))
76 {
77 // Setting up the sysmem flush buffer needs to be done very early in some cases
78 // as it's required for the GPU to perform a system flush. One such case is
79 // resetting GPU FALCONs and in particular resetting the PMU as part of VBIOS
80 // init.
81 NV_ASSERT_OK_OR_RETURN(kmemsysInitFlushSysmemBuffer_HAL(pGpu, pKernelMemorySystem));
82
83 }
84
85 return NV_OK;
86 }
87
88 /*
89 * Initialize the Kernel Memory System state.
90 *
91 * @param[in] pGpu pointer to the GPU instance.
92 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance.
93 *
94 * @return NV_OK upon success.
95 */
kmemsysStateInitLocked_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)96 NV_STATUS kmemsysStateInitLocked_IMPL
97 (
98 OBJGPU *pGpu,
99 KernelMemorySystem *pKernelMemorySystem
100 )
101 {
102 MEMORY_SYSTEM_STATIC_CONFIG *pStaticConfig;
103 NV_STATUS status = NV_OK;
104
105 NV_ASSERT_OK_OR_GOTO(status, kmemsysEnsureSysmemFlushBufferInitialized(pGpu, pKernelMemorySystem), fail);
106
107 pStaticConfig = portMemAllocNonPaged(sizeof(*pStaticConfig));
108 NV_CHECK_OR_RETURN(LEVEL_ERROR, pStaticConfig != NULL, NV_ERR_INSUFFICIENT_RESOURCES);
109 portMemSet(pStaticConfig, 0, sizeof(*pStaticConfig));
110
111 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
112 kmemsysInitStaticConfig_HAL(pGpu, pKernelMemorySystem, pStaticConfig),
113 fail);
114
115 pKernelMemorySystem->pStaticConfig = pStaticConfig;
116
117 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
118
119 pKernelMemorySystem->memPartitionNumaInfo = portMemAllocNonPaged(sizeof(MEM_PARTITION_NUMA_INFO) * KMIGMGR_MAX_GPU_SWIZZID);
120 if (pKernelMemorySystem->memPartitionNumaInfo == NULL)
121 {
122 NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory for numa information.\n");
123 status = NV_ERR_NO_MEMORY;
124 NV_ASSERT_OR_GOTO(0, fail);
125 }
126 portMemSet(pKernelMemorySystem->memPartitionNumaInfo, 0, sizeof(MEM_PARTITION_NUMA_INFO) * KMIGMGR_MAX_GPU_SWIZZID);
127
128 if (gpuIsSelfHosted(pGpu) &&
129 (pKernelBif != NULL) && pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
130 {
131 //
132 // memsysSetupCoherentCpuLink should be done only for the self hosted
133 // configuration(SHH) where the coherent C2C link connects host CPU(TH500) and GPU
134 // and not in the externally hosted(EHH) case where host CPU(say x86) is connected
135 // to GPU through PCIe and C2C only connects the TH500 (for EGM memory) and GPU.
136 // The gpuIsSelfHosted(pGpu) check here is to distinguish between the SHH
137 // and EHH configuration as C2C link is up in both of these cases.
138 //
139
140 if (IS_GSP_CLIENT(pGpu))
141 {
142 GspStaticConfigInfo *pGSCI = GPU_GET_GSP_STATIC_INFO(pGpu);
143
144 if (pGSCI->bAtsSupported)
145 {
146 NV_PRINTF(LEVEL_INFO, "ATS supported\n");
147
148 pGpu->setProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED, NV_TRUE);
149 }
150 }
151 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL_WITH_SRIOV(pGpu))
152 {
153 //
154 // PDB_PROP_GPU_C2C_SYSMEM is already set in physical-RM but not in
155 // in Kernel-RM/Guest-RM where it is actually consumed. setting PDB_PROP_GPU_C2C_SYSMEM
156 // in Kernel-RM/Guest-RM when the platform is self-hosted and the C2C links are up, which
157 // indicate the C2C is connected to CPU and Physical-RM would have set up the HSHUB
158 // to route sysmem through C2C.
159 //
160 pGpu->setProperty(pGpu, PDB_PROP_GPU_C2C_SYSMEM, NV_TRUE);
161 }
162
163 //
164 // kmemesysSetupCoherentCpuLink should not be called from physical RM as
165 // it is intended to be called on kernel side to update
166 // KernelMemorySystem for C2C, NUMA functionality.
167 //
168 NV_ASSERT_OK_OR_GOTO(status, kmemsysSetupCoherentCpuLink(pGpu, pKernelMemorySystem, NV_FALSE), fail);
169 }
170
171 {
172 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu);
173
174 //
175 // Ask GMMU to set the large page size after we have initialized
176 // memory and before we initialize BAR2.
177 //
178 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
179 kgmmuCheckAndDecideBigPageSize_HAL(pGpu, pKernelGmmu));
180 }
181
182 fail:
183 if (status != NV_OK)
184 {
185 portMemFree((void *)pKernelMemorySystem->pStaticConfig);
186 }
187
188 return status;
189 }
190
191 NV_STATUS
kmemsysStatePreLoad_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)192 kmemsysStatePreLoad_IMPL
193 (
194 OBJGPU *pGpu,
195 KernelMemorySystem *pKernelMemorySystem,
196 NvU32 flags
197 )
198 {
199 //
200 // Program the sysmem flush buffer address and assert that the register contents are valid.
201 // The HAL wiring is such that a given RM build will only do one or the other (e.g., RM offloaded
202 // to ucode won't program the register itself but will assert that its contents are valid).
203 //
204 kmemsysProgramSysmemFlushBuffer_HAL(pGpu, pKernelMemorySystem);
205 kmemsysAssertSysmemFlushBufferValid_HAL(pGpu, pKernelMemorySystem);
206
207 // Self Hosted GPUs should have its memory onlined by now.
208 if (gpuIsSelfHosted(pGpu) &&
209 pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
210 osNumaOnliningEnabled(pGpu->pOsGpuInfo) &&
211 !pKernelMemorySystem->bNumaNodesAdded)
212 {
213 //
214 // TODO: Bug 1945658: Deferred error checking from stateInit so that stateDestroy
215 // gets called. Refer kmemsysNumaAddMemory_HAL call site for further
216 // details.
217 //
218 return NV_ERR_INVALID_STATE;
219 }
220
221 return NV_OK;
222 }
223
224 NV_STATUS
kmemsysStatePostLoad_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)225 kmemsysStatePostLoad_IMPL
226 (
227 OBJGPU *pGpu,
228 KernelMemorySystem *pKernelMemorySystem,
229 NvU32 flags
230 )
231 {
232 if (IS_SILICON(pGpu) &&
233 pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED))
234 {
235 NV_STATUS status = kmemsysSetupAllAtsPeers_HAL(pGpu, pKernelMemorySystem);
236 if (status != NV_OK)
237 {
238 NV_PRINTF(LEVEL_ERROR, "ATS peer setup failed.\n");
239 return status;
240 }
241 }
242
243 return NV_OK;
244 }
245
246 NV_STATUS
kmemsysStatePreUnload_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)247 kmemsysStatePreUnload_IMPL
248 (
249 OBJGPU *pGpu,
250 KernelMemorySystem *pKernelMemorySystem,
251 NvU32 flags
252 )
253 {
254 if (IS_SILICON(pGpu) &&
255 pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED))
256 {
257 kmemsysRemoveAllAtsPeers_HAL(pGpu, pKernelMemorySystem);
258 }
259 return NV_OK;
260 }
261
262 /*
263 * Release the state accumulated in StateInit.
264 * @param[in] pGpu pointer to the GPU instance.
265 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance.
266 */
kmemsysStateDestroy_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)267 void kmemsysStateDestroy_IMPL
268 (
269 OBJGPU *pGpu,
270 KernelMemorySystem *pKernelMemorySystem
271 )
272 {
273
274 // Teardown of Coherent Cpu Link is not required on Physical RM
275 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
276
277 if (pKernelBif && pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP) &&
278 pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING))
279 {
280 kmemsysTeardownCoherentCpuLink(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu), NV_FALSE);
281 }
282
283 portMemFree((void *)pKernelMemorySystem->pStaticConfig);
284 }
285
286 /*!
287 * Returns MemorySystem settings that are static after GPU state init/load is
288 * finished.
289 */
290 const MEMORY_SYSTEM_STATIC_CONFIG *
kmemsysGetStaticConfig_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)291 kmemsysGetStaticConfig_IMPL
292 (
293 OBJGPU *pGpu,
294 KernelMemorySystem *pKernelMemorySystem
295 )
296 {
297 // check if state Init has not completed.
298 NV_ASSERT_OR_ELSE(pKernelMemorySystem != NULL, return NULL);
299
300 return pKernelMemorySystem->pStaticConfig;
301 }
302
303 void
kmemsysDestruct_IMPL(KernelMemorySystem * pKernelMemorySystem)304 kmemsysDestruct_IMPL
305 (
306 KernelMemorySystem *pKernelMemorySystem
307 )
308 {
309 OBJGPU *pGpu = ENG_GET_GPU(pKernelMemorySystem);
310
311 //
312 // kmemsysNumaRemoveAllMemory_HAL() is called here in Destruct instead of
313 // Destroy to guarantee that NUMA memory is removed. This goes against
314 // Init/Destroy symmetry, but it is necessary because kmemsysStateDestroy
315 // may not be called for all cases when kmemsysStateInit was called
316 // (e.g., when kmemsys or another engine afterwards fails Init).
317 //
318 // If NUMA memory is not removed, then all subsequent attempts to add NUMA
319 // memory will fail, which will cause failures in future RM init attempts.
320 //
321 if (pKernelMemorySystem->memPartitionNumaInfo != NULL)
322 {
323 if (pKernelMemorySystem->bNumaNodesAdded == NV_TRUE)
324 {
325 kmemsysNumaRemoveAllMemory_HAL(pGpu, pKernelMemorySystem);
326 }
327 portMemFree(pKernelMemorySystem->memPartitionNumaInfo);
328 }
329
330 pKernelMemorySystem->sysmemFlushBuffer = 0;
331 memdescFree(pKernelMemorySystem->pSysmemFlushBufferMemDesc);
332 memdescDestroy(pKernelMemorySystem->pSysmemFlushBufferMemDesc);
333 pKernelMemorySystem->pSysmemFlushBufferMemDesc = NULL;
334
335 portMemSet(pKernelMemorySystem->gpuInstanceMemConfig, 0, sizeof(pKernelMemorySystem->gpuInstanceMemConfig));
336
337 }
338
339 NV_STATUS
kmemsysAllocComprResources_KERNEL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,FB_ALLOC_INFO * pFbAllocInfo,NvU64 origSize,NvU32 kindChosen,NvU32 * pRetAttr,NvU32 retAttr2)340 kmemsysAllocComprResources_KERNEL
341 (
342 OBJGPU *pGpu,
343 KernelMemorySystem *pKernelMemorySystem,
344 FB_ALLOC_INFO *pFbAllocInfo,
345 NvU64 origSize,
346 NvU32 kindChosen,
347 NvU32 *pRetAttr,
348 NvU32 retAttr2
349 )
350 {
351 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
352 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = kmemsysGetStaticConfig(pGpu, pKernelMemorySystem);
353 NvU32 gfid;
354
355 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid));
356
357 NV_ASSERT_OR_RETURN(pMemorySystemConfig->bOneToOneComptagLineAllocation || pMemorySystemConfig->bUseRawModeComptaglineAllocation,
358 NV_ERR_INVALID_STATE);
359
360 NV_CHECK_OR_RETURN(LEVEL_ERROR,
361 !FLD_TEST_DRF(OS32, _ALLOC, _COMPTAG_OFFSET_USAGE, _FIXED, pFbAllocInfo->ctagOffset),
362 NV_ERR_INVALID_ARGUMENT);
363
364 // Failing the allocation if scrub on free is disabled
365 if (!memmgrIsScrubOnFreeEnabled(pMemoryManager))
366 {
367 if (!(IS_SIMULATION(pGpu) || IsDFPGA(pGpu) || (IS_EMULATION(pGpu) && RMCFG_FEATURE_PLATFORM_MODS)
368 ||(RMCFG_FEATURE_PLATFORM_WINDOWS && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE))
369 ||hypervisorIsVgxHyper()
370 ||IS_GFID_VF(gfid)
371 ||(IsSLIEnabled(pGpu) && !(RMCFG_FEATURE_PLATFORM_WINDOWS &&
372 !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE))))
373 )
374 {
375 NV_PRINTF(LEVEL_ERROR, "Compressible surfaces cannot be allocated on a system, "
376 "where scrub on free is disabled\n");
377 return NV_ERR_INVALID_STATE;
378 }
379 }
380 else if (pMemorySystemConfig->bOneToOneComptagLineAllocation)
381 {
382 NV_ASSERT_OR_RETURN(memmgrUseVasForCeMemoryOps(pMemoryManager), NV_ERR_INVALID_STATE);
383 }
384
385 FB_SET_HWRESID_CTAGID_FERMI(pFbAllocInfo->hwResId, FB_HWRESID_CTAGID_VAL_FERMI(-1));
386 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _COMPR, _REQUIRED, *pRetAttr);
387 return NV_OK;
388 }
389
390 /*!
391 * @brief Initializes static config data from the Physical side.
392 * @param[in] pGpu pointer to the GPU instance.
393 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance.
394 * @param[out] pConfig pointer to the static config init on Physical driver.
395 *
396 * @return NV_OK upon success.
397 * NV_ERR* otherwise.
398 */
399 NV_STATUS
kmemsysInitStaticConfig_KERNEL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,MEMORY_SYSTEM_STATIC_CONFIG * pConfig)400 kmemsysInitStaticConfig_KERNEL
401 (
402 OBJGPU *pGpu,
403 KernelMemorySystem *pKernelMemorySystem,
404 MEMORY_SYSTEM_STATIC_CONFIG *pConfig
405 )
406 {
407 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
408
409 return pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
410 NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG,
411 pConfig, sizeof(*pConfig));
412 }
413
414 /*!
415 * @brief Function to map swizzId to mem size given total mem
416 *
417 * @param[IN] pGpu
418 * @param[IN] pKernelMemorySystem
419 * @param[IN] swizzId
420 * @param[IN] totalRange total memory range
421 * @param[OUT] pPartitionSizeFlag Flag stating partition memory size
422 * @param[OUT] pSizeInBytes Memory size in bytes supported by partition
423 */
424 NV_STATUS
kmemsysSwizzIdToMIGMemSize_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NV_RANGE totalRange,NvU32 * pPartitionSizeFlag,NvU64 * pSizeInBytes)425 kmemsysSwizzIdToMIGMemSize_IMPL
426 (
427 OBJGPU *pGpu,
428 KernelMemorySystem *pKernelMemorySystem,
429 NvU32 swizzId,
430 NV_RANGE totalRange,
431 NvU32 *pPartitionSizeFlag,
432 NvU64 *pSizeInBytes
433 )
434 {
435 //
436 // To handle the straddling issue we always consider memory for different
437 // swizzIds as addition of minimum sized segements allowed in partitioning
438 //
439 NvU64 memSize = rangeLength(totalRange) / KMIGMGR_MAX_GPU_INSTANCES;
440
441 switch (swizzId)
442 {
443 case 0:
444 {
445 *pSizeInBytes = memSize * KMIGMGR_MAX_GPU_INSTANCES;
446 *pPartitionSizeFlag = DRF_DEF(2080_CTRL_GPU, _PARTITION_FLAG, _MEMORY_SIZE, _FULL);
447 break;
448 }
449
450 case 1:
451 case 2:
452 {
453 *pSizeInBytes = (memSize * (KMIGMGR_MAX_GPU_INSTANCES / 2));
454 *pPartitionSizeFlag = DRF_DEF(2080_CTRL_GPU, _PARTITION_FLAG, _MEMORY_SIZE, _HALF);
455 break;
456 }
457
458 case 3:
459 case 4:
460 case 5:
461 case 6:
462 {
463 *pSizeInBytes = (memSize * (KMIGMGR_MAX_GPU_INSTANCES / 4));
464 *pPartitionSizeFlag = DRF_DEF(2080_CTRL_GPU, _PARTITION_FLAG, _MEMORY_SIZE, _QUARTER);
465 break;
466 }
467
468 case 7:
469 case 8:
470 case 9:
471 case 10:
472 case 11:
473 case 12:
474 case 13:
475 case 14:
476 {
477 *pSizeInBytes = memSize;
478 *pPartitionSizeFlag = DRF_DEF(2080_CTRL_GPU, _PARTITION_FLAG, _MEMORY_SIZE, _EIGHTH);
479 break;
480 }
481
482 default:
483 {
484 NV_PRINTF(LEVEL_ERROR, "Unsupported SwizzId %d\n", swizzId);
485 DBG_BREAKPOINT();
486 return NV_ERR_INVALID_ARGUMENT;
487 }
488 }
489
490 if ((*pSizeInBytes == 0) &&
491 !pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB) &&
492 !pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB))
493 {
494 NV_PRINTF(LEVEL_ERROR, "Insufficient memory\n");
495 DBG_BREAKPOINT();
496 return NV_ERR_INSUFFICIENT_RESOURCES;
497 }
498 return NV_OK;
499 }
500
501 /*!
502 * @brief Function to map swizzId to mem range given total range
503 */
504 NV_STATUS
kmemsysSwizzIdToMIGMemRange_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NV_RANGE totalRange,NV_RANGE * pAddrRange)505 kmemsysSwizzIdToMIGMemRange_IMPL
506 (
507 OBJGPU *pGpu,
508 KernelMemorySystem *pKernelMemorySystem,
509 NvU32 swizzId,
510 NV_RANGE totalRange,
511 NV_RANGE *pAddrRange
512 )
513 {
514 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
515 NV_STATUS rmStatus = NV_OK;
516 NvU32 memSizeFlag = 0;
517 NvU32 minSwizzId = 0;
518 NvU64 unalignedStartAddr = 0;
519 NvU64 memSize = 0;
520 NV_RANGE swizzIdRange = NV_RANGE_EMPTY;
521
522 NV_ASSERT_OR_RETURN(!rangeIsEmpty(totalRange), NV_ERR_INVALID_ARGUMENT);
523
524 // Get SwizzId to size mapping
525 NV_ASSERT_OK_OR_RETURN(
526 kmemsysSwizzIdToMIGMemSize(pGpu, pKernelMemorySystem, swizzId, totalRange, &memSizeFlag, &memSize));
527
528 swizzIdRange = kmigmgrMemSizeFlagToSwizzIdRange_HAL(pGpu, pKernelMIGManager, memSizeFlag);
529 NV_ASSERT_OR_RETURN(!rangeIsEmpty(swizzIdRange), NV_ERR_INVALID_ARGUMENT);
530
531 minSwizzId = swizzIdRange.lo;
532
533 unalignedStartAddr = (totalRange.lo + (memSize * (swizzId - minSwizzId)));
534 *pAddrRange = rangeMake(unalignedStartAddr, unalignedStartAddr + memSize - 1);
535
536 return rmStatus;
537 }
538
539 /*!
540 * @brief Function to return GPU instance memory address range
541 */
542 NV_STATUS
kmemsysGetMIGGPUInstanceMemInfo_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NV_RANGE * pAddrRange)543 kmemsysGetMIGGPUInstanceMemInfo_IMPL
544 (
545 OBJGPU *pGpu,
546 KernelMemorySystem *pKernelMemorySystem,
547 NvU32 swizzId,
548 NV_RANGE *pAddrRange
549 )
550 {
551 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
552 NvU64 vmmuSegmentSize;
553 NvU64 startAddr;
554 NvU64 endAddr;
555 NvU64 partitionSize;
556
557 NV_ASSERT_OR_RETURN(pAddrRange != NULL, NV_ERR_INVALID_ARGUMENT);
558 *pAddrRange = NV_RANGE_EMPTY;
559 NV_ASSERT_OR_RETURN(swizzId < KMIGMGR_MAX_GPU_SWIZZID, NV_ERR_INVALID_STATE);
560
561 // Not supported in vGPU or ZERO_FB configs
562 NV_CHECK_OR_RETURN(LEVEL_SILENT,
563 !(IS_VIRTUAL(pGpu) || (pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB))),
564 NV_OK);
565
566 //
567 // VMMU not supported in AMODEL. Use legacy swizz-ID calculation instead of relying on vMMU segments
568 // to calculate address range
569 //
570 if (IsAMODEL(pGpu))
571 {
572 NV_RANGE partitionableMemoryRange = memmgrGetMIGPartitionableMemoryRange(pGpu, pMemoryManager);
573 return kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, partitionableMemoryRange, pAddrRange);
574 }
575
576 // Get the VMMU segment size
577 vmmuSegmentSize = gpuGetVmmuSegmentSize(pGpu);
578 NV_ASSERT_OR_RETURN((vmmuSegmentSize != 0), NV_ERR_INVALID_STATE);
579
580 startAddr = pKernelMemorySystem->gpuInstanceMemConfig[swizzId].startingVmmuSegment * vmmuSegmentSize;
581 partitionSize = pKernelMemorySystem->gpuInstanceMemConfig[swizzId].memSizeInVmmuSegment * vmmuSegmentSize;
582
583 if (osNumaOnliningEnabled(pGpu->pOsGpuInfo))
584 {
585 NvU64 memblockSize;
586 NvU64 alignedStartAddr;
587
588 NV_ASSERT_OK_OR_RETURN(osNumaMemblockSize(&memblockSize));
589
590 //
591 // Align the partition start address and size to memblock size
592 // Some FB memory is wasted here if it is not already aligned.
593 //
594 alignedStartAddr = NV_ALIGN_UP64(startAddr, memblockSize);
595
596 if(pKernelMemorySystem->bNumaMigPartitionSizeEnumerated)
597 {
598 partitionSize = pKernelMemorySystem->numaMigPartitionSize[swizzId];
599 }
600 else
601 {
602 partitionSize -= (alignedStartAddr - startAddr);
603 }
604
605 partitionSize = NV_ALIGN_DOWN64(partitionSize, memblockSize);
606 startAddr = alignedStartAddr;
607 }
608
609 endAddr = startAddr + partitionSize - 1;
610
611 *pAddrRange = rangeMake(startAddr, endAddr);
612
613 return NV_OK;
614 }
615
616 /**
617 * @brief Modifies numaMigPartitionSize array such that memory size of
618 all the mig partitions with swizzId between startSwizzId and
619 endSwizzId is assigned the minimum value among all partition's
620 memory size.
621 *
622 * @param[IN] pKernelMemorySystem
623 * @param[IN] startSwizzId
624 * @param[IN] endSwizzId
625 *
626 */
627 static void
_kmemsysSetNumaMigPartitionSizeSubArrayToMinimumValue(KernelMemorySystem * pKernelMemorySystem,NvU64 startSwizzId,NvU64 endSwizzId)628 _kmemsysSetNumaMigPartitionSizeSubArrayToMinimumValue
629 (
630 KernelMemorySystem *pKernelMemorySystem,
631 NvU64 startSwizzId,
632 NvU64 endSwizzId
633 )
634 {
635 NvU64 minPartitionSize = pKernelMemorySystem->numaMigPartitionSize[startSwizzId];
636 NvU64 index;
637
638 for (index = startSwizzId; index <= endSwizzId; index++)
639 {
640 if(pKernelMemorySystem->numaMigPartitionSize[index] < minPartitionSize)
641 minPartitionSize = pKernelMemorySystem->numaMigPartitionSize[index];
642 }
643
644 for (index = startSwizzId; index <= endSwizzId; index++)
645 {
646 pKernelMemorySystem->numaMigPartitionSize[index] = minPartitionSize;
647 }
648 }
649
650 /*!
651 * @brief Function to populate static GPU instance memory config which will be
652 * utilized for GPU instance memory query and memory allocation
653 */
654 NV_STATUS
kmemsysPopulateMIGGPUInstanceMemConfig_KERNEL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)655 kmemsysPopulateMIGGPUInstanceMemConfig_KERNEL
656 (
657 OBJGPU *pGpu,
658 KernelMemorySystem *pKernelMemorySystem
659 )
660 {
661 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
662 NV_RANGE partitionableMemoryRange = memmgrGetMIGPartitionableMemoryRange(pGpu, pMemoryManager);
663 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
664 NvU64 vmmuSegmentSize;
665 NvU64 totalVmmuSegments;
666 NvU64 alignedStartAddr;
667 NvU64 alignedEndAddr;
668 NvU32 swizzId;
669
670 // Not needed in vGPU or zero_fb configs
671 NV_CHECK_OR_RETURN(LEVEL_SILENT,
672 !(IS_VIRTUAL(pGpu) || (pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB))),
673 NV_OK);
674
675 // Nothing to do if MIG is not supported
676 NV_CHECK_OR_RETURN(LEVEL_SILENT, kmigmgrIsMIGSupported(pGpu, pKernelMIGManager), NV_OK);
677
678 // Get the VMMU segment size
679 vmmuSegmentSize = gpuGetVmmuSegmentSize(pGpu);
680 NV_ASSERT_OR_RETURN((vmmuSegmentSize != 0), NV_ERR_INVALID_STATE);
681
682 alignedStartAddr = partitionableMemoryRange.lo;
683 alignedEndAddr = partitionableMemoryRange.hi;
684 if (alignedStartAddr != 0)
685 {
686 alignedStartAddr = NV_IS_ALIGNED64(alignedStartAddr, vmmuSegmentSize) ?
687 alignedStartAddr + vmmuSegmentSize :
688 NV_ALIGN_UP64(alignedStartAddr, vmmuSegmentSize);
689 }
690
691 if (NV_IS_ALIGNED64(alignedEndAddr + 1, vmmuSegmentSize))
692 {
693 alignedEndAddr = alignedEndAddr - vmmuSegmentSize;
694 }
695
696 totalVmmuSegments = (alignedEndAddr - alignedStartAddr + 1) / vmmuSegmentSize;
697 for (swizzId = 0; swizzId < KMIGMGR_MAX_GPU_SWIZZID; swizzId++)
698 {
699 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
700 kmemsysSwizzIdToVmmuSegmentsRange_HAL(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments));
701 }
702
703 if (osNumaOnliningEnabled(pGpu->pOsGpuInfo))
704 {
705 NV_RANGE addrRange = NV_RANGE_EMPTY;
706 NvU32 memSize;
707
708 for(swizzId = 0; swizzId < KMIGMGR_MAX_GPU_SWIZZID; swizzId++)
709 {
710 kmemsysGetMIGGPUInstanceMemInfo(pGpu, pKernelMemorySystem, swizzId, &addrRange);
711 pKernelMemorySystem->numaMigPartitionSize[swizzId] = addrRange.hi - addrRange.lo + 1;
712 }
713
714 //
715 // In GH180 for all the swizzId's for a given memory profile (FULL, HALF, QUARTER
716 // and EIGHTH partitions) might not be same. Modify numaMigPartitionSize array
717 // for the partition size to be constant for a given profile. BUG 4284299.
718 //
719 for (memSize = NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE_FULL; memSize < NV2080_CTRL_GPU_PARTITION_FLAG_MEMORY_SIZE__SIZE; memSize++)
720 {
721 NV_RANGE swizzRange = kmigmgrMemSizeFlagToSwizzIdRange(pGpu, pKernelMIGManager,
722 DRF_NUM(2080_CTRL_GPU, _PARTITION_FLAG, _MEMORY_SIZE, memSize));
723 _kmemsysSetNumaMigPartitionSizeSubArrayToMinimumValue(pKernelMemorySystem, swizzRange.lo, swizzRange.hi);
724 }
725 pKernelMemorySystem->bNumaMigPartitionSizeEnumerated = NV_TRUE;
726 }
727 return NV_OK;
728 }
729
730 /*!
731 * @brief Gets GPU instance memory configuration based on swizzId
732 */
733 NV_STATUS
kmemsysGetMIGGPUInstanceMemConfigFromSwizzId_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,const MIG_GPU_INSTANCE_MEMORY_CONFIG ** ppGPUInstanceMemConfig)734 kmemsysGetMIGGPUInstanceMemConfigFromSwizzId_IMPL
735 (
736 OBJGPU *pGpu,
737 KernelMemorySystem *pKernelMemorySystem,
738 NvU32 swizzId,
739 const MIG_GPU_INSTANCE_MEMORY_CONFIG **ppGPUInstanceMemConfig
740 )
741 {
742 NV_ASSERT_OR_RETURN(swizzId < KMIGMGR_MAX_GPU_SWIZZID, NV_ERR_INVALID_ARGUMENT);
743
744 if (IS_VIRTUAL(pGpu))
745 {
746 // VMMU Segment details are populated on Host and not Guest.
747 return NV_ERR_NOT_SUPPORTED;
748 }
749
750 // MODS makes a control call to describe GPU instances before this is populated. Return invalid data anyways
751 NV_ASSERT_OR_RETURN(pKernelMemorySystem->gpuInstanceMemConfig[swizzId].bInitialized, NV_ERR_INVALID_STATE);
752
753 *ppGPUInstanceMemConfig = &pKernelMemorySystem->gpuInstanceMemConfig[swizzId];
754 return NV_OK;
755 }
756
757 /*!
758 * @brief Set GPU Instance memory config information and mark initialized
759 */
760 NV_STATUS
kmemsysInitMIGGPUInstanceMemConfigForSwizzId_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU64 startingVmmuSegment,NvU64 memSizeInVmmuSegment)761 kmemsysInitMIGGPUInstanceMemConfigForSwizzId_IMPL
762 (
763 OBJGPU *pGpu,
764 KernelMemorySystem *pKernelMemorySystem,
765 NvU32 swizzId,
766 NvU64 startingVmmuSegment,
767 NvU64 memSizeInVmmuSegment
768 )
769 {
770 NV_ASSERT_OR_RETURN(swizzId < KMIGMGR_MAX_GPU_SWIZZID, NV_ERR_INVALID_ARGUMENT);
771
772 pKernelMemorySystem->gpuInstanceMemConfig[swizzId].startingVmmuSegment = startingVmmuSegment;
773 pKernelMemorySystem->gpuInstanceMemConfig[swizzId].memSizeInVmmuSegment = memSizeInVmmuSegment;
774 pKernelMemorySystem->gpuInstanceMemConfig[swizzId].bInitialized = NV_TRUE;
775
776 NV_PRINTF(LEVEL_INFO,
777 "GPU Instance Mem Config for swizzId = 0x%x : MemStartSegment = 0x%llx, MemSizeInSegments = 0x%llx\n",
778 swizzId,
779 pKernelMemorySystem->gpuInstanceMemConfig[swizzId].startingVmmuSegment,
780 pKernelMemorySystem->gpuInstanceMemConfig[swizzId].memSizeInVmmuSegment);
781
782 return NV_OK;
783 }
784
785 /*!
786 * @brief Ensure that the sysmem flush sysmem buffer has been initialized
787 *
788 * Setting up the sysmem flush buffer needs to be done very early in some cases
789 * as it's required for the GPU to perform a system flush. One such case is
790 * resetting GPU FALCONs and in particular resetting the PMU as part of VBIOS
791 * init.
792 *
793 * @returns NV_OK if the sysmem flush buffer has been initialized.
794 */
795 NV_STATUS
kmemsysEnsureSysmemFlushBufferInitialized_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)796 kmemsysEnsureSysmemFlushBufferInitialized_IMPL
797 (
798 OBJGPU *pGpu,
799 KernelMemorySystem *pKernelMemorySystem
800 )
801 {
802 if (IS_VIRTUAL(pGpu) ||
803 IS_GSP_CLIENT(pGpu) ||
804 RMCFG_FEATURE_PLATFORM_GSP)
805 {
806 return NV_OK;
807 }
808
809 return kmemsysInitFlushSysmemBuffer_HAL(pGpu, pKernelMemorySystem);
810 }
811
812 /*!
813 * @brief Handle sysmem NVLink/C2C, NUMA and ATS functionality
814 *
815 * @param[in] pGpu OBJGPU pointer
816 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance.
817 * @param[in] bFlush Whether the CPU cache of the GPU mapping
818 * should be flushed
819 *
820 * @return NV_OK on success
821 */
822 NV_STATUS
kmemsysSetupCoherentCpuLink_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvBool bFlush)823 kmemsysSetupCoherentCpuLink_IMPL
824 (
825 OBJGPU *pGpu,
826 KernelMemorySystem *pKernelMemorySystem,
827 NvBool bFlush
828 )
829 {
830 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
831 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
832 NvU64 numaOnlineSize = 0;
833 NvU64 fbSize = (pMemoryManager->Ram.fbTotalMemSizeMb << 20);
834 NvU32 data32;
835 NvBool bCpuMapping = NV_TRUE; // Default enable
836 NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE;
837 NvU64 memblockSize = 0;
838 NvU64 numaOnlineBase = 0;
839 NvU64 rsvdFastSize = 0;
840 NvU64 rsvdSlowSize = 0;
841 NvU64 rsvdISOSize = 0;
842 NvU64 totalRsvdBytes = 0;
843
844 {
845 NV_ASSERT_OK_OR_RETURN(kmemsysGetFbNumaInfo_HAL(pGpu, pKernelMemorySystem,
846 &pKernelMemorySystem->coherentCpuFbBase,
847 &pKernelMemorySystem->coherentRsvdFbBase,
848 &numaNodeId));
849 if (pKernelMemorySystem->coherentCpuFbBase != 0)
850 {
851 if (gpuIsSelfHosted(pGpu))
852 {
853 //
854 // For self-hosted, coherentCpuFbEnd is only till the FB size
855 // end and NOT till the FB AMAP end since self-hosted doesn't
856 // support indirect peer and requires GPU nvlink for peer.
857 //
858 pKernelMemorySystem->coherentCpuFbEnd = pKernelMemorySystem->coherentCpuFbBase + fbSize;
859 }
860 else
861 {
862 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
863 NV2080_CTRL_INTERNAL_GET_COHERENT_FB_APERTURE_SIZE_PARAMS params = {0};
864
865 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
866 pGpu->hInternalClient,
867 pGpu->hInternalSubdevice,
868 NV2080_CTRL_CMD_INTERNAL_GET_COHERENT_FB_APERTURE_SIZE,
869 ¶ms,
870 sizeof(NV2080_CTRL_INTERNAL_GET_COHERENT_FB_APERTURE_SIZE_PARAMS)));
871 //
872 // Indirect peer(uses P9 to reach other GV100) in P9+GV100 requires coherentCpuFbEnd to
873 // also include the entire FB AMAP range even when FB size is less than the FB AMAP size.
874 //
875 pKernelMemorySystem->coherentCpuFbEnd = pKernelMemorySystem->coherentCpuFbBase +
876 params.coherentFbApertureSize;
877 }
878 }
879 }
880
881 if ((osReadRegistryDword(pGpu,
882 NV_REG_STR_OVERRIDE_GPU_NUMA_NODE_ID, &data32)) == NV_OK)
883 {
884 numaNodeId = (NvS32)data32;
885 NV_PRINTF(LEVEL_ERROR, "Override GPU NUMA node ID %d!\n", numaNodeId);
886 }
887
888 // Parse regkey here
889 if ((osReadRegistryDword(pGpu,
890 NV_REG_STR_RM_FORCE_BAR_PATH, &data32) == NV_OK) &&
891 (data32 == 1))
892 {
893 NV_PRINTF(LEVEL_ERROR,
894 "Force disabling NVLINK/C2C mappings through regkey.\n");
895
896 bCpuMapping = NV_FALSE;
897 }
898
899 if ((pKernelMemorySystem->coherentCpuFbBase == 0) || !bCpuMapping)
900 {
901 return NV_OK;
902 }
903
904 NV_ASSERT_OK_OR_RETURN(osNumaMemblockSize(&memblockSize));
905
906 memmgrCalcReservedFbSpaceHal_HAL(pGpu, pMemoryManager, &rsvdFastSize, &rsvdSlowSize, &rsvdISOSize);
907
908 //
909 // Calculate the size of the memory which can be safely onlined to the
910 // kernel after accounting for different reserved memory requirements.
911 //
912 // Align rsvd memory to 64K granularity.
913 // TODO : rsvdMemorySize is not finalized at this point of time in
914 // GH180, currently rsvdMemorySize is not increasing after this
915 // point. This needs to be fixed.
916 //
917 totalRsvdBytes += NV_ALIGN_UP(pMemoryManager->rsvdMemorySize, 0x10000);
918 totalRsvdBytes += (rsvdFastSize + rsvdSlowSize + rsvdISOSize);
919 totalRsvdBytes += pMemoryManager->Ram.reservedMemSize;
920
921 // For SRIOV guest, take into account FB tax paid on host side for each VF
922 // This FB tax is non zero only for SRIOV guest RM environment.
923 totalRsvdBytes += memmgrGetFbTaxSize_HAL(pGpu, pMemoryManager);
924
925 //
926 // TODO: make sure the onlineable memory is aligned to memblockSize
927 // Currently, if we have leftover memory, it'll just be wasted because no
928 // one can access it. If FB size itself is memblock size unaligned(because
929 // of CBC and row remapper deductions), then the memory wastage is unavoidable.
930 //
931 numaOnlineSize = NV_ALIGN_DOWN64(fbSize - totalRsvdBytes, memblockSize);
932
933 if (IS_PASSTHRU(pGpu) && pKernelMemorySystem->bBug3656943WAR)
934 {
935 // For passthrough case, reserved memory size is fixed as 1GB
936 NvU64 rsvdSize = 1 * 1024 * 1024 * 1024;
937
938 NV_ASSERT_OR_RETURN(rsvdSize >= totalRsvdBytes, NV_ERR_INVALID_STATE);
939 totalRsvdBytes = rsvdSize;
940 //
941 // Aligning to hardcoded 512MB size as both host and guest need to use
942 // the same alignment irrespective of the kernel page size. 512MB size
943 // works for both 4K and 64K page size kernels but more memory is
944 // wasted being part of non onlined region which can't be avoided
945 // per the design.
946 //
947 numaOnlineSize = NV_ALIGN_DOWN64(fbSize - totalRsvdBytes, 512 * 1024 * 1024);
948 }
949
950
951 NV_PRINTF(LEVEL_INFO,
952 "fbSize: 0x%llx NUMA reserved memory size: 0x%llx online memory size: 0x%llx\n",
953 fbSize, totalRsvdBytes, numaOnlineSize);
954 if (osNumaOnliningEnabled(pGpu->pOsGpuInfo))
955 {
956 pKernelMemorySystem->numaOnlineBase = numaOnlineBase;
957 pKernelMemorySystem->numaOnlineSize = numaOnlineSize;
958 //
959 // TODO: Bug 1945658: Soldier through on GPU memory add
960 // failure(which is often possible because of missing auto online
961 // setting) and instead check for failure on stateLoad.
962 // Any failure in StateInit results in gpuStateDestroy not getting called.
963 // kgspUnloadRm_IMPL from gpuStateDestroy also doesn't get called leaving
964 // GSP in unclean state and requiring GPU reset to recover from that.
965 //
966 // kmemsysNumaAddMemory_HAL by itself cannot be called from stateLoad
967 // because the memory mapping that follows this call site comes from linear
968 // kernel virtual address when memory is added to the kernel vs the
969 // VMALLOC_START region when memory is not added.
970 //
971 NV_ASSERT_OK(kmemsysNumaAddMemory_HAL(pGpu, pKernelMemorySystem, 0, 0,
972 numaOnlineSize, &numaNodeId));
973 }
974 pGpu->numaNodeId = numaNodeId;
975
976 NV_ASSERT_OK_OR_RETURN(kbusCreateCoherentCpuMapping_HAL(pGpu, pKernelBus, numaOnlineSize, bFlush));
977
978 // Switch the toggle for coherent link mapping only if migration is successful
979 pGpu->setProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING, NV_TRUE);
980
981 NV_ASSERT_OK_OR_RETURN(kbusVerifyCoherentLink_HAL(pGpu, pKernelBus));
982
983 return NV_OK;
984 }
985
986 /*!
987 * @brief Teardown sysmem NVLink/C2C NUMA and ATS functionality
988 *
989 * @param[in] pGpu OBJGPU pointer
990 * @param[in] pKernelMemorySystem Kernel Memory System pointer
991 * @param[in] bFlush Whether the CPU cache of the GPU mapping
992 * should be flushed
993 */
994 void
kmemsysTeardownCoherentCpuLink_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvBool bFlush)995 kmemsysTeardownCoherentCpuLink_IMPL
996 (
997 OBJGPU *pGpu,
998 KernelMemorySystem *pKernelMemorySystem,
999 NvBool bFlush
1000 )
1001 {
1002 kbusTeardownCoherentCpuMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), bFlush);
1003 pGpu->setProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING, NV_FALSE);
1004 }
1005
1006 NV_STATUS
kmemsysSendL2InvalidateEvict_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)1007 kmemsysSendL2InvalidateEvict_IMPL
1008 (
1009 OBJGPU *pGpu,
1010 KernelMemorySystem *pKernelMemorySystem,
1011 NvU32 flags)
1012 {
1013 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1014 NV2080_CTRL_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT_PARAMS params = {0};
1015
1016 params.flags = flags;
1017
1018 return pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
1019 NV2080_CTRL_CMD_INTERNAL_MEMSYS_L2_INVALIDATE_EVICT,
1020 ¶ms, sizeof(params));
1021 }
1022
1023 NV_STATUS
kmemsysSendFlushL2AllRamsAndCaches_IMPL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)1024 kmemsysSendFlushL2AllRamsAndCaches_IMPL
1025 (
1026 OBJGPU *pGpu,
1027 KernelMemorySystem *pKernelMemorySystem
1028 )
1029 {
1030 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
1031
1032 return pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
1033 NV2080_CTRL_CMD_INTERNAL_MEMSYS_FLUSH_L2_ALL_RAMS_AND_CACHES,
1034 NULL, 0);
1035 }
1036
1037 NV_STATUS
kmemsysGetUsableFbSize_KERNEL(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU64 * pFbSize)1038 kmemsysGetUsableFbSize_KERNEL
1039 (
1040 OBJGPU *pGpu,
1041 KernelMemorySystem *pKernelMemorySystem,
1042 NvU64 *pFbSize
1043 )
1044 {
1045 return kmemsysReadUsableFbSize_HAL(pGpu, pKernelMemorySystem, pFbSize);
1046 }
1047
1048 NV_STATUS
kmemsysStateLoad_VF(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)1049 kmemsysStateLoad_VF(OBJGPU *pGpu, KernelMemorySystem *pKernelMemorySystem, NvU32 flags)
1050 {
1051 NV_STATUS status = NV_OK;
1052
1053 if (flags & GPU_STATE_FLAGS_PRESERVING)
1054 {
1055 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1056
1057 NV_ASSERT(!(flags & GPU_STATE_FLAGS_GC6_TRANSITION));
1058
1059 status = memmgrRestorePowerMgmtState(pGpu, pMemoryManager);
1060 if (status != NV_OK)
1061 memmgrFreeFbsrMemory(pGpu, pMemoryManager);
1062
1063 NV_ASSERT_OK(status);
1064 }
1065
1066 return status;
1067 }
1068
1069 NV_STATUS
kmemsysStateUnload_VF(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 flags)1070 kmemsysStateUnload_VF(OBJGPU *pGpu, KernelMemorySystem *pKernelMemorySystem, NvU32 flags)
1071 {
1072 NV_STATUS status = NV_OK;
1073
1074 if (flags & GPU_STATE_FLAGS_PRESERVING)
1075 {
1076 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1077
1078 NV_ASSERT(!(flags & GPU_STATE_FLAGS_GC6_TRANSITION));
1079
1080 status = memmgrSavePowerMgmtState(pGpu, pMemoryManager);
1081 if (status != NV_OK)
1082 memmgrFreeFbsrMemory(pGpu, pMemoryManager);
1083
1084 NV_ASSERT_OK(status);
1085 }
1086
1087 return status;
1088 }
1089