1 #ifndef _G_KERN_MEM_SYS_NVOC_H_ 2 #define _G_KERN_MEM_SYS_NVOC_H_ 3 #include "nvoc/runtime.h" 4 5 #ifdef __cplusplus 6 extern "C" { 7 #endif 8 9 /* 10 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 11 * SPDX-License-Identifier: MIT 12 * 13 * Permission is hereby granted, free of charge, to any person obtaining a 14 * copy of this software and associated documentation files (the "Software"), 15 * to deal in the Software without restriction, including without limitation 16 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 17 * and/or sell copies of the Software, and to permit persons to whom the 18 * Software is furnished to do so, subject to the following conditions: 19 * 20 * The above copyright notice and this permission notice shall be included in 21 * all copies or substantial portions of the Software. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 26 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 29 * DEALINGS IN THE SOFTWARE. 30 */ 31 32 #include "g_kern_mem_sys_nvoc.h" 33 34 #ifndef KERN_MEM_SYS_H 35 #define KERN_MEM_SYS_H 36 37 #include "core/core.h" 38 #include "gpu/eng_state.h" 39 #include "gpu/gpu.h" 40 #include "containers/map.h" 41 #include "gpu/mem_mgr/heap_base.h" 42 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 43 #include "ctrl/ctrl2080/ctrl2080fb.h" 44 #include "ctrl/ctrl2080/ctrl2080internal.h" 45 #include "ctrl/ctrl2080/ctrl2080fb.h" 46 // 47 // FB Cache (opcode, mem target) defines used by kmemsysCacheOp hal API 48 // 49 typedef enum 50 { 51 FB_CACHE_OP_UNDEFINED = 0, 52 // invalidate cache lines without writeback of dirty lines to memory 53 FB_CACHE_INVALIDATE = 1, 54 // writeback dirty lines but leave the lines in valid cache state 55 FB_CACHE_WRITEBACK, 56 // writeback dirty lines and then invalidates the cache state 57 FB_CACHE_EVICT, 58 } FB_CACHE_OP; 59 60 // target memory types for cache operations 61 typedef enum 62 { 63 FB_CACHE_MEM_UNDEFINED = 0, 64 FB_CACHE_SYSTEM_MEMORY = 1, 65 FB_CACHE_VIDEO_MEMORY, 66 FB_CACHE_PEER_MEMORY, 67 FB_CACHE_DIRTY, 68 FB_CACHE_COMPTAG_MEMORY, 69 FB_CACHE_DIRTY_ALL, 70 } FB_CACHE_MEMTYPE; 71 72 typedef enum 73 { 74 FB_CACHE_STATE_ENABLED, 75 FB_CACHE_STATE_DISABLED, 76 } FB_CACHE_STATE; 77 78 typedef enum 79 { 80 FB_CACHE_WRITE_MODE_WRITETHROUGH, 81 FB_CACHE_WRITE_MODE_WRITEBACK, 82 } FB_CACHE_WRITE_MODE; 83 84 typedef enum 85 { 86 FB_CACHE_BYPASS_MODE_ENABLED, 87 FB_CACHE_BYPASS_MODE_DISABLED, 88 } FB_CACHE_BYPASS_MODE; // FERMI (TEST) ONLY 89 90 typedef enum 91 { 92 FB_CACHE_RCM_STATE_FULL, 93 FB_CACHE_RCM_STATE_TRANSITIONING, 94 FB_CACHE_RCM_STATE_REDUCED, 95 FB_CACHE_RCM_STATE_ZERO_CACHE, 96 } FB_CACHE_RCM_STATE; 97 98 /*! Tracks NUMA information of GPU memory partitions */ 99 typedef struct 100 { 101 NvBool bInUse; // Is the partition in use? 102 NvU64 offset; // FB offset of the partition 103 NvU64 size; // FB size of the partition 104 NvU32 numaNodeId; // OS NUMA Node Id of the partition. 105 } MEM_PARTITION_NUMA_INFO; 106 107 typedef struct MIG_MEM_BOUNDARY_CONFIG_TABLE 108 { 109 /*! 110 * Memory boundary config A (4KB aligned) 111 */ 112 NvU64 memBoundaryCfgA; 113 114 /*! 115 * Memory boundary config B (4KB aligned) 116 */ 117 NvU64 memBoundaryCfgB; 118 119 /*! 120 * Memory boundary config C (64KB aligned) 121 */ 122 NvU32 memBoundaryCfgC; 123 } MIG_MEM_BOUNDARY_CONFIG_TABLE; 124 125 /*! 126 * @brief Structure carrying memory configuration information for specific GPU instance 127 * The information will be used to allocate memory when a GPU instance is 128 * created or queried. The structure will be indexed with swizzIDs 129 */ 130 typedef struct MIG_GPU_INSTANCE_MEMORY_CONFIG 131 { 132 /*! 133 * First VMMU segment from where the GPU instance memory starts 134 */ 135 NvU64 startingVmmuSegment; 136 137 /*! 138 * Size of the GPU instance memory in the form of number of vmmu segments 139 */ 140 NvU64 memSizeInVmmuSegment; 141 142 /*! 143 * GPU Instance memory config initialization state 144 */ 145 NvBool bInitialized; 146 } MIG_GPU_INSTANCE_MEMORY_CONFIG; 147 148 /* @ref NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS */ 149 typedef NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS MEMORY_SYSTEM_STATIC_CONFIG; 150 151 #define FB_HWRESID_CTAGID_FERMI 15:0 152 #define FB_HWRESID_ZCULL_FERMI 30:16 153 154 #define FB_HWRESID_ZCULL_SHIFT_FERMI(i) (1 << (i)) 155 156 #define FB_HWRESID_CTAGID_VAL_FERMI(n) \ 157 (((n) >> DRF_SHIFT(FB_HWRESID_CTAGID_FERMI)) & DRF_MASK(FB_HWRESID_CTAGID_FERMI)) 158 159 #define FB_HWRESID_CTAGID_NUM_FERMI(i) \ 160 (((i) & DRF_MASK(FB_HWRESID_CTAGID_FERMI)) << DRF_SHIFT(FB_HWRESID_CTAGID_FERMI)) 161 162 #define FB_SET_HWRESID_CTAGID_FERMI(h, i) \ 163 h = ( ((h) & ~(DRF_MASK(FB_HWRESID_CTAGID_FERMI) << DRF_SHIFT(FB_HWRESID_CTAGID_FERMI))) | \ 164 FB_HWRESID_CTAGID_NUM_FERMI(i) ) 165 166 #define FB_HWRESID_ZCULL_NUM_FERMI(i) \ 167 (((1<<i) & DRF_MASK(FB_HWRESID_ZCULL_FERMI)) << DRF_SHIFT(FB_HWRESID_ZCULL_FERMI)) 168 169 #define FB_HWRESID_ZCULL_VAL_FERMI(n) \ 170 (((n) >> DRF_SHIFT(FB_HWRESID_ZCULL_FERMI)) & DRF_MASK(FB_HWRESID_ZCULL_FERMI)) 171 172 /*! 173 * KernelMemorySystem is a logical abstraction of the GPU memory system. This 174 * type is instantiated in VGPU guest/GSP Client as well as the VGPU 175 * host/GSP-RM. 176 * 177 * When KernelMemorySystem wants to read or write hardware state, it does not 178 * have access to the registers on the GPU, it can however perform operations 179 * using the following mechanisms: 180 * 181 * 1.) access registers are virtualized across VFs, e.g.: registers within 182 * NV_VIRTUAL_FUNCTION_PRIV_XYZ. 183 * 184 * 2.) send a RPC to the VGPU Host/GSP-RM to perform the operation. 185 * 186 * Operations such as "get memory system bus width" are appropriate for this 187 * interface. Anything related to managing of the memory page 188 * tables/allocations should live in MemoryManager. 189 */ 190 191 192 // Private field names are wrapped in PRIVATE_FIELD, which does nothing for 193 // the matching C source file, but causes diagnostics to be issued if another 194 // source file references the field. 195 #ifdef NVOC_KERN_MEM_SYS_H_PRIVATE_ACCESS_ALLOWED 196 #define PRIVATE_FIELD(x) x 197 #else 198 #define PRIVATE_FIELD(x) NVOC_PRIVATE_FIELD(x) 199 #endif 200 201 struct KernelMemorySystem { 202 const struct NVOC_RTTI *__nvoc_rtti; 203 struct OBJENGSTATE __nvoc_base_OBJENGSTATE; 204 struct Object *__nvoc_pbase_Object; 205 struct OBJENGSTATE *__nvoc_pbase_OBJENGSTATE; 206 struct KernelMemorySystem *__nvoc_pbase_KernelMemorySystem; 207 NV_STATUS (*__kmemsysConstructEngine__)(OBJGPU *, struct KernelMemorySystem *, ENGDESCRIPTOR); 208 NV_STATUS (*__kmemsysStateInitLocked__)(OBJGPU *, struct KernelMemorySystem *); 209 NV_STATUS (*__kmemsysStatePreLoad__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 210 NV_STATUS (*__kmemsysStatePostLoad__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 211 NV_STATUS (*__kmemsysStateLoad__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 212 NV_STATUS (*__kmemsysStatePreUnload__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 213 NV_STATUS (*__kmemsysStateUnload__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 214 void (*__kmemsysStateDestroy__)(OBJGPU *, struct KernelMemorySystem *); 215 NV_STATUS (*__kmemsysGetFbNumaInfo__)(OBJGPU *, struct KernelMemorySystem *, NvU64 *, NvU64 *, NvS32 *); 216 NV_STATUS (*__kmemsysReadUsableFbSize__)(OBJGPU *, struct KernelMemorySystem *, NvU64 *); 217 NV_STATUS (*__kmemsysGetUsableFbSize__)(OBJGPU *, struct KernelMemorySystem *, NvU64 *); 218 NV_STATUS (*__kmemsysCacheOp__)(OBJGPU *, struct KernelMemorySystem *, PMEMORY_DESCRIPTOR, FB_CACHE_MEMTYPE, FB_CACHE_OP); 219 NV_STATUS (*__kmemsysDoCacheOp__)(OBJGPU *, struct KernelMemorySystem *, NvU32, NvU32, NvU32, PRMTIMEOUT); 220 NvU32 (*__kmemsysReadL2SysmemInvalidateReg__)(OBJGPU *, struct KernelMemorySystem *); 221 void (*__kmemsysWriteL2SysmemInvalidateReg__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 222 NvU32 (*__kmemsysReadL2PeermemInvalidateReg__)(OBJGPU *, struct KernelMemorySystem *); 223 void (*__kmemsysWriteL2PeermemInvalidateReg__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 224 NV_STATUS (*__kmemsysInitFlushSysmemBuffer__)(OBJGPU *, struct KernelMemorySystem *); 225 void (*__kmemsysProgramSysmemFlushBuffer__)(OBJGPU *, struct KernelMemorySystem *); 226 NvU32 (*__kmemsysGetFlushSysmemBufferAddrShift__)(OBJGPU *, struct KernelMemorySystem *); 227 NvBool (*__kmemsysIsPagePLCable__)(OBJGPU *, struct KernelMemorySystem *, NvU64, NvU64); 228 NV_STATUS (*__kmemsysReadMIGMemoryCfg__)(OBJGPU *, struct KernelMemorySystem *); 229 NV_STATUS (*__kmemsysInitMIGMemoryPartitionTable__)(OBJGPU *, struct KernelMemorySystem *); 230 NV_STATUS (*__kmemsysSwizzIdToVmmuSegmentsRange__)(OBJGPU *, struct KernelMemorySystem *, NvU32, NvU32, NvU32); 231 NV_STATUS (*__kmemsysNumaAddMemory__)(OBJGPU *, struct KernelMemorySystem *, NvU32, NvU64, NvU64, NvS32 *); 232 void (*__kmemsysNumaRemoveMemory__)(OBJGPU *, struct KernelMemorySystem *, NvU32); 233 void (*__kmemsysNumaRemoveAllMemory__)(OBJGPU *, struct KernelMemorySystem *); 234 NV_STATUS (*__kmemsysPopulateMIGGPUInstanceMemConfig__)(OBJGPU *, struct KernelMemorySystem *); 235 NV_STATUS (*__kmemsysSetupAllAtsPeers__)(OBJGPU *, struct KernelMemorySystem *); 236 void (*__kmemsysRemoveAllAtsPeers__)(OBJGPU *, struct KernelMemorySystem *); 237 NvBool (*__kmemsysAssertFbAckTimeoutPending__)(OBJGPU *, struct KernelMemorySystem *); 238 NvU32 (*__kmemsysGetMaxFbpas__)(OBJGPU *, struct KernelMemorySystem *); 239 NvU32 (*__kmemsysGetEccDedCountSize__)(OBJGPU *, struct KernelMemorySystem *); 240 NvU32 (*__kmemsysGetEccDedCountRegAddr__)(OBJGPU *, struct KernelMemorySystem *, NvU32, NvU32); 241 void (*__kmemsysGetEccCounts__)(OBJGPU *, struct KernelMemorySystem *, NvU32 *, NvU32 *); 242 void (*__kmemsysClearEccCounts__)(OBJGPU *, struct KernelMemorySystem *); 243 NvU16 (*__kmemsysGetMaximumBlacklistPages__)(OBJGPU *, struct KernelMemorySystem *); 244 NV_STATUS (*__kmemsysGetFbInfos__)(OBJGPU *, struct KernelMemorySystem *, struct RsClient *, Device *, NvHandle, NV2080_CTRL_FB_GET_INFO_V2_PARAMS *, NvU64 *); 245 NV_STATUS (*__kmemsysStatePostUnload__)(POBJGPU, struct KernelMemorySystem *, NvU32); 246 NV_STATUS (*__kmemsysStateInitUnlocked__)(POBJGPU, struct KernelMemorySystem *); 247 void (*__kmemsysInitMissing__)(POBJGPU, struct KernelMemorySystem *); 248 NV_STATUS (*__kmemsysStatePreInitLocked__)(POBJGPU, struct KernelMemorySystem *); 249 NV_STATUS (*__kmemsysStatePreInitUnlocked__)(POBJGPU, struct KernelMemorySystem *); 250 NvBool (*__kmemsysIsPresent__)(POBJGPU, struct KernelMemorySystem *); 251 NvBool bDisableTiledCachingInvalidatesWithEccBug1521641; 252 NvBool bGpuCacheEnable; 253 NvBool bNumaNodesAdded; 254 NvBool bL2CleanFbPull; 255 NvU32 l2WriteMode; 256 NvBool bPreserveComptagBackingStoreOnSuspend; 257 NvBool bBug3656943WAR; 258 NvU32 overrideToGMK; 259 NvBool bDisablePlcForCertainOffsetsBug3046774; 260 const MEMORY_SYSTEM_STATIC_CONFIG *pStaticConfig; 261 MEM_PARTITION_NUMA_INFO *memPartitionNumaInfo; 262 MIG_MEM_BOUNDARY_CONFIG_TABLE memBoundaryCfgTable; 263 MIG_GPU_INSTANCE_MEMORY_CONFIG gpuInstanceMemConfig[15]; 264 NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE_PARAMS migMemoryPartitionTable; 265 PMEMORY_DESCRIPTOR pSysmemFlushBufferMemDesc; 266 NvU64 sysmemFlushBuffer; 267 NvU64 fbOverrideStartKb; 268 NvU64 coherentCpuFbBase; 269 NvU64 coherentCpuFbEnd; 270 NvU64 coherentRsvdFbBase; 271 NvU64 numaOnlineBase; 272 NvU64 numaOnlineSize; 273 NvU64 numaMigPartitionSize[15]; 274 NvBool bNumaMigPartitionSizeEnumerated; 275 }; 276 277 #ifndef __NVOC_CLASS_KernelMemorySystem_TYPEDEF__ 278 #define __NVOC_CLASS_KernelMemorySystem_TYPEDEF__ 279 typedef struct KernelMemorySystem KernelMemorySystem; 280 #endif /* __NVOC_CLASS_KernelMemorySystem_TYPEDEF__ */ 281 282 #ifndef __nvoc_class_id_KernelMemorySystem 283 #define __nvoc_class_id_KernelMemorySystem 0x7faff1 284 #endif /* __nvoc_class_id_KernelMemorySystem */ 285 286 extern const struct NVOC_CLASS_DEF __nvoc_class_def_KernelMemorySystem; 287 288 #define __staticCast_KernelMemorySystem(pThis) \ 289 ((pThis)->__nvoc_pbase_KernelMemorySystem) 290 291 #ifdef __nvoc_kern_mem_sys_h_disabled 292 #define __dynamicCast_KernelMemorySystem(pThis) ((KernelMemorySystem*)NULL) 293 #else //__nvoc_kern_mem_sys_h_disabled 294 #define __dynamicCast_KernelMemorySystem(pThis) \ 295 ((KernelMemorySystem*)__nvoc_dynamicCast(staticCast((pThis), Dynamic), classInfo(KernelMemorySystem))) 296 #endif //__nvoc_kern_mem_sys_h_disabled 297 298 #define PDB_PROP_KMEMSYS_IS_MISSING_BASE_CAST __nvoc_base_OBJENGSTATE. 299 #define PDB_PROP_KMEMSYS_IS_MISSING_BASE_NAME PDB_PROP_ENGSTATE_IS_MISSING 300 301 NV_STATUS __nvoc_objCreateDynamic_KernelMemorySystem(KernelMemorySystem**, Dynamic*, NvU32, va_list); 302 303 NV_STATUS __nvoc_objCreate_KernelMemorySystem(KernelMemorySystem**, Dynamic*, NvU32); 304 #define __objCreate_KernelMemorySystem(ppNewObj, pParent, createFlags) \ 305 __nvoc_objCreate_KernelMemorySystem((ppNewObj), staticCast((pParent), Dynamic), (createFlags)) 306 307 #define kmemsysConstructEngine(pGpu, pKernelMemorySystem, arg0) kmemsysConstructEngine_DISPATCH(pGpu, pKernelMemorySystem, arg0) 308 #define kmemsysStateInitLocked(pGpu, pKernelMemorySystem) kmemsysStateInitLocked_DISPATCH(pGpu, pKernelMemorySystem) 309 #define kmemsysStatePreLoad(pGpu, pKernelMemorySystem, flags) kmemsysStatePreLoad_DISPATCH(pGpu, pKernelMemorySystem, flags) 310 #define kmemsysStatePostLoad(pGpu, pKernelMemorySystem, flags) kmemsysStatePostLoad_DISPATCH(pGpu, pKernelMemorySystem, flags) 311 #define kmemsysStateLoad(pGpu, pKernelMemorySystem, flags) kmemsysStateLoad_DISPATCH(pGpu, pKernelMemorySystem, flags) 312 #define kmemsysStateLoad_HAL(pGpu, pKernelMemorySystem, flags) kmemsysStateLoad_DISPATCH(pGpu, pKernelMemorySystem, flags) 313 #define kmemsysStatePreUnload(pGpu, pKernelMemorySystem, flags) kmemsysStatePreUnload_DISPATCH(pGpu, pKernelMemorySystem, flags) 314 #define kmemsysStateUnload(pGpu, pKernelMemorySystem, flags) kmemsysStateUnload_DISPATCH(pGpu, pKernelMemorySystem, flags) 315 #define kmemsysStateUnload_HAL(pGpu, pKernelMemorySystem, flags) kmemsysStateUnload_DISPATCH(pGpu, pKernelMemorySystem, flags) 316 #define kmemsysStateDestroy(pGpu, pKernelMemorySystem) kmemsysStateDestroy_DISPATCH(pGpu, pKernelMemorySystem) 317 #define kmemsysGetFbNumaInfo(pGpu, pKernelMemorySystem, physAddr, rsvdPhysAddr, numaNodeId) kmemsysGetFbNumaInfo_DISPATCH(pGpu, pKernelMemorySystem, physAddr, rsvdPhysAddr, numaNodeId) 318 #define kmemsysGetFbNumaInfo_HAL(pGpu, pKernelMemorySystem, physAddr, rsvdPhysAddr, numaNodeId) kmemsysGetFbNumaInfo_DISPATCH(pGpu, pKernelMemorySystem, physAddr, rsvdPhysAddr, numaNodeId) 319 #define kmemsysReadUsableFbSize(pGpu, pKernelMemorySystem, pFbSize) kmemsysReadUsableFbSize_DISPATCH(pGpu, pKernelMemorySystem, pFbSize) 320 #define kmemsysReadUsableFbSize_HAL(pGpu, pKernelMemorySystem, pFbSize) kmemsysReadUsableFbSize_DISPATCH(pGpu, pKernelMemorySystem, pFbSize) 321 #define kmemsysGetUsableFbSize(pGpu, pKernelMemorySystem, pFbSize) kmemsysGetUsableFbSize_DISPATCH(pGpu, pKernelMemorySystem, pFbSize) 322 #define kmemsysGetUsableFbSize_HAL(pGpu, pKernelMemorySystem, pFbSize) kmemsysGetUsableFbSize_DISPATCH(pGpu, pKernelMemorySystem, pFbSize) 323 #define kmemsysCacheOp(pGpu, pKernelMemorySystem, arg0, arg1, operation) kmemsysCacheOp_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1, operation) 324 #define kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, arg0, arg1, operation) kmemsysCacheOp_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1, operation) 325 #define kmemsysDoCacheOp(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3) kmemsysDoCacheOp_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3) 326 #define kmemsysDoCacheOp_HAL(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3) kmemsysDoCacheOp_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3) 327 #define kmemsysReadL2SysmemInvalidateReg(pGpu, pKernelMemorySystem) kmemsysReadL2SysmemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem) 328 #define kmemsysReadL2SysmemInvalidateReg_HAL(pGpu, pKernelMemorySystem) kmemsysReadL2SysmemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem) 329 #define kmemsysWriteL2SysmemInvalidateReg(pGpu, pKernelMemorySystem, arg0) kmemsysWriteL2SysmemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem, arg0) 330 #define kmemsysWriteL2SysmemInvalidateReg_HAL(pGpu, pKernelMemorySystem, arg0) kmemsysWriteL2SysmemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem, arg0) 331 #define kmemsysReadL2PeermemInvalidateReg(pGpu, pKernelMemorySystem) kmemsysReadL2PeermemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem) 332 #define kmemsysReadL2PeermemInvalidateReg_HAL(pGpu, pKernelMemorySystem) kmemsysReadL2PeermemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem) 333 #define kmemsysWriteL2PeermemInvalidateReg(pGpu, pKernelMemorySystem, arg0) kmemsysWriteL2PeermemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem, arg0) 334 #define kmemsysWriteL2PeermemInvalidateReg_HAL(pGpu, pKernelMemorySystem, arg0) kmemsysWriteL2PeermemInvalidateReg_DISPATCH(pGpu, pKernelMemorySystem, arg0) 335 #define kmemsysInitFlushSysmemBuffer(pGpu, pKernelMemorySystem) kmemsysInitFlushSysmemBuffer_DISPATCH(pGpu, pKernelMemorySystem) 336 #define kmemsysInitFlushSysmemBuffer_HAL(pGpu, pKernelMemorySystem) kmemsysInitFlushSysmemBuffer_DISPATCH(pGpu, pKernelMemorySystem) 337 #define kmemsysProgramSysmemFlushBuffer(pGpu, pKernelMemorySystem) kmemsysProgramSysmemFlushBuffer_DISPATCH(pGpu, pKernelMemorySystem) 338 #define kmemsysProgramSysmemFlushBuffer_HAL(pGpu, pKernelMemorySystem) kmemsysProgramSysmemFlushBuffer_DISPATCH(pGpu, pKernelMemorySystem) 339 #define kmemsysGetFlushSysmemBufferAddrShift(pGpu, pKernelMemorySystem) kmemsysGetFlushSysmemBufferAddrShift_DISPATCH(pGpu, pKernelMemorySystem) 340 #define kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem) kmemsysGetFlushSysmemBufferAddrShift_DISPATCH(pGpu, pKernelMemorySystem) 341 #define kmemsysIsPagePLCable(pGpu, KernelMemorySystem, physAddr, pageSize) kmemsysIsPagePLCable_DISPATCH(pGpu, KernelMemorySystem, physAddr, pageSize) 342 #define kmemsysIsPagePLCable_HAL(pGpu, KernelMemorySystem, physAddr, pageSize) kmemsysIsPagePLCable_DISPATCH(pGpu, KernelMemorySystem, physAddr, pageSize) 343 #define kmemsysReadMIGMemoryCfg(pGpu, pKernelMemorySystem) kmemsysReadMIGMemoryCfg_DISPATCH(pGpu, pKernelMemorySystem) 344 #define kmemsysReadMIGMemoryCfg_HAL(pGpu, pKernelMemorySystem) kmemsysReadMIGMemoryCfg_DISPATCH(pGpu, pKernelMemorySystem) 345 #define kmemsysInitMIGMemoryPartitionTable(pGpu, pKernelMemorySystem) kmemsysInitMIGMemoryPartitionTable_DISPATCH(pGpu, pKernelMemorySystem) 346 #define kmemsysInitMIGMemoryPartitionTable_HAL(pGpu, pKernelMemorySystem) kmemsysInitMIGMemoryPartitionTable_DISPATCH(pGpu, pKernelMemorySystem) 347 #define kmemsysSwizzIdToVmmuSegmentsRange(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments) kmemsysSwizzIdToVmmuSegmentsRange_DISPATCH(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments) 348 #define kmemsysSwizzIdToVmmuSegmentsRange_HAL(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments) kmemsysSwizzIdToVmmuSegmentsRange_DISPATCH(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments) 349 #define kmemsysNumaAddMemory(pGpu, pKernelMemorySystem, swizzId, offset, size, numaNodeId) kmemsysNumaAddMemory_DISPATCH(pGpu, pKernelMemorySystem, swizzId, offset, size, numaNodeId) 350 #define kmemsysNumaAddMemory_HAL(pGpu, pKernelMemorySystem, swizzId, offset, size, numaNodeId) kmemsysNumaAddMemory_DISPATCH(pGpu, pKernelMemorySystem, swizzId, offset, size, numaNodeId) 351 #define kmemsysNumaRemoveMemory(pGpu, pKernelMemorySystem, swizzId) kmemsysNumaRemoveMemory_DISPATCH(pGpu, pKernelMemorySystem, swizzId) 352 #define kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, swizzId) kmemsysNumaRemoveMemory_DISPATCH(pGpu, pKernelMemorySystem, swizzId) 353 #define kmemsysNumaRemoveAllMemory(pGpu, pKernelMemorySystem) kmemsysNumaRemoveAllMemory_DISPATCH(pGpu, pKernelMemorySystem) 354 #define kmemsysNumaRemoveAllMemory_HAL(pGpu, pKernelMemorySystem) kmemsysNumaRemoveAllMemory_DISPATCH(pGpu, pKernelMemorySystem) 355 #define kmemsysPopulateMIGGPUInstanceMemConfig(pGpu, pKernelMemorySystem) kmemsysPopulateMIGGPUInstanceMemConfig_DISPATCH(pGpu, pKernelMemorySystem) 356 #define kmemsysPopulateMIGGPUInstanceMemConfig_HAL(pGpu, pKernelMemorySystem) kmemsysPopulateMIGGPUInstanceMemConfig_DISPATCH(pGpu, pKernelMemorySystem) 357 #define kmemsysSetupAllAtsPeers(pGpu, pKernelMemorySystem) kmemsysSetupAllAtsPeers_DISPATCH(pGpu, pKernelMemorySystem) 358 #define kmemsysSetupAllAtsPeers_HAL(pGpu, pKernelMemorySystem) kmemsysSetupAllAtsPeers_DISPATCH(pGpu, pKernelMemorySystem) 359 #define kmemsysRemoveAllAtsPeers(pGpu, pKernelMemorySystem) kmemsysRemoveAllAtsPeers_DISPATCH(pGpu, pKernelMemorySystem) 360 #define kmemsysRemoveAllAtsPeers_HAL(pGpu, pKernelMemorySystem) kmemsysRemoveAllAtsPeers_DISPATCH(pGpu, pKernelMemorySystem) 361 #define kmemsysAssertFbAckTimeoutPending(pGpu, pKernelMemorySystem) kmemsysAssertFbAckTimeoutPending_DISPATCH(pGpu, pKernelMemorySystem) 362 #define kmemsysAssertFbAckTimeoutPending_HAL(pGpu, pKernelMemorySystem) kmemsysAssertFbAckTimeoutPending_DISPATCH(pGpu, pKernelMemorySystem) 363 #define kmemsysGetMaxFbpas(pGpu, pKernelMemorySystem) kmemsysGetMaxFbpas_DISPATCH(pGpu, pKernelMemorySystem) 364 #define kmemsysGetMaxFbpas_HAL(pGpu, pKernelMemorySystem) kmemsysGetMaxFbpas_DISPATCH(pGpu, pKernelMemorySystem) 365 #define kmemsysGetEccDedCountSize(pGpu, pKernelMemorySystem) kmemsysGetEccDedCountSize_DISPATCH(pGpu, pKernelMemorySystem) 366 #define kmemsysGetEccDedCountSize_HAL(pGpu, pKernelMemorySystem) kmemsysGetEccDedCountSize_DISPATCH(pGpu, pKernelMemorySystem) 367 #define kmemsysGetEccDedCountRegAddr(pGpu, pKernelMemorySystem, fbpa, subp) kmemsysGetEccDedCountRegAddr_DISPATCH(pGpu, pKernelMemorySystem, fbpa, subp) 368 #define kmemsysGetEccDedCountRegAddr_HAL(pGpu, pKernelMemorySystem, fbpa, subp) kmemsysGetEccDedCountRegAddr_DISPATCH(pGpu, pKernelMemorySystem, fbpa, subp) 369 #define kmemsysGetEccCounts(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysGetEccCounts_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1) 370 #define kmemsysGetEccCounts_HAL(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysGetEccCounts_DISPATCH(pGpu, pKernelMemorySystem, arg0, arg1) 371 #define kmemsysClearEccCounts(pGpu, pKernelMemorySystem) kmemsysClearEccCounts_DISPATCH(pGpu, pKernelMemorySystem) 372 #define kmemsysClearEccCounts_HAL(pGpu, pKernelMemorySystem) kmemsysClearEccCounts_DISPATCH(pGpu, pKernelMemorySystem) 373 #define kmemsysGetMaximumBlacklistPages(pGpu, pKernelMemorySystem) kmemsysGetMaximumBlacklistPages_DISPATCH(pGpu, pKernelMemorySystem) 374 #define kmemsysGetMaximumBlacklistPages_HAL(pGpu, pKernelMemorySystem) kmemsysGetMaximumBlacklistPages_DISPATCH(pGpu, pKernelMemorySystem) 375 #define kmemsysGetFbInfos(arg0, arg1, arg2, arg3, hSubdevice, pParams, pFbInfoListIndicesUnset) kmemsysGetFbInfos_DISPATCH(arg0, arg1, arg2, arg3, hSubdevice, pParams, pFbInfoListIndicesUnset) 376 #define kmemsysGetFbInfos_HAL(arg0, arg1, arg2, arg3, hSubdevice, pParams, pFbInfoListIndicesUnset) kmemsysGetFbInfos_DISPATCH(arg0, arg1, arg2, arg3, hSubdevice, pParams, pFbInfoListIndicesUnset) 377 #define kmemsysStatePostUnload(pGpu, pEngstate, arg0) kmemsysStatePostUnload_DISPATCH(pGpu, pEngstate, arg0) 378 #define kmemsysStateInitUnlocked(pGpu, pEngstate) kmemsysStateInitUnlocked_DISPATCH(pGpu, pEngstate) 379 #define kmemsysInitMissing(pGpu, pEngstate) kmemsysInitMissing_DISPATCH(pGpu, pEngstate) 380 #define kmemsysStatePreInitLocked(pGpu, pEngstate) kmemsysStatePreInitLocked_DISPATCH(pGpu, pEngstate) 381 #define kmemsysStatePreInitUnlocked(pGpu, pEngstate) kmemsysStatePreInitUnlocked_DISPATCH(pGpu, pEngstate) 382 #define kmemsysIsPresent(pGpu, pEngstate) kmemsysIsPresent_DISPATCH(pGpu, pEngstate) 383 static inline void kmemsysAssertSysmemFlushBufferValid_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 384 return; 385 } 386 387 void kmemsysAssertSysmemFlushBufferValid_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 388 389 void kmemsysAssertSysmemFlushBufferValid_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 390 391 void kmemsysAssertSysmemFlushBufferValid_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 392 393 394 #ifdef __nvoc_kern_mem_sys_h_disabled 395 static inline void kmemsysAssertSysmemFlushBufferValid(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 396 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 397 } 398 #else //__nvoc_kern_mem_sys_h_disabled 399 #define kmemsysAssertSysmemFlushBufferValid(pGpu, pKernelMemorySystem) kmemsysAssertSysmemFlushBufferValid_b3696a(pGpu, pKernelMemorySystem) 400 #endif //__nvoc_kern_mem_sys_h_disabled 401 402 #define kmemsysAssertSysmemFlushBufferValid_HAL(pGpu, pKernelMemorySystem) kmemsysAssertSysmemFlushBufferValid(pGpu, pKernelMemorySystem) 403 404 NV_STATUS kmemsysInitStaticConfig_KERNEL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, MEMORY_SYSTEM_STATIC_CONFIG *pConfig); 405 406 407 #ifdef __nvoc_kern_mem_sys_h_disabled 408 static inline NV_STATUS kmemsysInitStaticConfig(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, MEMORY_SYSTEM_STATIC_CONFIG *pConfig) { 409 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 410 return NV_ERR_NOT_SUPPORTED; 411 } 412 #else //__nvoc_kern_mem_sys_h_disabled 413 #define kmemsysInitStaticConfig(pGpu, pKernelMemorySystem, pConfig) kmemsysInitStaticConfig_KERNEL(pGpu, pKernelMemorySystem, pConfig) 414 #endif //__nvoc_kern_mem_sys_h_disabled 415 416 #define kmemsysInitStaticConfig_HAL(pGpu, pKernelMemorySystem, pConfig) kmemsysInitStaticConfig(pGpu, pKernelMemorySystem, pConfig) 417 418 static inline NV_STATUS kmemsysPreFillCacheOnlyMemory_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 arg0, NvU64 arg1) { 419 return NV_OK; 420 } 421 422 NV_STATUS kmemsysPreFillCacheOnlyMemory_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 arg0, NvU64 arg1); 423 424 425 #ifdef __nvoc_kern_mem_sys_h_disabled 426 static inline NV_STATUS kmemsysPreFillCacheOnlyMemory(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 arg0, NvU64 arg1) { 427 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 428 return NV_ERR_NOT_SUPPORTED; 429 } 430 #else //__nvoc_kern_mem_sys_h_disabled 431 #define kmemsysPreFillCacheOnlyMemory(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysPreFillCacheOnlyMemory_56cd7a(pGpu, pKernelMemorySystem, arg0, arg1) 432 #endif //__nvoc_kern_mem_sys_h_disabled 433 434 #define kmemsysPreFillCacheOnlyMemory_HAL(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysPreFillCacheOnlyMemory(pGpu, pKernelMemorySystem, arg0, arg1) 435 436 static inline NV_STATUS kmemsysCheckDisplayRemapperRange_14278f(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 arg0, NvU64 arg1) { 437 NV_ASSERT_OR_RETURN_PRECOMP(0, NV_ERR_INVALID_STATE); 438 } 439 440 441 #ifdef __nvoc_kern_mem_sys_h_disabled 442 static inline NV_STATUS kmemsysCheckDisplayRemapperRange(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 arg0, NvU64 arg1) { 443 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 444 return NV_ERR_NOT_SUPPORTED; 445 } 446 #else //__nvoc_kern_mem_sys_h_disabled 447 #define kmemsysCheckDisplayRemapperRange(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysCheckDisplayRemapperRange_14278f(pGpu, pKernelMemorySystem, arg0, arg1) 448 #endif //__nvoc_kern_mem_sys_h_disabled 449 450 #define kmemsysCheckDisplayRemapperRange_HAL(pGpu, pKernelMemorySystem, arg0, arg1) kmemsysCheckDisplayRemapperRange(pGpu, pKernelMemorySystem, arg0, arg1) 451 452 static inline void kmemsysPostHeapCreate_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 453 return; 454 } 455 456 457 #ifdef __nvoc_kern_mem_sys_h_disabled 458 static inline void kmemsysPostHeapCreate(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 459 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 460 } 461 #else //__nvoc_kern_mem_sys_h_disabled 462 #define kmemsysPostHeapCreate(pGpu, pKernelMemorySystem) kmemsysPostHeapCreate_b3696a(pGpu, pKernelMemorySystem) 463 #endif //__nvoc_kern_mem_sys_h_disabled 464 465 #define kmemsysPostHeapCreate_HAL(pGpu, pKernelMemorySystem) kmemsysPostHeapCreate(pGpu, pKernelMemorySystem) 466 467 static inline void kmemsysPreHeapDestruct_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 468 return; 469 } 470 471 472 #ifdef __nvoc_kern_mem_sys_h_disabled 473 static inline void kmemsysPreHeapDestruct(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 474 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 475 } 476 #else //__nvoc_kern_mem_sys_h_disabled 477 #define kmemsysPreHeapDestruct(pGpu, pKernelMemorySystem) kmemsysPreHeapDestruct_b3696a(pGpu, pKernelMemorySystem) 478 #endif //__nvoc_kern_mem_sys_h_disabled 479 480 #define kmemsysPreHeapDestruct_HAL(pGpu, pKernelMemorySystem) kmemsysPreHeapDestruct(pGpu, pKernelMemorySystem) 481 482 NV_STATUS kmemsysAllocComprResources_KERNEL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, FB_ALLOC_INFO *arg0, NvU64 arg1, NvU32 arg2, NvU32 *arg3, NvU32 arg4); 483 484 485 #ifdef __nvoc_kern_mem_sys_h_disabled 486 static inline NV_STATUS kmemsysAllocComprResources(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, FB_ALLOC_INFO *arg0, NvU64 arg1, NvU32 arg2, NvU32 *arg3, NvU32 arg4) { 487 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 488 return NV_ERR_NOT_SUPPORTED; 489 } 490 #else //__nvoc_kern_mem_sys_h_disabled 491 #define kmemsysAllocComprResources(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3, arg4) kmemsysAllocComprResources_KERNEL(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3, arg4) 492 #endif //__nvoc_kern_mem_sys_h_disabled 493 494 #define kmemsysAllocComprResources_HAL(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3, arg4) kmemsysAllocComprResources(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3, arg4) 495 496 static inline void kmemsysFreeComprResources_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 hwResId) { 497 return; 498 } 499 500 501 #ifdef __nvoc_kern_mem_sys_h_disabled 502 static inline void kmemsysFreeComprResources(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 hwResId) { 503 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 504 } 505 #else //__nvoc_kern_mem_sys_h_disabled 506 #define kmemsysFreeComprResources(pGpu, pKernelMemorySystem, hwResId) kmemsysFreeComprResources_b3696a(pGpu, pKernelMemorySystem, hwResId) 507 #endif //__nvoc_kern_mem_sys_h_disabled 508 509 #define kmemsysFreeComprResources_HAL(pGpu, pKernelMemorySystem, hwResId) kmemsysFreeComprResources(pGpu, pKernelMemorySystem, hwResId) 510 511 NvBool kmemsysNeedInvalidateGpuCacheOnMap_GV100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bIsVolatile, NvU32 aperture); 512 513 514 #ifdef __nvoc_kern_mem_sys_h_disabled 515 static inline NvBool kmemsysNeedInvalidateGpuCacheOnMap(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bIsVolatile, NvU32 aperture) { 516 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 517 return NV_FALSE; 518 } 519 #else //__nvoc_kern_mem_sys_h_disabled 520 #define kmemsysNeedInvalidateGpuCacheOnMap(pGpu, pKernelMemorySystem, bIsVolatile, aperture) kmemsysNeedInvalidateGpuCacheOnMap_GV100(pGpu, pKernelMemorySystem, bIsVolatile, aperture) 521 #endif //__nvoc_kern_mem_sys_h_disabled 522 523 #define kmemsysNeedInvalidateGpuCacheOnMap_HAL(pGpu, pKernelMemorySystem, bIsVolatile, aperture) kmemsysNeedInvalidateGpuCacheOnMap(pGpu, pKernelMemorySystem, bIsVolatile, aperture) 524 525 static inline NvBool kmemsysCbcIsSafe_cbe027(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 526 return ((NvBool)(0 == 0)); 527 } 528 529 530 #ifdef __nvoc_kern_mem_sys_h_disabled 531 static inline NvBool kmemsysCbcIsSafe(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 532 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 533 return NV_FALSE; 534 } 535 #else //__nvoc_kern_mem_sys_h_disabled 536 #define kmemsysCbcIsSafe(pGpu, pKernelMemorySystem) kmemsysCbcIsSafe_cbe027(pGpu, pKernelMemorySystem) 537 #endif //__nvoc_kern_mem_sys_h_disabled 538 539 #define kmemsysCbcIsSafe_HAL(pGpu, pKernelMemorySystem) kmemsysCbcIsSafe(pGpu, pKernelMemorySystem) 540 541 static inline NV_STATUS kmemsysPrepareForXVEReset_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 542 return NV_OK; 543 } 544 545 546 #ifdef __nvoc_kern_mem_sys_h_disabled 547 static inline NV_STATUS kmemsysPrepareForXVEReset(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 548 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 549 return NV_ERR_NOT_SUPPORTED; 550 } 551 #else //__nvoc_kern_mem_sys_h_disabled 552 #define kmemsysPrepareForXVEReset(pGpu, pKernelMemorySystem) kmemsysPrepareForXVEReset_56cd7a(pGpu, pKernelMemorySystem) 553 #endif //__nvoc_kern_mem_sys_h_disabled 554 555 #define kmemsysPrepareForXVEReset_HAL(pGpu, pKernelMemorySystem) kmemsysPrepareForXVEReset(pGpu, pKernelMemorySystem) 556 557 NV_STATUS kmemsysConstructEngine_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, ENGDESCRIPTOR arg0); 558 559 static inline NV_STATUS kmemsysConstructEngine_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, ENGDESCRIPTOR arg0) { 560 return pKernelMemorySystem->__kmemsysConstructEngine__(pGpu, pKernelMemorySystem, arg0); 561 } 562 563 NV_STATUS kmemsysStateInitLocked_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 564 565 static inline NV_STATUS kmemsysStateInitLocked_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 566 return pKernelMemorySystem->__kmemsysStateInitLocked__(pGpu, pKernelMemorySystem); 567 } 568 569 NV_STATUS kmemsysStatePreLoad_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 570 571 static inline NV_STATUS kmemsysStatePreLoad_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 572 return pKernelMemorySystem->__kmemsysStatePreLoad__(pGpu, pKernelMemorySystem, flags); 573 } 574 575 NV_STATUS kmemsysStatePostLoad_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 576 577 static inline NV_STATUS kmemsysStatePostLoad_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 578 return pKernelMemorySystem->__kmemsysStatePostLoad__(pGpu, pKernelMemorySystem, flags); 579 } 580 581 NV_STATUS kmemsysStateLoad_VF(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 582 583 static inline NV_STATUS kmemsysStateLoad_ac1694(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 584 return NV_OK; 585 } 586 587 static inline NV_STATUS kmemsysStateLoad_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 588 return pKernelMemorySystem->__kmemsysStateLoad__(pGpu, pKernelMemorySystem, flags); 589 } 590 591 NV_STATUS kmemsysStatePreUnload_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 592 593 static inline NV_STATUS kmemsysStatePreUnload_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 594 return pKernelMemorySystem->__kmemsysStatePreUnload__(pGpu, pKernelMemorySystem, flags); 595 } 596 597 NV_STATUS kmemsysStateUnload_VF(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 598 599 static inline NV_STATUS kmemsysStateUnload_ac1694(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 600 return NV_OK; 601 } 602 603 static inline NV_STATUS kmemsysStateUnload_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 604 return pKernelMemorySystem->__kmemsysStateUnload__(pGpu, pKernelMemorySystem, flags); 605 } 606 607 void kmemsysStateDestroy_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 608 609 static inline void kmemsysStateDestroy_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 610 pKernelMemorySystem->__kmemsysStateDestroy__(pGpu, pKernelMemorySystem); 611 } 612 613 NV_STATUS kmemsysGetFbNumaInfo_GV100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *physAddr, NvU64 *rsvdPhysAddr, NvS32 *numaNodeId); 614 615 static inline NV_STATUS kmemsysGetFbNumaInfo_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *physAddr, NvU64 *rsvdPhysAddr, NvS32 *numaNodeId) { 616 return NV_OK; 617 } 618 619 static inline NV_STATUS kmemsysGetFbNumaInfo_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *physAddr, NvU64 *rsvdPhysAddr, NvS32 *numaNodeId) { 620 return pKernelMemorySystem->__kmemsysGetFbNumaInfo__(pGpu, pKernelMemorySystem, physAddr, rsvdPhysAddr, numaNodeId); 621 } 622 623 NV_STATUS kmemsysReadUsableFbSize_GP102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize); 624 625 NV_STATUS kmemsysReadUsableFbSize_GA102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize); 626 627 static inline NV_STATUS kmemsysReadUsableFbSize_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize) { 628 return pKernelMemorySystem->__kmemsysReadUsableFbSize__(pGpu, pKernelMemorySystem, pFbSize); 629 } 630 631 NV_STATUS kmemsysGetUsableFbSize_KERNEL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize); 632 633 static inline NV_STATUS kmemsysGetUsableFbSize_5baef9(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize) { 634 NV_ASSERT_OR_RETURN_PRECOMP(0, NV_ERR_NOT_SUPPORTED); 635 } 636 637 static inline NV_STATUS kmemsysGetUsableFbSize_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU64 *pFbSize) { 638 return pKernelMemorySystem->__kmemsysGetUsableFbSize__(pGpu, pKernelMemorySystem, pFbSize); 639 } 640 641 NV_STATUS kmemsysCacheOp_GM200(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, PMEMORY_DESCRIPTOR arg0, FB_CACHE_MEMTYPE arg1, FB_CACHE_OP operation); 642 643 NV_STATUS kmemsysCacheOp_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, PMEMORY_DESCRIPTOR arg0, FB_CACHE_MEMTYPE arg1, FB_CACHE_OP operation); 644 645 static inline NV_STATUS kmemsysCacheOp_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, PMEMORY_DESCRIPTOR arg0, FB_CACHE_MEMTYPE arg1, FB_CACHE_OP operation) { 646 return pKernelMemorySystem->__kmemsysCacheOp__(pGpu, pKernelMemorySystem, arg0, arg1, operation); 647 } 648 649 NV_STATUS kmemsysDoCacheOp_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0, NvU32 arg1, NvU32 arg2, PRMTIMEOUT arg3); 650 651 NV_STATUS kmemsysDoCacheOp_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0, NvU32 arg1, NvU32 arg2, PRMTIMEOUT arg3); 652 653 static inline NV_STATUS kmemsysDoCacheOp_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0, NvU32 arg1, NvU32 arg2, PRMTIMEOUT arg3) { 654 return pKernelMemorySystem->__kmemsysDoCacheOp__(pGpu, pKernelMemorySystem, arg0, arg1, arg2, arg3); 655 } 656 657 NvU32 kmemsysReadL2SysmemInvalidateReg_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 658 659 static inline NvU32 kmemsysReadL2SysmemInvalidateReg_68b109(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 660 NV_ASSERT_OR_RETURN_PRECOMP(0, -1); 661 } 662 663 static inline NvU32 kmemsysReadL2SysmemInvalidateReg_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 664 return pKernelMemorySystem->__kmemsysReadL2SysmemInvalidateReg__(pGpu, pKernelMemorySystem); 665 } 666 667 void kmemsysWriteL2SysmemInvalidateReg_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0); 668 669 static inline void kmemsysWriteL2SysmemInvalidateReg_f2d351(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0) { 670 NV_ASSERT_PRECOMP(0); 671 } 672 673 static inline void kmemsysWriteL2SysmemInvalidateReg_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0) { 674 pKernelMemorySystem->__kmemsysWriteL2SysmemInvalidateReg__(pGpu, pKernelMemorySystem, arg0); 675 } 676 677 NvU32 kmemsysReadL2PeermemInvalidateReg_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 678 679 static inline NvU32 kmemsysReadL2PeermemInvalidateReg_68b109(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 680 NV_ASSERT_OR_RETURN_PRECOMP(0, -1); 681 } 682 683 static inline NvU32 kmemsysReadL2PeermemInvalidateReg_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 684 return pKernelMemorySystem->__kmemsysReadL2PeermemInvalidateReg__(pGpu, pKernelMemorySystem); 685 } 686 687 void kmemsysWriteL2PeermemInvalidateReg_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0); 688 689 static inline void kmemsysWriteL2PeermemInvalidateReg_f2d351(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0) { 690 NV_ASSERT_PRECOMP(0); 691 } 692 693 static inline void kmemsysWriteL2PeermemInvalidateReg_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 arg0) { 694 pKernelMemorySystem->__kmemsysWriteL2PeermemInvalidateReg__(pGpu, pKernelMemorySystem, arg0); 695 } 696 697 static inline NV_STATUS kmemsysInitFlushSysmemBuffer_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 698 return NV_OK; 699 } 700 701 NV_STATUS kmemsysInitFlushSysmemBuffer_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 702 703 NV_STATUS kmemsysInitFlushSysmemBuffer_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 704 705 static inline NV_STATUS kmemsysInitFlushSysmemBuffer_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 706 return pKernelMemorySystem->__kmemsysInitFlushSysmemBuffer__(pGpu, pKernelMemorySystem); 707 } 708 709 static inline void kmemsysProgramSysmemFlushBuffer_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 710 return; 711 } 712 713 void kmemsysProgramSysmemFlushBuffer_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 714 715 void kmemsysProgramSysmemFlushBuffer_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 716 717 void kmemsysProgramSysmemFlushBuffer_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 718 719 static inline void kmemsysProgramSysmemFlushBuffer_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 720 pKernelMemorySystem->__kmemsysProgramSysmemFlushBuffer__(pGpu, pKernelMemorySystem); 721 } 722 723 static inline NvU32 kmemsysGetFlushSysmemBufferAddrShift_4a4dee(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 724 return 0; 725 } 726 727 NvU32 kmemsysGetFlushSysmemBufferAddrShift_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 728 729 static inline NvU32 kmemsysGetFlushSysmemBufferAddrShift_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 730 return pKernelMemorySystem->__kmemsysGetFlushSysmemBufferAddrShift__(pGpu, pKernelMemorySystem); 731 } 732 733 static inline NvBool kmemsysIsPagePLCable_cbe027(OBJGPU *pGpu, struct KernelMemorySystem *KernelMemorySystem, NvU64 physAddr, NvU64 pageSize) { 734 return ((NvBool)(0 == 0)); 735 } 736 737 NvBool kmemsysIsPagePLCable_GA100(OBJGPU *pGpu, struct KernelMemorySystem *KernelMemorySystem, NvU64 physAddr, NvU64 pageSize); 738 739 NvBool kmemsysIsPagePLCable_GA102(OBJGPU *pGpu, struct KernelMemorySystem *KernelMemorySystem, NvU64 physAddr, NvU64 pageSize); 740 741 static inline NvBool kmemsysIsPagePLCable_510167(OBJGPU *pGpu, struct KernelMemorySystem *KernelMemorySystem, NvU64 physAddr, NvU64 pageSize) { 742 NV_ASSERT_OR_RETURN_PRECOMP(0, ((NvBool)(0 == 0))); 743 } 744 745 static inline NvBool kmemsysIsPagePLCable_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *KernelMemorySystem, NvU64 physAddr, NvU64 pageSize) { 746 return KernelMemorySystem->__kmemsysIsPagePLCable__(pGpu, KernelMemorySystem, physAddr, pageSize); 747 } 748 749 static inline NV_STATUS kmemsysReadMIGMemoryCfg_46f6a7(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 750 return NV_ERR_NOT_SUPPORTED; 751 } 752 753 NV_STATUS kmemsysReadMIGMemoryCfg_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 754 755 static inline NV_STATUS kmemsysReadMIGMemoryCfg_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 756 return pKernelMemorySystem->__kmemsysReadMIGMemoryCfg__(pGpu, pKernelMemorySystem); 757 } 758 759 static inline NV_STATUS kmemsysInitMIGMemoryPartitionTable_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 760 return NV_OK; 761 } 762 763 NV_STATUS kmemsysInitMIGMemoryPartitionTable_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 764 765 static inline NV_STATUS kmemsysInitMIGMemoryPartitionTable_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 766 return pKernelMemorySystem->__kmemsysInitMIGMemoryPartitionTable__(pGpu, pKernelMemorySystem); 767 } 768 769 NV_STATUS kmemsysSwizzIdToVmmuSegmentsRange_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU32 vmmuSegmentSize, NvU32 totalVmmuSegments); 770 771 NV_STATUS kmemsysSwizzIdToVmmuSegmentsRange_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU32 vmmuSegmentSize, NvU32 totalVmmuSegments); 772 773 static inline NV_STATUS kmemsysSwizzIdToVmmuSegmentsRange_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU32 vmmuSegmentSize, NvU32 totalVmmuSegments) { 774 return pKernelMemorySystem->__kmemsysSwizzIdToVmmuSegmentsRange__(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, totalVmmuSegments); 775 } 776 777 NV_STATUS kmemsysNumaAddMemory_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU64 offset, NvU64 size, NvS32 *numaNodeId); 778 779 static inline NV_STATUS kmemsysNumaAddMemory_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU64 offset, NvU64 size, NvS32 *numaNodeId) { 780 return NV_OK; 781 } 782 783 static inline NV_STATUS kmemsysNumaAddMemory_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, NvU64 offset, NvU64 size, NvS32 *numaNodeId) { 784 return pKernelMemorySystem->__kmemsysNumaAddMemory__(pGpu, pKernelMemorySystem, swizzId, offset, size, numaNodeId); 785 } 786 787 void kmemsysNumaRemoveMemory_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId); 788 789 static inline void kmemsysNumaRemoveMemory_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId) { 790 return; 791 } 792 793 static inline void kmemsysNumaRemoveMemory_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId) { 794 pKernelMemorySystem->__kmemsysNumaRemoveMemory__(pGpu, pKernelMemorySystem, swizzId); 795 } 796 797 void kmemsysNumaRemoveAllMemory_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 798 799 static inline void kmemsysNumaRemoveAllMemory_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 800 return; 801 } 802 803 static inline void kmemsysNumaRemoveAllMemory_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 804 pKernelMemorySystem->__kmemsysNumaRemoveAllMemory__(pGpu, pKernelMemorySystem); 805 } 806 807 static inline NV_STATUS kmemsysPopulateMIGGPUInstanceMemConfig_56cd7a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 808 return NV_OK; 809 } 810 811 NV_STATUS kmemsysPopulateMIGGPUInstanceMemConfig_KERNEL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 812 813 static inline NV_STATUS kmemsysPopulateMIGGPUInstanceMemConfig_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 814 return pKernelMemorySystem->__kmemsysPopulateMIGGPUInstanceMemConfig__(pGpu, pKernelMemorySystem); 815 } 816 817 static inline NV_STATUS kmemsysSetupAllAtsPeers_46f6a7(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 818 return NV_ERR_NOT_SUPPORTED; 819 } 820 821 NV_STATUS kmemsysSetupAllAtsPeers_GV100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 822 823 static inline NV_STATUS kmemsysSetupAllAtsPeers_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 824 return pKernelMemorySystem->__kmemsysSetupAllAtsPeers__(pGpu, pKernelMemorySystem); 825 } 826 827 static inline void kmemsysRemoveAllAtsPeers_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 828 return; 829 } 830 831 void kmemsysRemoveAllAtsPeers_GV100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 832 833 static inline void kmemsysRemoveAllAtsPeers_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 834 pKernelMemorySystem->__kmemsysRemoveAllAtsPeers__(pGpu, pKernelMemorySystem); 835 } 836 837 NvBool kmemsysAssertFbAckTimeoutPending_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 838 839 static inline NvBool kmemsysAssertFbAckTimeoutPending_491d52(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 840 return ((NvBool)(0 != 0)); 841 } 842 843 static inline NvBool kmemsysAssertFbAckTimeoutPending_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 844 return pKernelMemorySystem->__kmemsysAssertFbAckTimeoutPending__(pGpu, pKernelMemorySystem); 845 } 846 847 NvU32 kmemsysGetMaxFbpas_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 848 849 NvU32 kmemsysGetMaxFbpas_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 850 851 static inline NvU32 kmemsysGetMaxFbpas_4a4dee(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 852 return 0; 853 } 854 855 static inline NvU32 kmemsysGetMaxFbpas_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 856 return pKernelMemorySystem->__kmemsysGetMaxFbpas__(pGpu, pKernelMemorySystem); 857 } 858 859 NvU32 kmemsysGetEccDedCountSize_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 860 861 NvU32 kmemsysGetEccDedCountSize_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 862 863 static inline NvU32 kmemsysGetEccDedCountSize_4a4dee(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 864 return 0; 865 } 866 867 static inline NvU32 kmemsysGetEccDedCountSize_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 868 return pKernelMemorySystem->__kmemsysGetEccDedCountSize__(pGpu, pKernelMemorySystem); 869 } 870 871 NvU32 kmemsysGetEccDedCountRegAddr_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 fbpa, NvU32 subp); 872 873 NvU32 kmemsysGetEccDedCountRegAddr_GH100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 fbpa, NvU32 subp); 874 875 static inline NvU32 kmemsysGetEccDedCountRegAddr_4a4dee(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 fbpa, NvU32 subp) { 876 return 0; 877 } 878 879 static inline NvU32 kmemsysGetEccDedCountRegAddr_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 fbpa, NvU32 subp) { 880 return pKernelMemorySystem->__kmemsysGetEccDedCountRegAddr__(pGpu, pKernelMemorySystem, fbpa, subp); 881 } 882 883 void kmemsysGetEccCounts_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 *arg0, NvU32 *arg1); 884 885 static inline void kmemsysGetEccCounts_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 *arg0, NvU32 *arg1) { 886 return; 887 } 888 889 static inline void kmemsysGetEccCounts_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 *arg0, NvU32 *arg1) { 890 pKernelMemorySystem->__kmemsysGetEccCounts__(pGpu, pKernelMemorySystem, arg0, arg1); 891 } 892 893 void kmemsysClearEccCounts_TU102(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 894 895 static inline void kmemsysClearEccCounts_b3696a(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 896 return; 897 } 898 899 static inline void kmemsysClearEccCounts_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 900 pKernelMemorySystem->__kmemsysClearEccCounts__(pGpu, pKernelMemorySystem); 901 } 902 903 NvU16 kmemsysGetMaximumBlacklistPages_GM107(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 904 905 NvU16 kmemsysGetMaximumBlacklistPages_GA100(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 906 907 static inline NvU16 kmemsysGetMaximumBlacklistPages_DISPATCH(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 908 return pKernelMemorySystem->__kmemsysGetMaximumBlacklistPages__(pGpu, pKernelMemorySystem); 909 } 910 911 static inline NV_STATUS kmemsysGetFbInfos_ac1694(OBJGPU *arg0, struct KernelMemorySystem *arg1, struct RsClient *arg2, Device *arg3, NvHandle hSubdevice, NV2080_CTRL_FB_GET_INFO_V2_PARAMS *pParams, NvU64 *pFbInfoListIndicesUnset) { 912 return NV_OK; 913 } 914 915 NV_STATUS kmemsysGetFbInfos_VF(OBJGPU *arg0, struct KernelMemorySystem *arg1, struct RsClient *arg2, Device *arg3, NvHandle hSubdevice, NV2080_CTRL_FB_GET_INFO_V2_PARAMS *pParams, NvU64 *pFbInfoListIndicesUnset); 916 917 static inline NV_STATUS kmemsysGetFbInfos_DISPATCH(OBJGPU *arg0, struct KernelMemorySystem *arg1, struct RsClient *arg2, Device *arg3, NvHandle hSubdevice, NV2080_CTRL_FB_GET_INFO_V2_PARAMS *pParams, NvU64 *pFbInfoListIndicesUnset) { 918 return arg1->__kmemsysGetFbInfos__(arg0, arg1, arg2, arg3, hSubdevice, pParams, pFbInfoListIndicesUnset); 919 } 920 921 static inline NV_STATUS kmemsysStatePostUnload_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate, NvU32 arg0) { 922 return pEngstate->__kmemsysStatePostUnload__(pGpu, pEngstate, arg0); 923 } 924 925 static inline NV_STATUS kmemsysStateInitUnlocked_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate) { 926 return pEngstate->__kmemsysStateInitUnlocked__(pGpu, pEngstate); 927 } 928 929 static inline void kmemsysInitMissing_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate) { 930 pEngstate->__kmemsysInitMissing__(pGpu, pEngstate); 931 } 932 933 static inline NV_STATUS kmemsysStatePreInitLocked_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate) { 934 return pEngstate->__kmemsysStatePreInitLocked__(pGpu, pEngstate); 935 } 936 937 static inline NV_STATUS kmemsysStatePreInitUnlocked_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate) { 938 return pEngstate->__kmemsysStatePreInitUnlocked__(pGpu, pEngstate); 939 } 940 941 static inline NvBool kmemsysIsPresent_DISPATCH(POBJGPU pGpu, struct KernelMemorySystem *pEngstate) { 942 return pEngstate->__kmemsysIsPresent__(pGpu, pEngstate); 943 } 944 945 static inline NvBool kmemsysIsL2CleanFbPull(struct KernelMemorySystem *pKernelMemorySystem) { 946 return pKernelMemorySystem->bL2CleanFbPull; 947 } 948 949 void kmemsysDestruct_IMPL(struct KernelMemorySystem *pKernelMemorySystem); 950 951 #define __nvoc_kmemsysDestruct(pKernelMemorySystem) kmemsysDestruct_IMPL(pKernelMemorySystem) 952 NV_STATUS kmemsysEnsureSysmemFlushBufferInitialized_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 953 954 #ifdef __nvoc_kern_mem_sys_h_disabled 955 static inline NV_STATUS kmemsysEnsureSysmemFlushBufferInitialized(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 956 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 957 return NV_ERR_NOT_SUPPORTED; 958 } 959 #else //__nvoc_kern_mem_sys_h_disabled 960 #define kmemsysEnsureSysmemFlushBufferInitialized(pGpu, pKernelMemorySystem) kmemsysEnsureSysmemFlushBufferInitialized_IMPL(pGpu, pKernelMemorySystem) 961 #endif //__nvoc_kern_mem_sys_h_disabled 962 963 const MEMORY_SYSTEM_STATIC_CONFIG *kmemsysGetStaticConfig_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 964 965 #ifdef __nvoc_kern_mem_sys_h_disabled 966 static inline const MEMORY_SYSTEM_STATIC_CONFIG *kmemsysGetStaticConfig(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 967 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 968 return NULL; 969 } 970 #else //__nvoc_kern_mem_sys_h_disabled 971 #define kmemsysGetStaticConfig(pGpu, pKernelMemorySystem) kmemsysGetStaticConfig_IMPL(pGpu, pKernelMemorySystem) 972 #endif //__nvoc_kern_mem_sys_h_disabled 973 974 NV_STATUS kmemsysSetupCoherentCpuLink_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bFlush); 975 976 #ifdef __nvoc_kern_mem_sys_h_disabled 977 static inline NV_STATUS kmemsysSetupCoherentCpuLink(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bFlush) { 978 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 979 return NV_ERR_NOT_SUPPORTED; 980 } 981 #else //__nvoc_kern_mem_sys_h_disabled 982 #define kmemsysSetupCoherentCpuLink(pGpu, pKernelMemorySystem, bFlush) kmemsysSetupCoherentCpuLink_IMPL(pGpu, pKernelMemorySystem, bFlush) 983 #endif //__nvoc_kern_mem_sys_h_disabled 984 985 void kmemsysTeardownCoherentCpuLink_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bFlush); 986 987 #ifdef __nvoc_kern_mem_sys_h_disabled 988 static inline void kmemsysTeardownCoherentCpuLink(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvBool bFlush) { 989 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 990 } 991 #else //__nvoc_kern_mem_sys_h_disabled 992 #define kmemsysTeardownCoherentCpuLink(pGpu, pKernelMemorySystem, bFlush) kmemsysTeardownCoherentCpuLink_IMPL(pGpu, pKernelMemorySystem, bFlush) 993 #endif //__nvoc_kern_mem_sys_h_disabled 994 995 NV_STATUS kmemsysSendL2InvalidateEvict_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags); 996 997 #ifdef __nvoc_kern_mem_sys_h_disabled 998 static inline NV_STATUS kmemsysSendL2InvalidateEvict(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 flags) { 999 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1000 return NV_ERR_NOT_SUPPORTED; 1001 } 1002 #else //__nvoc_kern_mem_sys_h_disabled 1003 #define kmemsysSendL2InvalidateEvict(pGpu, pKernelMemorySystem, flags) kmemsysSendL2InvalidateEvict_IMPL(pGpu, pKernelMemorySystem, flags) 1004 #endif //__nvoc_kern_mem_sys_h_disabled 1005 1006 NV_STATUS kmemsysSendFlushL2AllRamsAndCaches_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem); 1007 1008 #ifdef __nvoc_kern_mem_sys_h_disabled 1009 static inline NV_STATUS kmemsysSendFlushL2AllRamsAndCaches(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem) { 1010 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1011 return NV_ERR_NOT_SUPPORTED; 1012 } 1013 #else //__nvoc_kern_mem_sys_h_disabled 1014 #define kmemsysSendFlushL2AllRamsAndCaches(pGpu, pKernelMemorySystem) kmemsysSendFlushL2AllRamsAndCaches_IMPL(pGpu, pKernelMemorySystem) 1015 #endif //__nvoc_kern_mem_sys_h_disabled 1016 1017 NV_STATUS kmemsysSwizzIdToMIGMemSize_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE totalRange, NvU32 *pPartitionSizeFlag, NvU64 *pSizeInBytes); 1018 1019 #ifdef __nvoc_kern_mem_sys_h_disabled 1020 static inline NV_STATUS kmemsysSwizzIdToMIGMemSize(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE totalRange, NvU32 *pPartitionSizeFlag, NvU64 *pSizeInBytes) { 1021 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1022 return NV_ERR_NOT_SUPPORTED; 1023 } 1024 #else //__nvoc_kern_mem_sys_h_disabled 1025 #define kmemsysSwizzIdToMIGMemSize(pGpu, pKernelMemorySystem, swizzId, totalRange, pPartitionSizeFlag, pSizeInBytes) kmemsysSwizzIdToMIGMemSize_IMPL(pGpu, pKernelMemorySystem, swizzId, totalRange, pPartitionSizeFlag, pSizeInBytes) 1026 #endif //__nvoc_kern_mem_sys_h_disabled 1027 1028 NV_STATUS kmemsysSwizzIdToMIGMemRange_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE totalRange, struct NV_RANGE *pAddrRange); 1029 1030 #ifdef __nvoc_kern_mem_sys_h_disabled 1031 static inline NV_STATUS kmemsysSwizzIdToMIGMemRange(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE totalRange, struct NV_RANGE *pAddrRange) { 1032 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1033 return NV_ERR_NOT_SUPPORTED; 1034 } 1035 #else //__nvoc_kern_mem_sys_h_disabled 1036 #define kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, totalRange, pAddrRange) kmemsysSwizzIdToMIGMemRange_IMPL(pGpu, pKernelMemorySystem, swizzId, totalRange, pAddrRange) 1037 #endif //__nvoc_kern_mem_sys_h_disabled 1038 1039 NV_STATUS kmemsysGetMIGGPUInstanceMemInfo_IMPL(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE *pAddrRange); 1040 1041 #ifdef __nvoc_kern_mem_sys_h_disabled 1042 static inline NV_STATUS kmemsysGetMIGGPUInstanceMemInfo(OBJGPU *pGpu, struct KernelMemorySystem *pKernelMemorySystem, NvU32 swizzId, struct NV_RANGE *pAddrRange) { 1043 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1044 return NV_ERR_NOT_SUPPORTED; 1045 } 1046 #else //__nvoc_kern_mem_sys_h_disabled 1047 #define kmemsysGetMIGGPUInstanceMemInfo(pGpu, pKernelMemorySystem, swizzId, pAddrRange) kmemsysGetMIGGPUInstanceMemInfo_IMPL(pGpu, pKernelMemorySystem, swizzId, pAddrRange) 1048 #endif //__nvoc_kern_mem_sys_h_disabled 1049 1050 NV_STATUS kmemsysGetMIGGPUInstanceMemConfigFromSwizzId_IMPL(OBJGPU *arg0, struct KernelMemorySystem *arg1, NvU32 swizzId, const MIG_GPU_INSTANCE_MEMORY_CONFIG **arg2); 1051 1052 #ifdef __nvoc_kern_mem_sys_h_disabled 1053 static inline NV_STATUS kmemsysGetMIGGPUInstanceMemConfigFromSwizzId(OBJGPU *arg0, struct KernelMemorySystem *arg1, NvU32 swizzId, const MIG_GPU_INSTANCE_MEMORY_CONFIG **arg2) { 1054 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1055 return NV_ERR_NOT_SUPPORTED; 1056 } 1057 #else //__nvoc_kern_mem_sys_h_disabled 1058 #define kmemsysGetMIGGPUInstanceMemConfigFromSwizzId(arg0, arg1, swizzId, arg2) kmemsysGetMIGGPUInstanceMemConfigFromSwizzId_IMPL(arg0, arg1, swizzId, arg2) 1059 #endif //__nvoc_kern_mem_sys_h_disabled 1060 1061 NV_STATUS kmemsysInitMIGGPUInstanceMemConfigForSwizzId_IMPL(OBJGPU *arg0, struct KernelMemorySystem *arg1, NvU32 swizzId, NvU64 startingVmmuSegment, NvU64 memSizeInVmmuSegment); 1062 1063 #ifdef __nvoc_kern_mem_sys_h_disabled 1064 static inline NV_STATUS kmemsysInitMIGGPUInstanceMemConfigForSwizzId(OBJGPU *arg0, struct KernelMemorySystem *arg1, NvU32 swizzId, NvU64 startingVmmuSegment, NvU64 memSizeInVmmuSegment) { 1065 NV_ASSERT_FAILED_PRECOMP("KernelMemorySystem was disabled!"); 1066 return NV_ERR_NOT_SUPPORTED; 1067 } 1068 #else //__nvoc_kern_mem_sys_h_disabled 1069 #define kmemsysInitMIGGPUInstanceMemConfigForSwizzId(arg0, arg1, swizzId, startingVmmuSegment, memSizeInVmmuSegment) kmemsysInitMIGGPUInstanceMemConfigForSwizzId_IMPL(arg0, arg1, swizzId, startingVmmuSegment, memSizeInVmmuSegment) 1070 #endif //__nvoc_kern_mem_sys_h_disabled 1071 1072 #undef PRIVATE_FIELD 1073 1074 1075 #define IS_COHERENT_CPU_ATS_OFFSET(kmemsys, offset, length) \ 1076 (kmemsys && ((offset) >= kmemsys->coherentCpuFbBase) && \ 1077 (((NvU64)offset + size) <= kmemsys->coherentCpuFbEnd)) 1078 1079 #endif // KERN_MEM_SYS_H 1080 1081 #ifdef __cplusplus 1082 } // extern "C" 1083 #endif 1084 1085 #endif // _G_KERN_MEM_SYS_NVOC_H_ 1086