1 #ifndef _G_FABRIC_VASPACE_NVOC_H_
2 #define _G_FABRIC_VASPACE_NVOC_H_
3 #include "nvoc/runtime.h"
4 
5 #ifdef __cplusplus
6 extern "C" {
7 #endif
8 
9 /*
10  * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
11  * SPDX-License-Identifier: MIT
12  *
13  * Permission is hereby granted, free of charge, to any person obtaining a
14  * copy of this software and associated documentation files (the "Software"),
15  * to deal in the Software without restriction, including without limitation
16  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17  * and/or sell copies of the Software, and to permit persons to whom the
18  * Software is furnished to do so, subject to the following conditions:
19  *
20  * The above copyright notice and this permission notice shall be included in
21  * all copies or substantial portions of the Software.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
26  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
29  * DEALINGS IN THE SOFTWARE.
30  */
31 
32 #include "g_fabric_vaspace_nvoc.h"
33 
34 #ifndef FABRIC_VASPACE_H
35 #define FABRIC_VASPACE_H
36 
37 /**************** Resource Manager Defines and Structures ******************\
38 *                                                                           *
39 * Module: fabric_vaspace.h                                                  *
40 *    Defines and structures used for Fabric Virtual Address Space Object.   *
41 \***************************************************************************/
42 
43 #include "mmu/mmu_walk.h"
44 #include "mmu/gmmu_fmt.h"
45 #include "core/core.h"
46 #include "mem_mgr/vaspace.h"
47 #include "mem_mgr/gpu_vaspace.h"
48 #include "gpu/mem_mgr/heap.h"
49 #include "gpu/mem_mgr/virt_mem_allocator.h"
50 #include "ctrl/ctrl0080/ctrl0080dma.h"
51 #include "gpu/mem_mgr/mem_desc.h"
52 
53 #include "containers/list.h"
54 #include "containers/map.h"
55 #include "mem_mgr/pool_alloc.h"
56 
57 typedef struct
58 {
59     NODE               Node;
60     MEMORY_DESCRIPTOR *pVidMemDesc;
61     NvU64              offset;
62 } FABRIC_VA_TO_GPA_MAP_NODE;
63 
64 #define FABRIC_VASPACE_MAP_FLAGS_READ_ONLY    NVBIT(0)
65 
66 /*!
67  * RM-registered/managed Fabric virtual address space.
68  */
69 #ifdef NVOC_FABRIC_VASPACE_H_PRIVATE_ACCESS_ALLOWED
70 #define PRIVATE_FIELD(x) x
71 #else
72 #define PRIVATE_FIELD(x) NVOC_PRIVATE_FIELD(x)
73 #endif
74 struct FABRIC_VASPACE {
75     const struct NVOC_RTTI *__nvoc_rtti;
76     struct OBJVASPACE __nvoc_base_OBJVASPACE;
77     struct Object *__nvoc_pbase_Object;
78     struct OBJVASPACE *__nvoc_pbase_OBJVASPACE;
79     struct FABRIC_VASPACE *__nvoc_pbase_FABRIC_VASPACE;
80     NV_STATUS (*__fabricvaspaceConstruct___)(struct FABRIC_VASPACE *, NvU32, NvU32, NvU64, NvU64, NvU64, NvU64, NvU32);
81     NV_STATUS (*__fabricvaspaceAlloc__)(struct FABRIC_VASPACE *, NvU64, NvU64, NvU64, NvU64, NvU64, VAS_ALLOC_FLAGS, NvU64 *);
82     NV_STATUS (*__fabricvaspaceFree__)(struct FABRIC_VASPACE *, NvU64);
83     NV_STATUS (*__fabricvaspaceMap__)(struct FABRIC_VASPACE *, struct OBJGPU *, const NvU64, const NvU64, const MMU_MAP_TARGET *, const VAS_MAP_FLAGS);
84     void (*__fabricvaspaceUnmap__)(struct FABRIC_VASPACE *, struct OBJGPU *, const NvU64, const NvU64);
85     NV_STATUS (*__fabricvaspaceApplyDefaultAlignment__)(struct FABRIC_VASPACE *, const FB_ALLOC_INFO *, NvU64 *, NvU64 *, NvU64 *);
86     NV_STATUS (*__fabricvaspaceGetVasInfo__)(struct FABRIC_VASPACE *, NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS *);
87     NV_STATUS (*__fabricvaspacePinRootPageDir__)(struct FABRIC_VASPACE *, struct OBJGPU *);
88     void (*__fabricvaspaceUnpinRootPageDir__)(struct FABRIC_VASPACE *, struct OBJGPU *);
89     void (*__fabricvaspaceInvalidateTlb__)(struct FABRIC_VASPACE *, struct OBJGPU *, VAS_PTE_UPDATE_TYPE);
90     NvBool (*__fabricvaspaceIsMirrored__)(struct FABRIC_VASPACE *);
91     NvBool (*__fabricvaspaceIsExternallyOwned__)(struct FABRIC_VASPACE *);
92     NvBool (*__fabricvaspaceIsInternalVaRestricted__)(struct FABRIC_VASPACE *);
93     NvU32 (*__fabricvaspaceGetFlags__)(struct FABRIC_VASPACE *);
94     NvBool (*__fabricvaspaceIsAtsEnabled__)(struct FABRIC_VASPACE *);
95     NvU32 (*__fabricvaspaceGetBigPageSize__)(struct FABRIC_VASPACE *);
96     NV_STATUS (*__fabricvaspaceGetPteInfo__)(struct FABRIC_VASPACE *, struct OBJGPU *, NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS *, RmPhysAddr *);
97     NvU64 (*__fabricvaspaceGetVaLimit__)(struct FABRIC_VASPACE *);
98     PMEMORY_DESCRIPTOR (*__fabricvaspaceGetPageDirBase__)(struct FABRIC_VASPACE *, struct OBJGPU *);
99     PMEMORY_DESCRIPTOR (*__fabricvaspaceGetKernelPageDirBase__)(struct FABRIC_VASPACE *, struct OBJGPU *);
100     NvU64 (*__fabricvaspaceGetMapPageSize__)(struct FABRIC_VASPACE *, struct OBJGPU *, EMEMBLOCK *);
101     struct OBJEHEAP *(*__fabricvaspaceGetHeap__)(struct FABRIC_VASPACE *);
102     NvBool (*__fabricvaspaceIsFaultCapable__)(struct FABRIC_VASPACE *);
103     NvU64 (*__fabricvaspaceGetVaStart__)(struct FABRIC_VASPACE *);
104     NV_STATUS (*__fabricvaspaceIncAllocRefCnt__)(struct FABRIC_VASPACE *, NvU64);
105     NV_STATUS (*__fabricvaspaceSetPteInfo__)(struct FABRIC_VASPACE *, struct OBJGPU *, NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS *);
106     NV_STATUS (*__fabricvaspaceFreeV2__)(struct FABRIC_VASPACE *, NvU64, NvU64 *);
107     NV_STATUS (*__fabricvaspaceGetPasid__)(struct FABRIC_VASPACE *, NvU32 *);
108     NV_STATUS (*__fabricvaspaceGetPageTableInfo__)(struct FABRIC_VASPACE *, NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS *);
109     NV_STATUS (*__fabricvaspaceReserveMempool__)(struct FABRIC_VASPACE *, struct OBJGPU *, NvHandle, NvU64, NvU64, NvU32);
110     struct OBJVASPACE *pGVAS;
111     NvU32 flags;
112     NvHandle hClient;
113     NvHandle hDevice;
114     NODE *pFabricVaToGpaMap;
115     NvU64 ucFabricBase;
116     NvU64 ucFabricLimit;
117     NvU64 ucFabricInUseSize;
118     NvU64 ucFabricFreeSize;
119     NvU32 gfid;
120     NvBool bRpcAlloc;
121 };
122 
123 #ifndef __NVOC_CLASS_FABRIC_VASPACE_TYPEDEF__
124 #define __NVOC_CLASS_FABRIC_VASPACE_TYPEDEF__
125 typedef struct FABRIC_VASPACE FABRIC_VASPACE;
126 #endif /* __NVOC_CLASS_FABRIC_VASPACE_TYPEDEF__ */
127 
128 #ifndef __nvoc_class_id_FABRIC_VASPACE
129 #define __nvoc_class_id_FABRIC_VASPACE 0x8c8f3d
130 #endif /* __nvoc_class_id_FABRIC_VASPACE */
131 
132 extern const struct NVOC_CLASS_DEF __nvoc_class_def_FABRIC_VASPACE;
133 
134 #define __staticCast_FABRIC_VASPACE(pThis) \
135     ((pThis)->__nvoc_pbase_FABRIC_VASPACE)
136 
137 #ifdef __nvoc_fabric_vaspace_h_disabled
138 #define __dynamicCast_FABRIC_VASPACE(pThis) ((FABRIC_VASPACE*)NULL)
139 #else //__nvoc_fabric_vaspace_h_disabled
140 #define __dynamicCast_FABRIC_VASPACE(pThis) \
141     ((FABRIC_VASPACE*)__nvoc_dynamicCast(staticCast((pThis), Dynamic), classInfo(FABRIC_VASPACE)))
142 #endif //__nvoc_fabric_vaspace_h_disabled
143 
144 
145 NV_STATUS __nvoc_objCreateDynamic_FABRIC_VASPACE(FABRIC_VASPACE**, Dynamic*, NvU32, va_list);
146 
147 NV_STATUS __nvoc_objCreate_FABRIC_VASPACE(FABRIC_VASPACE**, Dynamic*, NvU32);
148 #define __objCreate_FABRIC_VASPACE(ppNewObj, pParent, createFlags) \
149     __nvoc_objCreate_FABRIC_VASPACE((ppNewObj), staticCast((pParent), Dynamic), (createFlags))
150 
151 #define fabricvaspaceConstruct_(pFabricVAS, classId, vaspaceId, vaStart, vaLimit, vaStartInternal, vaLimitInternal, flags) fabricvaspaceConstruct__DISPATCH(pFabricVAS, classId, vaspaceId, vaStart, vaLimit, vaStartInternal, vaLimitInternal, flags)
152 #define fabricvaspaceAlloc(pFabricVAS, size, align, rangeLo, rangeHi, pageSize, flags, pAddr) fabricvaspaceAlloc_DISPATCH(pFabricVAS, size, align, rangeLo, rangeHi, pageSize, flags, pAddr)
153 #define fabricvaspaceFree(pFabricVAS, vAddr) fabricvaspaceFree_DISPATCH(pFabricVAS, vAddr)
154 #define fabricvaspaceMap(pFabricVAS, pGpu, vaLo, vaHi, pTarget, flags) fabricvaspaceMap_DISPATCH(pFabricVAS, pGpu, vaLo, vaHi, pTarget, flags)
155 #define fabricvaspaceUnmap(pFabricVAS, pGpu, vaLo, vaHi) fabricvaspaceUnmap_DISPATCH(pFabricVAS, pGpu, vaLo, vaHi)
156 #define fabricvaspaceApplyDefaultAlignment(pFabricVAS, pAllocInfo, pAlign, pSize, pPageSizeLockMask) fabricvaspaceApplyDefaultAlignment_DISPATCH(pFabricVAS, pAllocInfo, pAlign, pSize, pPageSizeLockMask)
157 #define fabricvaspaceGetVasInfo(pFabricVAS, pParams) fabricvaspaceGetVasInfo_DISPATCH(pFabricVAS, pParams)
158 #define fabricvaspacePinRootPageDir(pFabricVAS, pGpu) fabricvaspacePinRootPageDir_DISPATCH(pFabricVAS, pGpu)
159 #define fabricvaspaceUnpinRootPageDir(pFabricVAS, pGpu) fabricvaspaceUnpinRootPageDir_DISPATCH(pFabricVAS, pGpu)
160 #define fabricvaspaceInvalidateTlb(pFabricVAS, pGpu, type) fabricvaspaceInvalidateTlb_DISPATCH(pFabricVAS, pGpu, type)
161 #define fabricvaspaceIsMirrored(pVAS) fabricvaspaceIsMirrored_DISPATCH(pVAS)
162 #define fabricvaspaceIsExternallyOwned(pVAS) fabricvaspaceIsExternallyOwned_DISPATCH(pVAS)
163 #define fabricvaspaceIsInternalVaRestricted(pVAS) fabricvaspaceIsInternalVaRestricted_DISPATCH(pVAS)
164 #define fabricvaspaceGetFlags(pVAS) fabricvaspaceGetFlags_DISPATCH(pVAS)
165 #define fabricvaspaceIsAtsEnabled(pVAS) fabricvaspaceIsAtsEnabled_DISPATCH(pVAS)
166 #define fabricvaspaceGetBigPageSize(pVAS) fabricvaspaceGetBigPageSize_DISPATCH(pVAS)
167 #define fabricvaspaceGetPteInfo(pVAS, pGpu, pParams, pPhysAddr) fabricvaspaceGetPteInfo_DISPATCH(pVAS, pGpu, pParams, pPhysAddr)
168 #define fabricvaspaceGetVaLimit(pVAS) fabricvaspaceGetVaLimit_DISPATCH(pVAS)
169 #define fabricvaspaceGetPageDirBase(pVAS, pGpu) fabricvaspaceGetPageDirBase_DISPATCH(pVAS, pGpu)
170 #define fabricvaspaceGetKernelPageDirBase(pVAS, pGpu) fabricvaspaceGetKernelPageDirBase_DISPATCH(pVAS, pGpu)
171 #define fabricvaspaceGetMapPageSize(pVAS, pGpu, pMemBlock) fabricvaspaceGetMapPageSize_DISPATCH(pVAS, pGpu, pMemBlock)
172 #define fabricvaspaceGetHeap(pVAS) fabricvaspaceGetHeap_DISPATCH(pVAS)
173 #define fabricvaspaceIsFaultCapable(pVAS) fabricvaspaceIsFaultCapable_DISPATCH(pVAS)
174 #define fabricvaspaceGetVaStart(pVAS) fabricvaspaceGetVaStart_DISPATCH(pVAS)
175 #define fabricvaspaceIncAllocRefCnt(pVAS, vAddr) fabricvaspaceIncAllocRefCnt_DISPATCH(pVAS, vAddr)
176 #define fabricvaspaceSetPteInfo(pVAS, pGpu, pParams) fabricvaspaceSetPteInfo_DISPATCH(pVAS, pGpu, pParams)
177 #define fabricvaspaceFreeV2(pVAS, vAddr, pSize) fabricvaspaceFreeV2_DISPATCH(pVAS, vAddr, pSize)
178 #define fabricvaspaceGetPasid(pVAS, pPasid) fabricvaspaceGetPasid_DISPATCH(pVAS, pPasid)
179 #define fabricvaspaceGetPageTableInfo(pVAS, pParams) fabricvaspaceGetPageTableInfo_DISPATCH(pVAS, pParams)
180 #define fabricvaspaceReserveMempool(pVAS, pGpu, hClient, size, pageSizeLockMask, flags) fabricvaspaceReserveMempool_DISPATCH(pVAS, pGpu, hClient, size, pageSizeLockMask, flags)
181 NV_STATUS fabricvaspaceConstruct__IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU32 classId, NvU32 vaspaceId, NvU64 vaStart, NvU64 vaLimit, NvU64 vaStartInternal, NvU64 vaLimitInternal, NvU32 flags);
182 
183 static inline NV_STATUS fabricvaspaceConstruct__DISPATCH(struct FABRIC_VASPACE *pFabricVAS, NvU32 classId, NvU32 vaspaceId, NvU64 vaStart, NvU64 vaLimit, NvU64 vaStartInternal, NvU64 vaLimitInternal, NvU32 flags) {
184     return pFabricVAS->__fabricvaspaceConstruct___(pFabricVAS, classId, vaspaceId, vaStart, vaLimit, vaStartInternal, vaLimitInternal, flags);
185 }
186 
187 NV_STATUS fabricvaspaceAlloc_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 size, NvU64 align, NvU64 rangeLo, NvU64 rangeHi, NvU64 pageSize, VAS_ALLOC_FLAGS flags, NvU64 *pAddr);
188 
189 static inline NV_STATUS fabricvaspaceAlloc_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, NvU64 size, NvU64 align, NvU64 rangeLo, NvU64 rangeHi, NvU64 pageSize, VAS_ALLOC_FLAGS flags, NvU64 *pAddr) {
190     return pFabricVAS->__fabricvaspaceAlloc__(pFabricVAS, size, align, rangeLo, rangeHi, pageSize, flags, pAddr);
191 }
192 
193 NV_STATUS fabricvaspaceFree_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr);
194 
195 static inline NV_STATUS fabricvaspaceFree_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr) {
196     return pFabricVAS->__fabricvaspaceFree__(pFabricVAS, vAddr);
197 }
198 
199 NV_STATUS fabricvaspaceMap_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, const NvU64 vaLo, const NvU64 vaHi, const MMU_MAP_TARGET *pTarget, const VAS_MAP_FLAGS flags);
200 
201 static inline NV_STATUS fabricvaspaceMap_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, const NvU64 vaLo, const NvU64 vaHi, const MMU_MAP_TARGET *pTarget, const VAS_MAP_FLAGS flags) {
202     return pFabricVAS->__fabricvaspaceMap__(pFabricVAS, pGpu, vaLo, vaHi, pTarget, flags);
203 }
204 
205 void fabricvaspaceUnmap_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, const NvU64 vaLo, const NvU64 vaHi);
206 
207 static inline void fabricvaspaceUnmap_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, const NvU64 vaLo, const NvU64 vaHi) {
208     pFabricVAS->__fabricvaspaceUnmap__(pFabricVAS, pGpu, vaLo, vaHi);
209 }
210 
211 NV_STATUS fabricvaspaceApplyDefaultAlignment_IMPL(struct FABRIC_VASPACE *pFabricVAS, const FB_ALLOC_INFO *pAllocInfo, NvU64 *pAlign, NvU64 *pSize, NvU64 *pPageSizeLockMask);
212 
213 static inline NV_STATUS fabricvaspaceApplyDefaultAlignment_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, const FB_ALLOC_INFO *pAllocInfo, NvU64 *pAlign, NvU64 *pSize, NvU64 *pPageSizeLockMask) {
214     return pFabricVAS->__fabricvaspaceApplyDefaultAlignment__(pFabricVAS, pAllocInfo, pAlign, pSize, pPageSizeLockMask);
215 }
216 
217 NV_STATUS fabricvaspaceGetVasInfo_IMPL(struct FABRIC_VASPACE *pFabricVAS, NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS *pParams);
218 
219 static inline NV_STATUS fabricvaspaceGetVasInfo_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS *pParams) {
220     return pFabricVAS->__fabricvaspaceGetVasInfo__(pFabricVAS, pParams);
221 }
222 
223 NV_STATUS fabricvaspacePinRootPageDir_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu);
224 
225 static inline NV_STATUS fabricvaspacePinRootPageDir_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu) {
226     return pFabricVAS->__fabricvaspacePinRootPageDir__(pFabricVAS, pGpu);
227 }
228 
229 void fabricvaspaceUnpinRootPageDir_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu);
230 
231 static inline void fabricvaspaceUnpinRootPageDir_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu) {
232     pFabricVAS->__fabricvaspaceUnpinRootPageDir__(pFabricVAS, pGpu);
233 }
234 
235 void fabricvaspaceInvalidateTlb_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, VAS_PTE_UPDATE_TYPE type);
236 
237 static inline void fabricvaspaceInvalidateTlb_DISPATCH(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, VAS_PTE_UPDATE_TYPE type) {
238     pFabricVAS->__fabricvaspaceInvalidateTlb__(pFabricVAS, pGpu, type);
239 }
240 
241 static inline NvBool fabricvaspaceIsMirrored_DISPATCH(struct FABRIC_VASPACE *pVAS) {
242     return pVAS->__fabricvaspaceIsMirrored__(pVAS);
243 }
244 
245 static inline NvBool fabricvaspaceIsExternallyOwned_DISPATCH(struct FABRIC_VASPACE *pVAS) {
246     return pVAS->__fabricvaspaceIsExternallyOwned__(pVAS);
247 }
248 
249 static inline NvBool fabricvaspaceIsInternalVaRestricted_DISPATCH(struct FABRIC_VASPACE *pVAS) {
250     return pVAS->__fabricvaspaceIsInternalVaRestricted__(pVAS);
251 }
252 
253 static inline NvU32 fabricvaspaceGetFlags_DISPATCH(struct FABRIC_VASPACE *pVAS) {
254     return pVAS->__fabricvaspaceGetFlags__(pVAS);
255 }
256 
257 static inline NvBool fabricvaspaceIsAtsEnabled_DISPATCH(struct FABRIC_VASPACE *pVAS) {
258     return pVAS->__fabricvaspaceIsAtsEnabled__(pVAS);
259 }
260 
261 static inline NvU32 fabricvaspaceGetBigPageSize_DISPATCH(struct FABRIC_VASPACE *pVAS) {
262     return pVAS->__fabricvaspaceGetBigPageSize__(pVAS);
263 }
264 
265 static inline NV_STATUS fabricvaspaceGetPteInfo_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu, NV0080_CTRL_DMA_GET_PTE_INFO_PARAMS *pParams, RmPhysAddr *pPhysAddr) {
266     return pVAS->__fabricvaspaceGetPteInfo__(pVAS, pGpu, pParams, pPhysAddr);
267 }
268 
269 static inline NvU64 fabricvaspaceGetVaLimit_DISPATCH(struct FABRIC_VASPACE *pVAS) {
270     return pVAS->__fabricvaspaceGetVaLimit__(pVAS);
271 }
272 
273 static inline PMEMORY_DESCRIPTOR fabricvaspaceGetPageDirBase_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu) {
274     return pVAS->__fabricvaspaceGetPageDirBase__(pVAS, pGpu);
275 }
276 
277 static inline PMEMORY_DESCRIPTOR fabricvaspaceGetKernelPageDirBase_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu) {
278     return pVAS->__fabricvaspaceGetKernelPageDirBase__(pVAS, pGpu);
279 }
280 
281 static inline NvU64 fabricvaspaceGetMapPageSize_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu, EMEMBLOCK *pMemBlock) {
282     return pVAS->__fabricvaspaceGetMapPageSize__(pVAS, pGpu, pMemBlock);
283 }
284 
285 static inline struct OBJEHEAP *fabricvaspaceGetHeap_DISPATCH(struct FABRIC_VASPACE *pVAS) {
286     return pVAS->__fabricvaspaceGetHeap__(pVAS);
287 }
288 
289 static inline NvBool fabricvaspaceIsFaultCapable_DISPATCH(struct FABRIC_VASPACE *pVAS) {
290     return pVAS->__fabricvaspaceIsFaultCapable__(pVAS);
291 }
292 
293 static inline NvU64 fabricvaspaceGetVaStart_DISPATCH(struct FABRIC_VASPACE *pVAS) {
294     return pVAS->__fabricvaspaceGetVaStart__(pVAS);
295 }
296 
297 static inline NV_STATUS fabricvaspaceIncAllocRefCnt_DISPATCH(struct FABRIC_VASPACE *pVAS, NvU64 vAddr) {
298     return pVAS->__fabricvaspaceIncAllocRefCnt__(pVAS, vAddr);
299 }
300 
301 static inline NV_STATUS fabricvaspaceSetPteInfo_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu, NV0080_CTRL_DMA_SET_PTE_INFO_PARAMS *pParams) {
302     return pVAS->__fabricvaspaceSetPteInfo__(pVAS, pGpu, pParams);
303 }
304 
305 static inline NV_STATUS fabricvaspaceFreeV2_DISPATCH(struct FABRIC_VASPACE *pVAS, NvU64 vAddr, NvU64 *pSize) {
306     return pVAS->__fabricvaspaceFreeV2__(pVAS, vAddr, pSize);
307 }
308 
309 static inline NV_STATUS fabricvaspaceGetPasid_DISPATCH(struct FABRIC_VASPACE *pVAS, NvU32 *pPasid) {
310     return pVAS->__fabricvaspaceGetPasid__(pVAS, pPasid);
311 }
312 
313 static inline NV_STATUS fabricvaspaceGetPageTableInfo_DISPATCH(struct FABRIC_VASPACE *pVAS, NV0080_CTRL_DMA_GET_PDE_INFO_PARAMS *pParams) {
314     return pVAS->__fabricvaspaceGetPageTableInfo__(pVAS, pParams);
315 }
316 
317 static inline NV_STATUS fabricvaspaceReserveMempool_DISPATCH(struct FABRIC_VASPACE *pVAS, struct OBJGPU *pGpu, NvHandle hClient, NvU64 size, NvU64 pageSizeLockMask, NvU32 flags) {
318     return pVAS->__fabricvaspaceReserveMempool__(pVAS, pGpu, hClient, size, pageSizeLockMask, flags);
319 }
320 
321 static inline NvU64 fabricvaspaceGetUCFlaStart(struct FABRIC_VASPACE *pFabricVAS) {
322     return pFabricVAS->ucFabricBase;
323 }
324 
325 static inline NvU64 fabricvaspaceGetUCFlaLimit(struct FABRIC_VASPACE *pFabricVAS) {
326     return pFabricVAS->ucFabricLimit;
327 }
328 
329 void fabricvaspaceDestruct_IMPL(struct FABRIC_VASPACE *pFabricVAS);
330 
331 #define __nvoc_fabricvaspaceDestruct(pFabricVAS) fabricvaspaceDestruct_IMPL(pFabricVAS)
332 NV_STATUS fabricvaspaceAllocNonContiguous_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 size, NvU64 align, NvU64 rangeLo, NvU64 rangeHi, NvU64 pageSize, VAS_ALLOC_FLAGS flags, NvU64 **ppAddr, NvU32 *pNumAddr);
333 
334 #ifdef __nvoc_fabric_vaspace_h_disabled
335 static inline NV_STATUS fabricvaspaceAllocNonContiguous(struct FABRIC_VASPACE *pFabricVAS, NvU64 size, NvU64 align, NvU64 rangeLo, NvU64 rangeHi, NvU64 pageSize, VAS_ALLOC_FLAGS flags, NvU64 **ppAddr, NvU32 *pNumAddr) {
336     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
337     return NV_ERR_NOT_SUPPORTED;
338 }
339 #else //__nvoc_fabric_vaspace_h_disabled
340 #define fabricvaspaceAllocNonContiguous(pFabricVAS, size, align, rangeLo, rangeHi, pageSize, flags, ppAddr, pNumAddr) fabricvaspaceAllocNonContiguous_IMPL(pFabricVAS, size, align, rangeLo, rangeHi, pageSize, flags, ppAddr, pNumAddr)
341 #endif //__nvoc_fabric_vaspace_h_disabled
342 
343 void fabricvaspaceBatchFree_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 *pAddr, NvU32 numAddr, NvU32 stride);
344 
345 #ifdef __nvoc_fabric_vaspace_h_disabled
346 static inline void fabricvaspaceBatchFree(struct FABRIC_VASPACE *pFabricVAS, NvU64 *pAddr, NvU32 numAddr, NvU32 stride) {
347     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
348 }
349 #else //__nvoc_fabric_vaspace_h_disabled
350 #define fabricvaspaceBatchFree(pFabricVAS, pAddr, numAddr, stride) fabricvaspaceBatchFree_IMPL(pFabricVAS, pAddr, numAddr, stride)
351 #endif //__nvoc_fabric_vaspace_h_disabled
352 
353 NV_STATUS fabricvaspaceGetFreeHeap_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 *pFreeSize);
354 
355 #ifdef __nvoc_fabric_vaspace_h_disabled
356 static inline NV_STATUS fabricvaspaceGetFreeHeap(struct FABRIC_VASPACE *pFabricVAS, NvU64 *pFreeSize) {
357     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
358     return NV_ERR_NOT_SUPPORTED;
359 }
360 #else //__nvoc_fabric_vaspace_h_disabled
361 #define fabricvaspaceGetFreeHeap(pFabricVAS, pFreeSize) fabricvaspaceGetFreeHeap_IMPL(pFabricVAS, pFreeSize)
362 #endif //__nvoc_fabric_vaspace_h_disabled
363 
364 NV_STATUS fabricvaspaceGetGpaMemdesc_IMPL(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemdesc, struct OBJGPU *pMappingGpu, MEMORY_DESCRIPTOR **ppAdjustedMemdesc);
365 
366 #ifdef __nvoc_fabric_vaspace_h_disabled
367 static inline NV_STATUS fabricvaspaceGetGpaMemdesc(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemdesc, struct OBJGPU *pMappingGpu, MEMORY_DESCRIPTOR **ppAdjustedMemdesc) {
368     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
369     return NV_ERR_NOT_SUPPORTED;
370 }
371 #else //__nvoc_fabric_vaspace_h_disabled
372 #define fabricvaspaceGetGpaMemdesc(pFabricVAS, pFabricMemdesc, pMappingGpu, ppAdjustedMemdesc) fabricvaspaceGetGpaMemdesc_IMPL(pFabricVAS, pFabricMemdesc, pMappingGpu, ppAdjustedMemdesc)
373 #endif //__nvoc_fabric_vaspace_h_disabled
374 
375 void fabricvaspacePutGpaMemdesc_IMPL(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pMemDesc);
376 
377 #ifdef __nvoc_fabric_vaspace_h_disabled
378 static inline void fabricvaspacePutGpaMemdesc(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pMemDesc) {
379     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
380 }
381 #else //__nvoc_fabric_vaspace_h_disabled
382 #define fabricvaspacePutGpaMemdesc(pFabricVAS, pMemDesc) fabricvaspacePutGpaMemdesc_IMPL(pFabricVAS, pMemDesc)
383 #endif //__nvoc_fabric_vaspace_h_disabled
384 
385 NV_STATUS fabricvaspaceVaToGpaMapInsert_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr, MEMORY_DESCRIPTOR *pVidMemDesc, NvU64 offset);
386 
387 #ifdef __nvoc_fabric_vaspace_h_disabled
388 static inline NV_STATUS fabricvaspaceVaToGpaMapInsert(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr, MEMORY_DESCRIPTOR *pVidMemDesc, NvU64 offset) {
389     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
390     return NV_ERR_NOT_SUPPORTED;
391 }
392 #else //__nvoc_fabric_vaspace_h_disabled
393 #define fabricvaspaceVaToGpaMapInsert(pFabricVAS, vAddr, pVidMemDesc, offset) fabricvaspaceVaToGpaMapInsert_IMPL(pFabricVAS, vAddr, pVidMemDesc, offset)
394 #endif //__nvoc_fabric_vaspace_h_disabled
395 
396 void fabricvaspaceVaToGpaMapRemove_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr);
397 
398 #ifdef __nvoc_fabric_vaspace_h_disabled
399 static inline void fabricvaspaceVaToGpaMapRemove(struct FABRIC_VASPACE *pFabricVAS, NvU64 vAddr) {
400     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
401 }
402 #else //__nvoc_fabric_vaspace_h_disabled
403 #define fabricvaspaceVaToGpaMapRemove(pFabricVAS, vAddr) fabricvaspaceVaToGpaMapRemove_IMPL(pFabricVAS, vAddr)
404 #endif //__nvoc_fabric_vaspace_h_disabled
405 
406 NV_STATUS fabricvaspaceAllocMulticast_IMPL(struct FABRIC_VASPACE *pFabricVAS, NvU64 pageSize, NvU64 alignment, VAS_ALLOC_FLAGS flags, NvU64 base, NvU64 size);
407 
408 #ifdef __nvoc_fabric_vaspace_h_disabled
409 static inline NV_STATUS fabricvaspaceAllocMulticast(struct FABRIC_VASPACE *pFabricVAS, NvU64 pageSize, NvU64 alignment, VAS_ALLOC_FLAGS flags, NvU64 base, NvU64 size) {
410     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
411     return NV_ERR_NOT_SUPPORTED;
412 }
413 #else //__nvoc_fabric_vaspace_h_disabled
414 #define fabricvaspaceAllocMulticast(pFabricVAS, pageSize, alignment, flags, base, size) fabricvaspaceAllocMulticast_IMPL(pFabricVAS, pageSize, alignment, flags, base, size)
415 #endif //__nvoc_fabric_vaspace_h_disabled
416 
417 NV_STATUS fabricvaspaceMapPhysMemdesc_IMPL(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemDesc, NvU64 fabricOffset, MEMORY_DESCRIPTOR *pPhysMemDesc, NvU64 physOffset, NvU64 physMapLength, NvU32 flags);
418 
419 #ifdef __nvoc_fabric_vaspace_h_disabled
420 static inline NV_STATUS fabricvaspaceMapPhysMemdesc(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemDesc, NvU64 fabricOffset, MEMORY_DESCRIPTOR *pPhysMemDesc, NvU64 physOffset, NvU64 physMapLength, NvU32 flags) {
421     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
422     return NV_ERR_NOT_SUPPORTED;
423 }
424 #else //__nvoc_fabric_vaspace_h_disabled
425 #define fabricvaspaceMapPhysMemdesc(pFabricVAS, pFabricMemDesc, fabricOffset, pPhysMemDesc, physOffset, physMapLength, flags) fabricvaspaceMapPhysMemdesc_IMPL(pFabricVAS, pFabricMemDesc, fabricOffset, pPhysMemDesc, physOffset, physMapLength, flags)
426 #endif //__nvoc_fabric_vaspace_h_disabled
427 
428 void fabricvaspaceUnmapPhysMemdesc_IMPL(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemDesc, NvU64 fabricOffset, MEMORY_DESCRIPTOR *pPhysMemDesc, NvU64 physMapLength);
429 
430 #ifdef __nvoc_fabric_vaspace_h_disabled
431 static inline void fabricvaspaceUnmapPhysMemdesc(struct FABRIC_VASPACE *pFabricVAS, MEMORY_DESCRIPTOR *pFabricMemDesc, NvU64 fabricOffset, MEMORY_DESCRIPTOR *pPhysMemDesc, NvU64 physMapLength) {
432     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
433 }
434 #else //__nvoc_fabric_vaspace_h_disabled
435 #define fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, fabricOffset, pPhysMemDesc, physMapLength) fabricvaspaceUnmapPhysMemdesc_IMPL(pFabricVAS, pFabricMemDesc, fabricOffset, pPhysMemDesc, physMapLength)
436 #endif //__nvoc_fabric_vaspace_h_disabled
437 
438 NV_STATUS fabricvaspaceInitUCRange_IMPL(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, NvU64 ucFabricBase, NvU64 ucFabricSize);
439 
440 #ifdef __nvoc_fabric_vaspace_h_disabled
441 static inline NV_STATUS fabricvaspaceInitUCRange(struct FABRIC_VASPACE *pFabricVAS, struct OBJGPU *pGpu, NvU64 ucFabricBase, NvU64 ucFabricSize) {
442     NV_ASSERT_FAILED_PRECOMP("FABRIC_VASPACE was disabled!");
443     return NV_ERR_NOT_SUPPORTED;
444 }
445 #else //__nvoc_fabric_vaspace_h_disabled
446 #define fabricvaspaceInitUCRange(pFabricVAS, pGpu, ucFabricBase, ucFabricSize) fabricvaspaceInitUCRange_IMPL(pFabricVAS, pGpu, ucFabricBase, ucFabricSize)
447 #endif //__nvoc_fabric_vaspace_h_disabled
448 
449 #undef PRIVATE_FIELD
450 
451 
452 #endif // FABRIC_VASPACE_H
453 
454 #ifdef __cplusplus
455 } // extern "C"
456 #endif
457 #endif // _G_FABRIC_VASPACE_NVOC_H_
458