1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 /******************************************************************************
25  *
26  *   Description:
27  *       This file contains the functions managing the memory fabric
28  *
29  *****************************************************************************/
30 
31 #include "core/core.h"
32 #include "core/locks.h"
33 #include "rmapi/resource.h"
34 #include "rmapi/rs_utils.h"
35 #include "mem_mgr_internal.h"
36 #include "mem_mgr/mem_fabric.h"
37 #include "mem_mgr/fabric_vaspace.h"
38 #include "mem_mgr/mem.h"
39 #include "mem_mgr/vaspace.h"
40 #include "gpu/mem_mgr/mem_mgr.h"
41 #include "gpu/mem_mgr/mem_utils.h"
42 #include "gpu/device/device.h"
43 #include "os/os.h"
44 #include "compute/fabric.h"
45 #include "gpu/mem_mgr/mem_desc.h"
46 #include "gpu/gpu.h"
47 #include "class/cl00f8.h"
48 #include "Nvcm.h"
49 #include "vgpu/rpc.h"
50 #include "gpu/bus/kern_bus.h"
51 #include "gpu/bus/p2p_api.h"
52 #include "kernel/gpu/nvlink/kernel_nvlink.h"
53 #include "kernel/gpu/gpu_fabric_probe.h"
54 #include "ctrl/ctrl0041.h"
55 
56 static NvU32
_memoryfabricMemDescGetNumAddr(MEMORY_DESCRIPTOR * pMemDesc)57 _memoryfabricMemDescGetNumAddr
58 (
59     MEMORY_DESCRIPTOR *pMemDesc
60 )
61 {
62     OBJGPU *pGpu     = pMemDesc->pGpu;
63     NvU64   pageSize = 0;
64 
65     // Get the page size from the memory descriptor.
66     pageSize = memdescGetPageSize(pMemDesc,
67                             VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
68 
69     // Get the number of addresses associated with this memory descriptor.
70     if (memdescGetContiguity(pMemDesc,
71                             VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS)))
72     {
73         // For contiguous allocation, there is just one entry _pteArray[0].
74         return 1;
75     }
76 
77     // For discontiguous allocations, numAddr is total size / page size.
78     return (memdescGetSize(pMemDesc) / pageSize);
79 }
80 
81 static NV_STATUS
_memoryfabricValidatePhysMem(RsClient * pRsClient,NvHandle hPhysMem,OBJGPU * pOwnerGpu,MEMORY_DESCRIPTOR ** ppPhysMemDesc)82 _memoryfabricValidatePhysMem
83 (
84     RsClient          *pRsClient,
85     NvHandle           hPhysMem,
86     OBJGPU            *pOwnerGpu,
87     MEMORY_DESCRIPTOR **ppPhysMemDesc
88 )
89 {
90     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pOwnerGpu);
91     MEMORY_DESCRIPTOR *pPhysMemDesc;
92     NvU64 physPageSize;
93     NV_STATUS status;
94     Memory *pMemory;
95 
96     status = memGetByHandle(pRsClient, hPhysMem, &pMemory);
97     if (status != NV_OK)
98     {
99         NV_PRINTF(LEVEL_ERROR, "Invalid object handle passed\n");
100         return status;
101     }
102 
103     pPhysMemDesc = pMemory->pMemDesc;
104 
105     if ((pOwnerGpu != pPhysMemDesc->pGpu) ||
106         !memmgrIsMemDescSupportedByFla_HAL(pOwnerGpu,
107                                            pMemoryManager,
108                                            pPhysMemDesc))
109     {
110         NV_PRINTF(LEVEL_ERROR, "Invalid physmem handle passed\n");
111 
112         return NV_ERR_INVALID_ARGUMENT;
113     }
114 
115     physPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU);
116     if (
117         (physPageSize != RM_PAGE_SIZE_HUGE) &&
118         (physPageSize != RM_PAGE_SIZE_512M))
119     {
120         NV_PRINTF(LEVEL_ERROR, "Physmem page size should be 2MB\n");
121 
122         return NV_ERR_INVALID_ARGUMENT;
123     }
124 
125     *ppPhysMemDesc = pPhysMemDesc;
126 
127     return NV_OK;
128 }
129 
130 static NV_STATUS
_memoryFabricDetachMem(MEMORY_DESCRIPTOR * pFabricMemDesc,NvU64 offset)131 _memoryFabricDetachMem
132 (
133     MEMORY_DESCRIPTOR   *pFabricMemDesc,
134     NvU64                offset
135 )
136 {
137     NV_STATUS status;
138     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
139     FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
140     NODE *pNode = NULL;
141     FABRIC_VASPACE *pFabricVAS;
142     MEMORY_DESCRIPTOR *pPhysMemDesc;
143     FABRIC_MEMDESC_DATA *pMemdescData;
144 
145     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
146 
147     status = btreeSearch(offset, &pNode, pMemdescData->pAttachMemInfoTree);
148     if (status != NV_OK)
149         return status;
150 
151     pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE *)pNode->Data;
152     pPhysMemDesc = pAttachMemInfoNode->pPhysMemDesc;
153     pFabricVAS = dynamicCast(pPhysMemDesc->pGpu->pFabricVAS, FABRIC_VASPACE);
154 
155     fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, offset,
156                                   pPhysMemDesc,
157                                   pAttachMemInfoNode->physMapLength);
158 
159     NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient,
160                               pAttachMemInfoNode->hDupedPhysMem));
161 
162     btreeUnlink(&pAttachMemInfoNode->node, &pMemdescData->pAttachMemInfoTree);
163 
164     portMemFree(pAttachMemInfoNode);
165 
166     return NV_OK;
167 }
168 
169 static void
_memoryFabricBatchDetachMem(MEMORY_DESCRIPTOR * pFabricMemDesc)170 _memoryFabricBatchDetachMem
171 (
172     MEMORY_DESCRIPTOR *pFabricMemDesc
173 )
174 {
175     FABRIC_MEMDESC_DATA *pMemdescData;
176     NODE *pNode = NULL;
177     NvU64 offset;
178     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
179 
180     if (pMemdescData == NULL)
181         return;
182 
183     btreeEnumStart(0, &pNode, pMemdescData->pAttachMemInfoTree);
184     while (pNode != NULL)
185     {
186         offset = pNode->keyStart;
187         btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
188         NV_ASSERT_OK(_memoryFabricDetachMem(pFabricMemDesc, offset));
189     }
190 }
191 
192 static NV_STATUS
_memoryFabricAttachMem(MemoryFabric * pMemoryFabric,NV00F8_CTRL_ATTACH_MEM_INFO * pAttachInfo)193 _memoryFabricAttachMem
194 (
195     MemoryFabric                *pMemoryFabric,
196     NV00F8_CTRL_ATTACH_MEM_INFO *pAttachInfo
197 )
198 {
199     NV_STATUS status;
200     Memory *pMemory = staticCast(pMemoryFabric, Memory);
201     OBJGPU *pGpu = pMemory->pGpu;
202     MEMORY_DESCRIPTOR *pPhysMemDesc;
203     MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
204     FABRIC_MEMDESC_DATA *pMemdescData;
205     NvHandle hDupedPhysMem = 0;
206     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
207     FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
208     FABRIC_ATTCH_MEM_INFO_NODE *pNode;
209 
210     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
211 
212     if (!(pMemdescData->allocFlags & NV00F8_ALLOC_FLAGS_FLEXIBLE_FLA))
213     {
214         NV_PRINTF(LEVEL_ERROR, "Unsupported fabric memory type\n");
215         return NV_ERR_NOT_SUPPORTED;
216     }
217 
218     if (gpuIsCCFeatureEnabled(pGpu) && !gpuIsCCMultiGpuProtectedPcieModeEnabled(pGpu))
219     {
220         NV_PRINTF(LEVEL_ERROR,
221                   "Unsupported when Confidential Computing is enabled in SPT\n");
222         return NV_ERR_NOT_SUPPORTED;
223     }
224 
225     status = _memoryfabricValidatePhysMem(RES_GET_CLIENT(pMemory),
226                                           pAttachInfo->hMemory,
227                                           pGpu, &pPhysMemDesc);
228 
229     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status);
230 
231     status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient, pFabricVAS->hDevice,
232                                &hDupedPhysMem, RES_GET_CLIENT_HANDLE(pMemory),
233                                pAttachInfo->hMemory, 0);
234     if (status != NV_OK)
235     {
236         NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
237         return status;
238     }
239 
240     status = fabricvaspaceMapPhysMemdesc(pFabricVAS,
241                                          pFabricMemDesc,
242                                          pAttachInfo->offset,
243                                          pPhysMemDesc,
244                                          pAttachInfo->mapOffset,
245                                          pAttachInfo->mapLength,
246                                          0);
247     if (status != NV_OK)
248     {
249         NV_PRINTF(LEVEL_ERROR, "Failed to map FLA\n");
250         goto freeDupedMem;
251     }
252 
253     pNode = portMemAllocNonPaged(sizeof(*pNode));
254     if (pNode == NULL)
255     {
256         status = NV_ERR_NO_MEMORY;
257         goto unmapVas;
258     }
259 
260     portMemSet(pNode, 0, sizeof(*pNode));
261 
262     pNode->node.keyStart = pAttachInfo->offset;
263     pNode->node.keyEnd   = pAttachInfo->offset;
264     pNode->physMapOffset = pAttachInfo->mapOffset;
265     pNode->physMapLength = pAttachInfo->mapLength;
266     pNode->pPhysMemDesc  = pPhysMemDesc;
267     pNode->hDupedPhysMem = hDupedPhysMem;
268     pNode->node.Data     = pNode;
269 
270     status = btreeInsert(&pNode->node, &pMemdescData->pAttachMemInfoTree);
271     if (status != NV_OK)
272     {
273         NV_PRINTF(LEVEL_ERROR, "Failed to track attach mem info\n");
274         goto freeNode;
275     }
276 
277     return NV_OK;
278 
279 freeNode:
280     portMemFree(pNode);
281 
282 unmapVas:
283     fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc,
284                                   pAttachInfo->offset,
285                                   pPhysMemDesc, pAttachInfo->mapLength);
286 
287 freeDupedMem:
288     NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient, hDupedPhysMem));
289 
290     return status;
291 }
292 
293 static void
_memoryfabricMemDescDestroyCallback(OBJGPU * pGpu,void * pObject,MEMORY_DESCRIPTOR * pMemDesc)294 _memoryfabricMemDescDestroyCallback
295 (
296     OBJGPU            *pGpu,
297     void              *pObject,
298     MEMORY_DESCRIPTOR *pMemDesc
299 )
300 {
301     RM_API              *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
302     FABRIC_VASPACE      *pFabricVAS;
303     RmPhysAddr          *pteArray;
304     FABRIC_MEMDESC_DATA *pMemdescData;
305     NvU32                numAddr;
306     NvU64                pageSize;
307 
308     NV_ASSERT_OR_RETURN_VOID(pGpu->pFabricVAS != NULL);
309 
310     pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
311     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemDesc);
312     pteArray = memdescGetPteArrayForGpu(pMemDesc, pGpu,
313                                     VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
314     numAddr = _memoryfabricMemDescGetNumAddr(pMemDesc);
315     // Get the page size from the memory descriptor.
316     pageSize = memdescGetPageSize(pMemDesc,
317                                     VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
318 
319     // Remove the fabric memory allocations from the map.
320     fabricvaspaceVaToGpaMapRemove(pFabricVAS, pteArray[0]);
321 
322     // Detach any pending memory...
323     _memoryFabricBatchDetachMem(pMemDesc);
324 
325     if (!pFabricVAS->bRpcAlloc)
326     {
327         //
328         // Call fabricvaspaceBatchFree to free the FLA allocations.
329         // _pteArray in memdesc is RM_PAGE_SIZE whereas page size for memory
330         // fabric allocations is either 2MB or 512MB. Pass stride accordingly.
331         //
332         fabricvaspaceBatchFree(pFabricVAS, pteArray, numAddr,
333                                (pageSize >> RM_PAGE_SHIFT));
334     }
335 
336     if (pMemdescData != NULL)
337     {
338         if (pMemdescData->hDupedPhysMem != 0)
339         {
340             NV_ASSERT(pRmApi->Free(pRmApi, pFabricVAS->hClient,
341                                    pMemdescData->hDupedPhysMem) == NV_OK);
342         }
343 
344         portMemFree(pMemDesc->_pMemData);
345         memdescSetMemData(pMemDesc, NULL, NULL);
346     }
347 
348     portMemFree(pObject);
349 }
350 
351 static void
_memoryfabricFreeFabricVa_VGPU(OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)352 _memoryfabricFreeFabricVa_VGPU
353 (
354     OBJGPU                       *pGpu,
355     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
356 )
357 {
358     NV_STATUS status = NV_OK;
359     NV_RM_RPC_FREE(pGpu, pParams->hClient,
360                    pParams->hParent, pParams->hResource, status);
361     NV_ASSERT(status == NV_OK);
362 }
363 
364 static void
_memoryfabricFreeFabricVa(FABRIC_VASPACE * pFabricVAS,OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NvU64 * pAddr,NvU32 numAddr)365 _memoryfabricFreeFabricVa
366 (
367     FABRIC_VASPACE               *pFabricVAS,
368     OBJGPU                       *pGpu,
369     RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
370     NvU64                        *pAddr,
371     NvU32                         numAddr
372 )
373 {
374     if (pFabricVAS->bRpcAlloc)
375     {
376         _memoryfabricFreeFabricVa_VGPU(pGpu, pParams);
377     }
378     else
379     {
380         fabricvaspaceBatchFree(pFabricVAS, pAddr, numAddr, 1);
381     }
382 }
383 
384 static NV_STATUS
_memoryfabricAllocFabricVa_VGPU(OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NV00F8_ALLOCATION_PARAMETERS * pAllocParams,NvU64 ** ppAddr,NvU32 * pNumAddr)385 _memoryfabricAllocFabricVa_VGPU
386 (
387     OBJGPU                       *pGpu,
388     RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
389     NV00F8_ALLOCATION_PARAMETERS *pAllocParams,
390     NvU64                       **ppAddr,
391     NvU32                        *pNumAddr
392 )
393 {
394     NV00F8_CTRL_DESCRIBE_PARAMS *pDescribeParams = NULL;
395     NvU32  i = 0;
396     NV_STATUS status = NV_OK;
397     NvU32 idx = 0;
398     NvU64 *pAddr  = NULL;
399 
400     NV_RM_RPC_ALLOC_OBJECT(pGpu, pParams->hClient,
401                            pParams->hParent,
402                            pParams->hResource,
403                            pParams->externalClassId,
404                            pAllocParams,
405                            sizeof(*pAllocParams),
406                            status);
407 
408     if (status != NV_OK)
409     {
410         NV_PRINTF(LEVEL_ERROR,
411                   "Alloc NV_MEMORY_FABRIC RPC failed, status: %x\n",
412                   status);
413         return status;
414     }
415 
416     pDescribeParams = portMemAllocNonPaged(sizeof(*pDescribeParams));
417     if (pDescribeParams == NULL)
418     {
419         status = NV_ERR_NO_MEMORY;
420         goto cleanup;
421     }
422 
423     portMemSet(pDescribeParams, 0, sizeof(*pDescribeParams));
424 
425     do
426     {
427         pDescribeParams->offset = idx;
428         NV_RM_RPC_CONTROL(pGpu, pParams->hClient,
429                           pParams->hResource,
430                           NV00F8_CTRL_CMD_DESCRIBE,
431                           pDescribeParams,
432                           sizeof(*pDescribeParams), status);
433         if (status != NV_OK)
434         {
435             NV_PRINTF(LEVEL_ERROR, "CTRL_CMD_DESCRIBE failed, status: 0x%x, "
436                       "numPfns: 0x%x, totalPfns: 0x%llx, readSoFar: 0x%x \n",
437                       status, pDescribeParams->numPfns,
438                       pDescribeParams->totalPfns, idx);
439             goto cleanup;
440         }
441 
442         if (pAddr == NULL)
443         {
444             pAddr = portMemAllocNonPaged(sizeof(NvU64) *
445                                          pDescribeParams->totalPfns);
446             if (pAddr == NULL)
447             {
448                 status = NV_ERR_NO_MEMORY;
449                 goto cleanup;
450             }
451         }
452 
453         for (i=0; i < pDescribeParams->numPfns; i++)
454         {
455             pAddr[idx + i] =
456                 (NvU64)((NvU64)pDescribeParams->pfnArray[i] << RM_PAGE_SHIFT_HUGE);
457         }
458 
459         idx += pDescribeParams->numPfns;
460     } while (idx < pDescribeParams->totalPfns);
461 
462     portMemFree(pDescribeParams);
463 
464     *ppAddr   = pAddr;
465     *pNumAddr = idx;
466 
467     return status;
468 
469 cleanup:
470     portMemFree(pAddr);
471     portMemFree(pDescribeParams);
472 
473     _memoryfabricFreeFabricVa_VGPU(pGpu, pParams);
474 
475     return status;
476 }
477 
478 static NV_STATUS
_memoryfabricAllocFabricVa(FABRIC_VASPACE * pFabricVAS,OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NV00F8_ALLOCATION_PARAMETERS * pAllocParams,VAS_ALLOC_FLAGS flags,NvU64 ** ppAddr,NvU32 * pNumAddr)479 _memoryfabricAllocFabricVa
480 (
481     FABRIC_VASPACE               *pFabricVAS,
482     OBJGPU                       *pGpu,
483     RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
484     NV00F8_ALLOCATION_PARAMETERS *pAllocParams,
485     VAS_ALLOC_FLAGS               flags,
486     NvU64                       **ppAddr,
487     NvU32                        *pNumAddr
488 )
489 {
490     if (pFabricVAS->bRpcAlloc)
491     {
492         return _memoryfabricAllocFabricVa_VGPU(pGpu, pParams,
493                                                pAllocParams, ppAddr,
494                                                pNumAddr);
495     }
496     else
497     {
498         return fabricvaspaceAllocNonContiguous(pFabricVAS,
499                                                pAllocParams->allocSize,
500                                                pAllocParams->alignment,
501                                                fabricvaspaceGetUCFlaStart(pFabricVAS),
502                                                fabricvaspaceGetUCFlaLimit(pFabricVAS),
503                                                pAllocParams->pageSize, flags,
504                                                ppAddr, pNumAddr);
505     }
506 }
507 
508 NV_STATUS
memoryfabricConstruct_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)509 memoryfabricConstruct_IMPL
510 (
511     MemoryFabric                 *pMemoryFabric,
512     CALL_CONTEXT                 *pCallContext,
513     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
514 )
515 {
516     Memory *pMemory = staticCast(pMemoryFabric, Memory);
517     OBJGPU *pGpu = pMemory->pGpu;
518     FABRIC_VASPACE *pFabricVAS  = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
519     NV00F8_ALLOCATION_PARAMETERS *pAllocParams   = pParams->pAllocParams;
520     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
521     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
522     MEMORY_DESCRIPTOR *pPhysMemDesc = NULL;
523     NV_STATUS status = NV_OK;
524     MEMORY_DESCRIPTOR *pMemDesc = NULL;
525     FABRIC_MEMDESC_DATA *pMemdescData = NULL;
526     MEM_DESC_DESTROY_CALLBACK *pCallback = NULL;
527     VAS_ALLOC_FLAGS flags = {0};
528     NvU64   *pAddr = NULL;
529     NvU32    numAddr = 0;
530     NvU32    pteKind = 0;
531     NvBool   bReadOnly = NV_FALSE;
532     NvHandle hPhysMem;
533     NvBool   bFlexible = NV_FALSE;
534     NvU32    mapFlags = 0;
535 
536     if (RS_IS_COPY_CTOR(pParams))
537     {
538         return memoryfabricCopyConstruct_IMPL(pMemoryFabric,
539                                               pCallContext,
540                                               pParams);
541     }
542 
543     hPhysMem  = pAllocParams->map.hVidMem;
544 
545     // Check if fabric vaspace is valid.
546     if (pFabricVAS == NULL)
547     {
548         NV_PRINTF(LEVEL_ERROR, "Fabric vaspace object not available\n");
549 
550         return NV_ERR_NOT_SUPPORTED;
551     }
552 
553     // initialize Fabric VAS Unicast range if not already setup
554     if (fabricvaspaceGetUCFlaLimit(pFabricVAS) == 0)
555     {
556         NV_PRINTF(LEVEL_ERROR,
557                   "UC FLA ranges should be initialized by this time!\n");
558         return NV_ERR_INVALID_STATE;
559     }
560 
561     if (
562         (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_256G) &&
563         (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_512M) &&
564         (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_2M))
565     {
566         NV_PRINTF(LEVEL_ERROR, "Unsupported pageSize: 0x%llx\n",
567                   pAllocParams->pageSize);
568 
569         return NV_ERR_INVALID_ARGUMENT;
570     }
571 
572     // Alignment should be pageSize aligned.
573     if (!NV_IS_ALIGNED64(pAllocParams->alignment, pAllocParams->pageSize))
574     {
575         NV_PRINTF(LEVEL_ERROR,
576                   "Alignment should be pageSize aligned\n");
577 
578         return NV_ERR_INVALID_ARGUMENT;
579     }
580 
581     // AllocSize should be page size aligned.
582     if (!NV_IS_ALIGNED64(pAllocParams->allocSize, pAllocParams->pageSize))
583     {
584         NV_PRINTF(LEVEL_ERROR,
585                   "AllocSize should be pageSize aligned\n");
586 
587         return NV_ERR_INVALID_ARGUMENT;
588     }
589 
590     bFlexible = !!(pAllocParams->allocFlags & NV00F8_ALLOC_FLAGS_FLEXIBLE_FLA);
591 
592     if (pAllocParams->allocFlags & NV00F8_ALLOC_FLAGS_READ_ONLY)
593     {
594 #if !defined(DEVELOP) && !defined(DEBUG) && !RMCFG_FEATURE_MODS_FEATURES
595         NV_PRINTF(LEVEL_ERROR,
596                   "RO mappings are only supported on non-release builds\n");
597 
598         return NV_ERR_NOT_SUPPORTED;
599 #else
600         bReadOnly = NV_TRUE;
601 #endif
602     }
603 
604     if (bFlexible && (hPhysMem != 0))
605     {
606         NV_PRINTF(LEVEL_ERROR,
607                   "Physmem can't be provided during flexible object alloc\n");
608 
609         return NV_ERR_NOT_SUPPORTED;
610     }
611     else if (!bFlexible)
612     {
613         status = _memoryfabricValidatePhysMem(pCallContext->pClient,
614                                               hPhysMem, pGpu, &pPhysMemDesc);
615         if (status != NV_OK)
616             return status;
617     }
618 
619     // Set the vaspace alloc flags.
620     flags.bSkipTlbInvalidateOnFree = NV_TRUE;
621 
622     flags.bForceContig = !!(pAllocParams->allocFlags &
623                             NV00F8_ALLOC_FLAGS_FORCE_CONTIGUOUS);
624 
625     flags.bForceNonContig = !!(pAllocParams->allocFlags &
626                                NV00F8_ALLOC_FLAGS_FORCE_NONCONTIGUOUS);
627 
628     status = _memoryfabricAllocFabricVa(pFabricVAS, pGpu,
629                                         pParams, pAllocParams,
630                                         flags, &pAddr, &numAddr);
631     if (status != NV_OK)
632     {
633         NV_PRINTF(LEVEL_ERROR,
634                   "VA Space alloc failed! Status Code: 0x%x Size: 0x%llx "
635                   "RangeLo: 0x%llx, RangeHi: 0x%llx, page size: 0x%llx\n",
636                   status, pAllocParams->allocSize,
637                   fabricvaspaceGetUCFlaStart(pFabricVAS),
638                   fabricvaspaceGetUCFlaLimit(pFabricVAS),
639                   pAllocParams->pageSize);
640 
641         return status;
642     }
643 
644     // Create a memdesc to associate with the above allocation.
645     status = memdescCreate(&pMemDesc, pGpu, pAllocParams->allocSize,
646                            0, (numAddr == 1), ADDR_FABRIC_V2, NV_MEMORY_UNCACHED,
647                            MEMDESC_FLAGS_NONE);
648     if (status != NV_OK)
649     {
650         NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory descriptor\n");
651         goto freeVaspace;
652     }
653 
654     // Associate the memdesc with the above FLA allocation.
655     if (numAddr == 1)
656     {
657         // For contiguous allocation, call memdescSetPte to set _pteArray[0].
658         memdescSetPte(pMemDesc, VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS),
659                       0, pAddr[0]);
660     }
661     else
662     {
663         // For discontiguous allocations, call memdescFillPages to fill ptes.
664         memdescFillPages(pMemDesc, 0, pAddr, numAddr, pAllocParams->pageSize);
665     }
666 
667     // Set the memdesc _pageSize.
668     memdescSetPageSize(pMemDesc, VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS),
669                        pAllocParams->pageSize);
670 
671     status = memConstructCommon(pMemory, NV_MEMORY_FABRIC, 0, pMemDesc, 0, NULL,
672                                 0, 0, 0, 0, NVOS32_MEM_TAG_NONE, NULL);
673     if (status != NV_OK)
674     {
675         NV_PRINTF(LEVEL_ERROR, "MemoryFabric memConstructCommon failed\n");
676         goto freeMemdesc;
677     }
678 
679     status = memmgrGetFlaKind_HAL(pGpu, pMemoryManager, &pteKind);
680     if (status != NV_OK)
681     {
682         NV_PRINTF(LEVEL_ERROR, "Error getting kind attr for fabric memory\n");
683         goto freeMemCommon;
684     }
685 
686     // Set PTE kind attribute for fabric memory.
687     memdescSetPteKind(pMemory->pMemDesc, pteKind);
688     memdescSetGpuCacheAttrib(pMemory->pMemDesc, NV_MEMORY_UNCACHED);
689 
690     // Allocate memory for memory fabric memdesc private data.
691     pMemdescData = portMemAllocNonPaged(sizeof(FABRIC_MEMDESC_DATA));
692     if (pMemdescData == NULL)
693     {
694         status = NV_ERR_NO_MEMORY;
695         goto freeMemCommon;
696     }
697     portMemSet(pMemdescData, 0, sizeof(FABRIC_MEMDESC_DATA));
698 
699     // Associate the memdesc data release callback function.
700     memdescSetMemData(pMemDesc, (void *)pMemdescData, NULL);
701 
702     // Allocate memory for the memory descriptor destroy callback.
703     pCallback = portMemAllocNonPaged(sizeof(MEM_DESC_DESTROY_CALLBACK));
704     if (pCallback == NULL)
705     {
706         status = NV_ERR_NO_MEMORY;
707         goto freeMemdescMemData;
708     }
709     portMemSet(pCallback, 0, sizeof(MEM_DESC_DESTROY_CALLBACK));
710 
711     // Associate the memdescDestroy callback function.
712     pCallback->pObject         = (void *)pCallback;
713     pCallback->destroyCallback =
714                 (MemDescDestroyCallBack*) &_memoryfabricMemDescDestroyCallback;
715 
716     memdescAddDestroyCallback(pMemDesc, pCallback);
717 
718     //
719     // In case of flexible mappings, we don't support:
720     //
721     // 1. Caching attributes of physical memory
722     // 2. FLA to GPA tracking to allow FLA object to be mapped as local memory
723     //
724     if (hPhysMem != 0)
725     {
726         // Dup the physical memory handle and cache it in memfabric memdesc.
727         status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient,
728                                    pFabricVAS->hDevice,
729                                    &pMemdescData->hDupedPhysMem,
730                                    pCallContext->pClient->hClient,
731                                    hPhysMem, 0);
732 
733         if (status != NV_OK)
734         {
735             NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
736             goto freeCallback;
737         }
738 
739         NV0041_CTRL_SURFACE_INFO surfaceInfo[2] = {0};
740         NV0041_CTRL_GET_SURFACE_INFO_PARAMS surfaceInfoParam = {0};
741 
742         surfaceInfo[0].index = NV0041_CTRL_SURFACE_INFO_INDEX_ADDR_SPACE_TYPE;
743         surfaceInfo[1].index = NV0041_CTRL_SURFACE_INFO_INDEX_ATTRS;
744         surfaceInfoParam.surfaceInfoListSize = 2;
745         surfaceInfoParam.surfaceInfoList = NvP64_VALUE(&surfaceInfo);
746 
747         status = pRmApi->Control(pRmApi,
748                                  pFabricVAS->hClient,
749                                  pMemdescData->hDupedPhysMem,
750                                  NV0041_CTRL_CMD_GET_SURFACE_INFO,
751                                  &surfaceInfoParam,
752                                  sizeof(surfaceInfoParam));
753         if (status != NV_OK)
754         {
755             NV_PRINTF(LEVEL_ERROR, "Failed to query physmem info\n");
756             goto freeDupedMem;
757         }
758 
759         pMemdescData->physAttrs.addressSpace = surfaceInfo[0].data;
760         //
761         // TODO: Bug 4322867: use NV0041_CTRL_SURFACE_INFO_ATTRS_COMPR
762         // instead of NV0041_CTRL_SURFACE_INFO_INDEX_COMPR_COVERAGE.
763         // NV0041_CTRL_SURFACE_INFO_INDEX_COMPR_COVERAGE is buggy and
764         // will be removed soon.
765         //
766         pMemdescData->physAttrs.compressionCoverage =
767             (surfaceInfo[1].data & NV0041_CTRL_SURFACE_INFO_ATTRS_COMPR) ?  0x1 : 0x0;
768 
769         mapFlags |= bReadOnly ? FABRIC_VASPACE_MAP_FLAGS_READ_ONLY : 0;
770 
771         //
772         // Sticky FLA object should be mapped completely, so pass
773         // pAllocParams->allocSize.as mapLength.
774         //
775         status = fabricvaspaceMapPhysMemdesc(pFabricVAS,
776                                              pMemDesc, 0,
777                                              pPhysMemDesc,
778                                              pAllocParams->map.offset,
779                                              pAllocParams->allocSize,
780                                              mapFlags);
781         if (status != NV_OK)
782         {
783             NV_PRINTF(LEVEL_ERROR,
784                     "Failed to map FLA at the given physmem offset\n");
785             goto freeDupedMem;
786         }
787 
788         //
789         // No need to unmap on failure. Unmap happens implicitly when fabric VA
790         // would be freed.
791         //
792         status = fabricvaspaceVaToGpaMapInsert(pFabricVAS, pAddr[0],
793                                                pPhysMemDesc,
794                                                pAllocParams->map.offset);
795         if (status != NV_OK)
796             goto freeDupedMem;
797     }
798 
799     pMemdescData->allocFlags = pAllocParams->allocFlags;
800     pMemory->bRpcAlloc = pFabricVAS->bRpcAlloc;
801 
802     portMemFree(pAddr);
803 
804     return NV_OK;
805 
806 freeDupedMem:
807     // Free the duped physmem handle.
808     NV_ASSERT(pRmApi->Free(pRmApi, pFabricVAS->hClient,
809                            pMemdescData->hDupedPhysMem) == NV_OK);
810 
811 freeCallback:
812     // Destroy the memdesc destroy callback.
813     memdescRemoveDestroyCallback(pMemDesc, pCallback);
814     portMemFree(pCallback);
815     pCallback = NULL;
816 
817 freeMemdescMemData:
818     // Free the memory fabric memdesc private data.
819     portMemFree(pMemdescData);
820     pMemdescData = NULL;
821 
822 freeMemCommon:
823     memDestructCommon(pMemory);
824 
825 freeMemdesc:
826     // Destroy the created memory descriptor.
827     memdescDestroy(pMemDesc);
828     pMemDesc = NULL;
829 
830 freeVaspace:
831     _memoryfabricFreeFabricVa(pFabricVAS, pGpu,
832                               pParams, pAddr, numAddr);
833 
834     // Free memory allocated for vaspace allocations.
835     portMemFree(pAddr);
836 
837     return status;
838 }
839 
840 void
memoryfabricDestruct_IMPL(MemoryFabric * pMemoryFabric)841 memoryfabricDestruct_IMPL
842 (
843     MemoryFabric *pMemoryFabric
844 )
845 {
846     return;
847 }
848 
849 NvBool
memoryfabricCanCopy_IMPL(MemoryFabric * pMemoryFabric)850 memoryfabricCanCopy_IMPL
851 (
852     MemoryFabric *pMemoryFabric
853 )
854 {
855     return NV_TRUE;
856 }
857 
858 NV_STATUS
memoryfabricCopyConstruct_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)859 memoryfabricCopyConstruct_IMPL
860 (
861     MemoryFabric                 *pMemoryFabric,
862     CALL_CONTEXT                 *pCallContext,
863     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
864 )
865 {
866     //
867     // Memory fabric object must enforce the source (owner) GPU for duping.
868     // However, CUDA and UVM drivers have been using destination (mapping)
869     // GPU to dup memory objects in general. The changes involved in the
870     // UVM driver would need more time as they are a bit involved. Thus,
871     // for now RM is temporarily relaxing this restriction.
872     //
873     // The duping restriction will be added back once UVM bug 3367020
874     // is fixed.
875     //
876 
877     return NV_OK;
878 }
879 
880 NV_STATUS
memoryfabricControl_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_CONTROL_PARAMS_INTERNAL * pParams)881 memoryfabricControl_IMPL
882 (
883     MemoryFabric                   *pMemoryFabric,
884     CALL_CONTEXT                   *pCallContext,
885     RS_RES_CONTROL_PARAMS_INTERNAL *pParams
886 )
887 {
888     NV_STATUS status;
889 
890     if (REF_VAL(NVXXXX_CTRL_CMD_CLASS, pParams->cmd) != NV_MEMORY_FABRIC)
891         return NV_ERR_INVALID_ARGUMENT;
892 
893     status = resControl_IMPL(staticCast(pMemoryFabric, RsResource),
894                              pCallContext, pParams);
895 
896     return status;
897 }
898 
899 NV_STATUS
memoryfabricCtrlGetInfo_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_INFO_PARAMS * pParams)900 memoryfabricCtrlGetInfo_IMPL
901 (
902     MemoryFabric                *pMemoryFabric,
903     NV00F8_CTRL_GET_INFO_PARAMS *pParams
904 )
905 {
906     Memory *pMemory = staticCast(pMemoryFabric, Memory);
907     FABRIC_MEMDESC_DATA *pMemdescData;
908 
909     if (pMemory->pMemDesc == NULL)
910         return NV_ERR_INVALID_ARGUMENT;
911 
912     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemory->pMemDesc);
913 
914     pParams->size = memdescGetSize(pMemory->pMemDesc);
915     pParams->pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
916     pParams->allocFlags = pMemdescData->allocFlags;
917     pParams->physAttrs = pMemdescData->physAttrs;
918 
919     return NV_OK;
920 }
921 
922 NV_STATUS
memoryfabricCtrlCmdDescribe_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_DESCRIBE_PARAMS * pParams)923 memoryfabricCtrlCmdDescribe_IMPL
924 (
925     MemoryFabric                  *pMemoryFabric,
926     NV00F8_CTRL_DESCRIBE_PARAMS   *pParams
927 )
928 {
929     NV_STATUS status;
930     Memory *pMemory = staticCast(pMemoryFabric, Memory);
931     NvU64  *pFabricArray;
932     NvU64   offset;
933     NvU64   pageSize;
934     NvU32   i;
935     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
936     NvHandle hClient = pCallContext->pClient->hClient;
937     FABRIC_MEMDESC_DATA *pMemdescData;
938     OBJGPU *pGpu;
939 
940     if (
941         !rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) &&
942         !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel))
943     {
944         return NV_ERR_INSUFFICIENT_PERMISSIONS;
945     }
946 
947     if (pMemory->pMemDesc == NULL)
948         return NV_ERR_INVALID_ARGUMENT;
949 
950     pGpu = pMemory->pMemDesc->pGpu;
951     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemory->pMemDesc);
952 
953     pParams->memFlags = pMemdescData->allocFlags;
954     pParams->physAttrs = pMemdescData->physAttrs;
955 
956     pParams->attrs.pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
957     pParams->attrs.kind = memdescGetPteKind(pMemory->pMemDesc);
958     pParams->attrs.size = memdescGetSize(pMemory->pMemDesc);
959 
960     if (gpuFabricProbeIsSupported(pGpu))
961     {
962         status = gpuFabricProbeGetFabricCliqueId(pGpu->pGpuFabricProbeInfoKernel,
963                                                  &pParams->attrs.cliqueId);
964         if (status != NV_OK)
965         {
966             NV_PRINTF(LEVEL_ERROR, "unable to query cliqueId 0x%x\n", status);
967             return status;
968         }
969     }
970     else
971     {
972         pParams->attrs.cliqueId = 0;
973     }
974 
975     pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
976 
977     if (memdescGetContiguity(pMemory->pMemDesc, AT_GPU))
978         pParams->totalPfns = 1;
979     else
980         pParams->totalPfns = memdescGetSize(pMemory->pMemDesc) / pageSize;
981 
982     if (pParams->offset >= pParams->totalPfns)
983     {
984         NV_PRINTF(LEVEL_ERROR, "offset: 0x%llx is out of range: 0x%llx\n",
985                   pParams->offset, pParams->totalPfns);
986         return NV_ERR_OUT_OF_RANGE;
987     }
988 
989     pParams->numPfns = NV_MIN(pParams->totalPfns - pParams->offset,
990                               NV00F8_CTRL_DESCRIBE_PFN_ARRAY_SIZE);
991 
992     pFabricArray = portMemAllocNonPaged(sizeof(NvU64) * pParams->numPfns);
993 
994     if (pFabricArray == NULL)
995         return NV_ERR_NO_MEMORY;
996 
997     offset = pParams->offset * pageSize;
998     memdescGetPhysAddrsForGpu(pMemory->pMemDesc, pGpu, AT_GPU, offset,
999                               pageSize, pParams->numPfns, pFabricArray);
1000 
1001     for (i = 0; i < pParams->numPfns; i++)
1002     {
1003         pParams->pfnArray[i] = (NvU32)(pFabricArray[i] >> RM_PAGE_SHIFT_HUGE);
1004     }
1005 
1006     portMemFree(pFabricArray);
1007 
1008     return NV_OK;
1009 }
1010 
1011 NV_STATUS
memoryfabricCtrlAttachMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_ATTACH_MEM_PARAMS * pParams)1012 memoryfabricCtrlAttachMem_IMPL
1013 (
1014     MemoryFabric                  *pMemoryFabric,
1015     NV00F8_CTRL_ATTACH_MEM_PARAMS *pParams
1016 )
1017 {
1018     NvU32 i;
1019     NV_STATUS status;
1020 
1021     pParams->numAttached = 0;
1022 
1023     if (pParams->flags != 0)
1024         return NV_ERR_INVALID_ARGUMENT;
1025 
1026     if ((pParams->numMemInfos == 0) ||
1027         (pParams->numMemInfos > NV00F8_MAX_ATTACHABLE_MEM_INFOS))
1028         return NV_ERR_INVALID_ARGUMENT;
1029 
1030     for (i = 0; i < pParams->numMemInfos; i++)
1031     {
1032         status = _memoryFabricAttachMem(pMemoryFabric, &pParams->memInfos[i]);
1033         if (status != NV_OK)
1034             return status;
1035 
1036         pParams->numAttached++;
1037     }
1038 
1039     return NV_OK;
1040 }
1041 
1042 NV_STATUS
memoryfabricCtrlDetachMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_DETACH_MEM_PARAMS * pParams)1043 memoryfabricCtrlDetachMem_IMPL
1044 (
1045     MemoryFabric                  *pMemoryFabric,
1046     NV00F8_CTRL_DETACH_MEM_PARAMS *pParams
1047 )
1048 {
1049     NvU32 i;
1050     NV_STATUS status;
1051     Memory *pMemory = staticCast(pMemoryFabric, Memory);
1052 
1053     pParams->numDetached = 0;
1054 
1055     if (pParams->flags != 0)
1056         return NV_ERR_INVALID_ARGUMENT;
1057 
1058     if ((pParams->numOffsets == 0) ||
1059          pParams->numOffsets > NV00F8_MAX_DETACHABLE_OFFSETS)
1060         return NV_ERR_INVALID_ARGUMENT;
1061 
1062     for (i = 0; i < pParams->numOffsets; i++)
1063     {
1064         status = _memoryFabricDetachMem(pMemory->pMemDesc, pParams->offsets[i]);
1065         if (status != NV_OK)
1066             return status;
1067 
1068         pParams->numDetached++;
1069     }
1070 
1071     return NV_OK;
1072 }
1073 
1074 NV_STATUS
memoryfabricCtrlGetNumAttachedMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_NUM_ATTACHED_MEM_PARAMS * pParams)1075 memoryfabricCtrlGetNumAttachedMem_IMPL
1076 (
1077     MemoryFabric                            *pMemoryFabric,
1078     NV00F8_CTRL_GET_NUM_ATTACHED_MEM_PARAMS *pParams
1079 )
1080 {
1081     Memory *pMemory = staticCast(pMemoryFabric, Memory);
1082     MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1083     FABRIC_MEMDESC_DATA *pMemdescData = \
1084                     (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1085     NODE *pNode = NULL;
1086 
1087     pParams->numMemInfos = 0;
1088 
1089     btreeEnumStart(pParams->offsetStart, &pNode,
1090                    pMemdescData->pAttachMemInfoTree);
1091 
1092     while ((pNode != NULL) && (pNode->keyStart <= pParams->offsetEnd))
1093     {
1094         pParams->numMemInfos++;
1095         btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
1096     }
1097 
1098     return NV_OK;
1099 }
1100 
1101 NV_STATUS
memoryfabricCtrlGetAttachedMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_ATTACHED_MEM_PARAMS * pParams)1102 memoryfabricCtrlGetAttachedMem_IMPL
1103 (
1104     MemoryFabric                        *pMemoryFabric,
1105     NV00F8_CTRL_GET_ATTACHED_MEM_PARAMS *pParams
1106 )
1107 {
1108     NV_STATUS status;
1109     Memory *pMemory = staticCast(pMemoryFabric, Memory);
1110     MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1111     FABRIC_MEMDESC_DATA *pMemdescData = \
1112                     (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1113     OBJGPU *pGpu = pMemory->pGpu;
1114     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
1115     FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1116     FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
1117     NODE *pNode = NULL;
1118     NvU16 i, count = 0;
1119 
1120     if ((pParams->numMemInfos == 0) ||
1121         (pParams->numMemInfos > NV00F8_MAX_ATTACHED_MEM_INFOS))
1122         return NV_ERR_INVALID_ARGUMENT;
1123 
1124     btreeEnumStart(pParams->offsetStart, &pNode,
1125                    pMemdescData->pAttachMemInfoTree);
1126 
1127     while (count < pParams->numMemInfos)
1128     {
1129         if (pNode == NULL)
1130         {
1131             status = NV_ERR_INVALID_ARGUMENT;
1132             goto cleanup;
1133         }
1134 
1135         pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE *)pNode->Data;
1136 
1137         status = pRmApi->DupObject(pRmApi, RES_GET_CLIENT_HANDLE(pMemory),
1138                                    RES_GET_HANDLE(pMemory->pDevice),
1139                                    &pParams->memInfos[count].hMemory,
1140                                    pFabricVAS->hClient,
1141                                    pAttachMemInfoNode->hDupedPhysMem, 0);
1142         if (status != NV_OK)
1143         {
1144             NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
1145             goto cleanup;
1146         }
1147 
1148         pParams->memInfos[count].offset = pNode->keyStart;
1149         pParams->memInfos[count].mapOffset = pAttachMemInfoNode->physMapOffset;
1150         pParams->memInfos[count].mapLength = pAttachMemInfoNode->physMapLength;
1151 
1152         btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
1153         count++;
1154     }
1155 
1156     return NV_OK;
1157 
1158 cleanup:
1159     for (i = 0; i < count; i++)
1160     {
1161         NV_ASSERT_OK(pRmApi->Free(pRmApi, RES_GET_CLIENT_HANDLE(pMemory),
1162                                   pParams->memInfos[i].hMemory));
1163         pParams->memInfos[i].offset = 0;
1164         pParams->memInfos[i].mapOffset = 0;
1165         pParams->memInfos[i].mapLength = 0;
1166     }
1167 
1168     return status;
1169 }
1170 
1171 static NV_STATUS
_memoryfabricGetPhysAttrsUsingFabricMemdesc(OBJGPU * pGpu,FABRIC_VASPACE * pFabricVAS,MEMORY_DESCRIPTOR * pFabricMemDesc,NvU64 offset,NvU64 * pPhysPageSize)1172 _memoryfabricGetPhysAttrsUsingFabricMemdesc
1173 (
1174     OBJGPU            *pGpu,
1175     FABRIC_VASPACE    *pFabricVAS,
1176     MEMORY_DESCRIPTOR *pFabricMemDesc,
1177     NvU64              offset,
1178     NvU64             *pPhysPageSize
1179 )
1180 {
1181     NV_STATUS status;
1182     MEMORY_DESCRIPTOR *pPhysMemDesc;
1183     FABRIC_MEMDESC_DATA *pMemdescData;
1184     NODE *pNode = NULL;
1185     FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
1186 
1187     if ((pFabricMemDesc == NULL) || (pPhysPageSize == NULL))
1188     {
1189         return NV_ERR_INVALID_ARGUMENT;
1190     }
1191 
1192     status = fabricvaspaceGetGpaMemdesc(pFabricVAS, pFabricMemDesc, pGpu,
1193                                         &pPhysMemDesc);
1194     if (status == NV_OK)
1195     {
1196         *pPhysPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU);
1197         memdescDestroy(pPhysMemDesc);
1198         return NV_OK;
1199     }
1200 
1201     if (status != NV_ERR_OBJECT_NOT_FOUND)
1202         return status;
1203 
1204     // If the object is flexible, check the attach mem info tree.
1205     pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1206     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, btreeSearch(offset, &pNode,
1207                           pMemdescData->pAttachMemInfoTree));
1208     pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE*)pNode->Data;
1209 
1210     *pPhysPageSize = memdescGetPageSize(pAttachMemInfoNode->pPhysMemDesc,
1211                                           AT_GPU);
1212 
1213     return NV_OK;
1214 }
1215 
1216 NV_STATUS
memoryfabricCtrlGetPageLevelInfo_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_PAGE_LEVEL_INFO_PARAMS * pParams)1217 memoryfabricCtrlGetPageLevelInfo_IMPL
1218 (
1219     MemoryFabric                           *pMemoryFabric,
1220     NV00F8_CTRL_GET_PAGE_LEVEL_INFO_PARAMS *pParams
1221 )
1222 {
1223     Memory *pMemory = staticCast(pMemoryFabric, Memory);
1224     OBJGPU *pGpu  = pMemory->pGpu;
1225     FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1226     NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS pageLevelInfoParams = {0};
1227     MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1228     NvU64 fabricAddr;
1229     NvU64 mappingPageSize;
1230     NvU64 physPageSize;
1231     NvU32 i;
1232 
1233     NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_ARGUMENT);
1234 
1235     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1236                           _memoryfabricGetPhysAttrsUsingFabricMemdesc(pGpu,
1237                           pFabricVAS, pFabricMemDesc, pParams->offset,
1238                           &physPageSize));
1239 
1240     mappingPageSize = NV_MIN(physPageSize,
1241                              memdescGetPageSize(pFabricMemDesc, AT_GPU));
1242 
1243     NV_CHECK_OR_RETURN(LEVEL_ERROR,
1244                        NV_IS_ALIGNED64(pParams->offset, mappingPageSize),
1245                        NV_ERR_INVALID_ARGUMENT);
1246 
1247     NV_CHECK_OR_RETURN(LEVEL_ERROR,
1248                        pParams->offset < memdescGetSize(pFabricMemDesc),
1249                        NV_ERR_INVALID_ARGUMENT);
1250 
1251     memdescGetPhysAddrsForGpu(pFabricMemDesc, pGpu, AT_GPU, pParams->offset,
1252                               mappingPageSize, 1, &fabricAddr);
1253 
1254     pageLevelInfoParams.virtAddress = fabricAddr;
1255     pageLevelInfoParams.pageSize = mappingPageSize;
1256 
1257     NV_ASSERT_OK_OR_RETURN(fabricvaspaceGetPageLevelInfo(pFabricVAS, pGpu,
1258                            &pageLevelInfoParams));
1259 
1260     pParams->numLevels = pageLevelInfoParams.numLevels;
1261     for (i = 0; i < pParams->numLevels; i++)
1262     {
1263         portMemCopy(&pParams->levels[i], sizeof(pParams->levels[i]),
1264                     &pageLevelInfoParams.levels[i],
1265                     sizeof(pageLevelInfoParams.levels[i]));
1266     }
1267 
1268     return NV_OK;
1269 }
1270