1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /******************************************************************************
25 *
26 * Description:
27 * This file contains the functions managing the memory fabric
28 *
29 *****************************************************************************/
30
31 #include "core/core.h"
32 #include "core/locks.h"
33 #include "rmapi/resource.h"
34 #include "rmapi/rs_utils.h"
35 #include "mem_mgr_internal.h"
36 #include "mem_mgr/mem_fabric.h"
37 #include "mem_mgr/fabric_vaspace.h"
38 #include "mem_mgr/mem.h"
39 #include "mem_mgr/vaspace.h"
40 #include "gpu/mem_mgr/mem_mgr.h"
41 #include "gpu/mem_mgr/mem_utils.h"
42 #include "gpu/device/device.h"
43 #include "os/os.h"
44 #include "compute/fabric.h"
45 #include "gpu/mem_mgr/mem_desc.h"
46 #include "gpu/gpu.h"
47 #include "class/cl00f8.h"
48 #include "Nvcm.h"
49 #include "vgpu/rpc.h"
50 #include "gpu/bus/kern_bus.h"
51 #include "gpu/bus/p2p_api.h"
52 #include "kernel/gpu/nvlink/kernel_nvlink.h"
53 #include "kernel/gpu/gpu_fabric_probe.h"
54 #include "ctrl/ctrl0041.h"
55
56 static NvU32
_memoryfabricMemDescGetNumAddr(MEMORY_DESCRIPTOR * pMemDesc)57 _memoryfabricMemDescGetNumAddr
58 (
59 MEMORY_DESCRIPTOR *pMemDesc
60 )
61 {
62 OBJGPU *pGpu = pMemDesc->pGpu;
63 NvU64 pageSize = 0;
64
65 // Get the page size from the memory descriptor.
66 pageSize = memdescGetPageSize(pMemDesc,
67 VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
68
69 // Get the number of addresses associated with this memory descriptor.
70 if (memdescGetContiguity(pMemDesc,
71 VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS)))
72 {
73 // For contiguous allocation, there is just one entry _pteArray[0].
74 return 1;
75 }
76
77 // For discontiguous allocations, numAddr is total size / page size.
78 return (memdescGetSize(pMemDesc) / pageSize);
79 }
80
81 static NV_STATUS
_memoryfabricValidatePhysMem(RsClient * pRsClient,NvHandle hPhysMem,OBJGPU * pOwnerGpu,MEMORY_DESCRIPTOR ** ppPhysMemDesc)82 _memoryfabricValidatePhysMem
83 (
84 RsClient *pRsClient,
85 NvHandle hPhysMem,
86 OBJGPU *pOwnerGpu,
87 MEMORY_DESCRIPTOR **ppPhysMemDesc
88 )
89 {
90 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pOwnerGpu);
91 MEMORY_DESCRIPTOR *pPhysMemDesc;
92 NvU64 physPageSize;
93 NV_STATUS status;
94 Memory *pMemory;
95
96 status = memGetByHandle(pRsClient, hPhysMem, &pMemory);
97 if (status != NV_OK)
98 {
99 NV_PRINTF(LEVEL_ERROR, "Invalid object handle passed\n");
100 return status;
101 }
102
103 pPhysMemDesc = pMemory->pMemDesc;
104
105 if ((pOwnerGpu != pPhysMemDesc->pGpu) ||
106 !memmgrIsMemDescSupportedByFla_HAL(pOwnerGpu,
107 pMemoryManager,
108 pPhysMemDesc))
109 {
110 NV_PRINTF(LEVEL_ERROR, "Invalid physmem handle passed\n");
111
112 return NV_ERR_INVALID_ARGUMENT;
113 }
114
115 physPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU);
116 if (
117 (physPageSize != RM_PAGE_SIZE_HUGE) &&
118 (physPageSize != RM_PAGE_SIZE_512M))
119 {
120 NV_PRINTF(LEVEL_ERROR, "Physmem page size should be 2MB\n");
121
122 return NV_ERR_INVALID_ARGUMENT;
123 }
124
125 *ppPhysMemDesc = pPhysMemDesc;
126
127 return NV_OK;
128 }
129
130 static NV_STATUS
_memoryFabricDetachMem(MEMORY_DESCRIPTOR * pFabricMemDesc,NvU64 offset)131 _memoryFabricDetachMem
132 (
133 MEMORY_DESCRIPTOR *pFabricMemDesc,
134 NvU64 offset
135 )
136 {
137 NV_STATUS status;
138 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
139 FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
140 NODE *pNode = NULL;
141 FABRIC_VASPACE *pFabricVAS;
142 MEMORY_DESCRIPTOR *pPhysMemDesc;
143 FABRIC_MEMDESC_DATA *pMemdescData;
144
145 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
146
147 status = btreeSearch(offset, &pNode, pMemdescData->pAttachMemInfoTree);
148 if (status != NV_OK)
149 return status;
150
151 pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE *)pNode->Data;
152 pPhysMemDesc = pAttachMemInfoNode->pPhysMemDesc;
153 pFabricVAS = dynamicCast(pPhysMemDesc->pGpu->pFabricVAS, FABRIC_VASPACE);
154
155 fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, offset,
156 pPhysMemDesc,
157 pAttachMemInfoNode->physMapLength);
158
159 NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient,
160 pAttachMemInfoNode->hDupedPhysMem));
161
162 btreeUnlink(&pAttachMemInfoNode->node, &pMemdescData->pAttachMemInfoTree);
163
164 portMemFree(pAttachMemInfoNode);
165
166 return NV_OK;
167 }
168
169 static void
_memoryFabricBatchDetachMem(MEMORY_DESCRIPTOR * pFabricMemDesc)170 _memoryFabricBatchDetachMem
171 (
172 MEMORY_DESCRIPTOR *pFabricMemDesc
173 )
174 {
175 FABRIC_MEMDESC_DATA *pMemdescData;
176 NODE *pNode = NULL;
177 NvU64 offset;
178 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
179
180 if (pMemdescData == NULL)
181 return;
182
183 btreeEnumStart(0, &pNode, pMemdescData->pAttachMemInfoTree);
184 while (pNode != NULL)
185 {
186 offset = pNode->keyStart;
187 btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
188 NV_ASSERT_OK(_memoryFabricDetachMem(pFabricMemDesc, offset));
189 }
190 }
191
192 static NV_STATUS
_memoryFabricAttachMem(MemoryFabric * pMemoryFabric,NV00F8_CTRL_ATTACH_MEM_INFO * pAttachInfo)193 _memoryFabricAttachMem
194 (
195 MemoryFabric *pMemoryFabric,
196 NV00F8_CTRL_ATTACH_MEM_INFO *pAttachInfo
197 )
198 {
199 NV_STATUS status;
200 Memory *pMemory = staticCast(pMemoryFabric, Memory);
201 OBJGPU *pGpu = pMemory->pGpu;
202 MEMORY_DESCRIPTOR *pPhysMemDesc;
203 MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
204 FABRIC_MEMDESC_DATA *pMemdescData;
205 NvHandle hDupedPhysMem = 0;
206 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
207 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
208 FABRIC_ATTCH_MEM_INFO_NODE *pNode;
209
210 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
211
212 if (!(pMemdescData->allocFlags & NV00F8_ALLOC_FLAGS_FLEXIBLE_FLA))
213 {
214 NV_PRINTF(LEVEL_ERROR, "Unsupported fabric memory type\n");
215 return NV_ERR_NOT_SUPPORTED;
216 }
217
218 if (gpuIsCCFeatureEnabled(pGpu) && !gpuIsCCMultiGpuProtectedPcieModeEnabled(pGpu))
219 {
220 NV_PRINTF(LEVEL_ERROR,
221 "Unsupported when Confidential Computing is enabled in SPT\n");
222 return NV_ERR_NOT_SUPPORTED;
223 }
224
225 status = _memoryfabricValidatePhysMem(RES_GET_CLIENT(pMemory),
226 pAttachInfo->hMemory,
227 pGpu, &pPhysMemDesc);
228
229 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status);
230
231 status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient, pFabricVAS->hDevice,
232 &hDupedPhysMem, RES_GET_CLIENT_HANDLE(pMemory),
233 pAttachInfo->hMemory, 0);
234 if (status != NV_OK)
235 {
236 NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
237 return status;
238 }
239
240 status = fabricvaspaceMapPhysMemdesc(pFabricVAS,
241 pFabricMemDesc,
242 pAttachInfo->offset,
243 pPhysMemDesc,
244 pAttachInfo->mapOffset,
245 pAttachInfo->mapLength,
246 0);
247 if (status != NV_OK)
248 {
249 NV_PRINTF(LEVEL_ERROR, "Failed to map FLA\n");
250 goto freeDupedMem;
251 }
252
253 pNode = portMemAllocNonPaged(sizeof(*pNode));
254 if (pNode == NULL)
255 {
256 status = NV_ERR_NO_MEMORY;
257 goto unmapVas;
258 }
259
260 portMemSet(pNode, 0, sizeof(*pNode));
261
262 pNode->node.keyStart = pAttachInfo->offset;
263 pNode->node.keyEnd = pAttachInfo->offset;
264 pNode->physMapOffset = pAttachInfo->mapOffset;
265 pNode->physMapLength = pAttachInfo->mapLength;
266 pNode->pPhysMemDesc = pPhysMemDesc;
267 pNode->hDupedPhysMem = hDupedPhysMem;
268 pNode->node.Data = pNode;
269
270 status = btreeInsert(&pNode->node, &pMemdescData->pAttachMemInfoTree);
271 if (status != NV_OK)
272 {
273 NV_PRINTF(LEVEL_ERROR, "Failed to track attach mem info\n");
274 goto freeNode;
275 }
276
277 return NV_OK;
278
279 freeNode:
280 portMemFree(pNode);
281
282 unmapVas:
283 fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc,
284 pAttachInfo->offset,
285 pPhysMemDesc, pAttachInfo->mapLength);
286
287 freeDupedMem:
288 NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient, hDupedPhysMem));
289
290 return status;
291 }
292
293 static void
_memoryfabricMemDescDestroyCallback(OBJGPU * pGpu,void * pObject,MEMORY_DESCRIPTOR * pMemDesc)294 _memoryfabricMemDescDestroyCallback
295 (
296 OBJGPU *pGpu,
297 void *pObject,
298 MEMORY_DESCRIPTOR *pMemDesc
299 )
300 {
301 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
302 FABRIC_VASPACE *pFabricVAS;
303 RmPhysAddr *pteArray;
304 FABRIC_MEMDESC_DATA *pMemdescData;
305 NvU32 numAddr;
306 NvU64 pageSize;
307
308 NV_ASSERT_OR_RETURN_VOID(pGpu->pFabricVAS != NULL);
309
310 pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
311 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemDesc);
312 pteArray = memdescGetPteArrayForGpu(pMemDesc, pGpu,
313 VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
314 numAddr = _memoryfabricMemDescGetNumAddr(pMemDesc);
315 // Get the page size from the memory descriptor.
316 pageSize = memdescGetPageSize(pMemDesc,
317 VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS));
318
319 // Remove the fabric memory allocations from the map.
320 fabricvaspaceVaToGpaMapRemove(pFabricVAS, pteArray[0]);
321
322 // Detach any pending memory...
323 _memoryFabricBatchDetachMem(pMemDesc);
324
325 if (!pFabricVAS->bRpcAlloc)
326 {
327 //
328 // Call fabricvaspaceBatchFree to free the FLA allocations.
329 // _pteArray in memdesc is RM_PAGE_SIZE whereas page size for memory
330 // fabric allocations is either 2MB or 512MB. Pass stride accordingly.
331 //
332 fabricvaspaceBatchFree(pFabricVAS, pteArray, numAddr,
333 (pageSize >> RM_PAGE_SHIFT));
334 }
335
336 if (pMemdescData != NULL)
337 {
338 if (pMemdescData->hDupedPhysMem != 0)
339 {
340 NV_ASSERT(pRmApi->Free(pRmApi, pFabricVAS->hClient,
341 pMemdescData->hDupedPhysMem) == NV_OK);
342 }
343
344 portMemFree(pMemDesc->_pMemData);
345 memdescSetMemData(pMemDesc, NULL, NULL);
346 }
347
348 portMemFree(pObject);
349 }
350
351 static void
_memoryfabricFreeFabricVa_VGPU(OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)352 _memoryfabricFreeFabricVa_VGPU
353 (
354 OBJGPU *pGpu,
355 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
356 )
357 {
358 NV_STATUS status = NV_OK;
359 NV_RM_RPC_FREE(pGpu, pParams->hClient,
360 pParams->hParent, pParams->hResource, status);
361 NV_ASSERT(status == NV_OK);
362 }
363
364 static void
_memoryfabricFreeFabricVa(FABRIC_VASPACE * pFabricVAS,OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NvU64 * pAddr,NvU32 numAddr)365 _memoryfabricFreeFabricVa
366 (
367 FABRIC_VASPACE *pFabricVAS,
368 OBJGPU *pGpu,
369 RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
370 NvU64 *pAddr,
371 NvU32 numAddr
372 )
373 {
374 if (pFabricVAS->bRpcAlloc)
375 {
376 _memoryfabricFreeFabricVa_VGPU(pGpu, pParams);
377 }
378 else
379 {
380 fabricvaspaceBatchFree(pFabricVAS, pAddr, numAddr, 1);
381 }
382 }
383
384 static NV_STATUS
_memoryfabricAllocFabricVa_VGPU(OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NV00F8_ALLOCATION_PARAMETERS * pAllocParams,NvU64 ** ppAddr,NvU32 * pNumAddr)385 _memoryfabricAllocFabricVa_VGPU
386 (
387 OBJGPU *pGpu,
388 RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
389 NV00F8_ALLOCATION_PARAMETERS *pAllocParams,
390 NvU64 **ppAddr,
391 NvU32 *pNumAddr
392 )
393 {
394 NV00F8_CTRL_DESCRIBE_PARAMS *pDescribeParams = NULL;
395 NvU32 i = 0;
396 NV_STATUS status = NV_OK;
397 NvU32 idx = 0;
398 NvU64 *pAddr = NULL;
399
400 NV_RM_RPC_ALLOC_OBJECT(pGpu, pParams->hClient,
401 pParams->hParent,
402 pParams->hResource,
403 pParams->externalClassId,
404 pAllocParams,
405 sizeof(*pAllocParams),
406 status);
407
408 if (status != NV_OK)
409 {
410 NV_PRINTF(LEVEL_ERROR,
411 "Alloc NV_MEMORY_FABRIC RPC failed, status: %x\n",
412 status);
413 return status;
414 }
415
416 pDescribeParams = portMemAllocNonPaged(sizeof(*pDescribeParams));
417 if (pDescribeParams == NULL)
418 {
419 status = NV_ERR_NO_MEMORY;
420 goto cleanup;
421 }
422
423 portMemSet(pDescribeParams, 0, sizeof(*pDescribeParams));
424
425 do
426 {
427 pDescribeParams->offset = idx;
428 NV_RM_RPC_CONTROL(pGpu, pParams->hClient,
429 pParams->hResource,
430 NV00F8_CTRL_CMD_DESCRIBE,
431 pDescribeParams,
432 sizeof(*pDescribeParams), status);
433 if (status != NV_OK)
434 {
435 NV_PRINTF(LEVEL_ERROR, "CTRL_CMD_DESCRIBE failed, status: 0x%x, "
436 "numPfns: 0x%x, totalPfns: 0x%llx, readSoFar: 0x%x \n",
437 status, pDescribeParams->numPfns,
438 pDescribeParams->totalPfns, idx);
439 goto cleanup;
440 }
441
442 if (pAddr == NULL)
443 {
444 pAddr = portMemAllocNonPaged(sizeof(NvU64) *
445 pDescribeParams->totalPfns);
446 if (pAddr == NULL)
447 {
448 status = NV_ERR_NO_MEMORY;
449 goto cleanup;
450 }
451 }
452
453 for (i=0; i < pDescribeParams->numPfns; i++)
454 {
455 pAddr[idx + i] =
456 (NvU64)((NvU64)pDescribeParams->pfnArray[i] << RM_PAGE_SHIFT_HUGE);
457 }
458
459 idx += pDescribeParams->numPfns;
460 } while (idx < pDescribeParams->totalPfns);
461
462 portMemFree(pDescribeParams);
463
464 *ppAddr = pAddr;
465 *pNumAddr = idx;
466
467 return status;
468
469 cleanup:
470 portMemFree(pAddr);
471 portMemFree(pDescribeParams);
472
473 _memoryfabricFreeFabricVa_VGPU(pGpu, pParams);
474
475 return status;
476 }
477
478 static NV_STATUS
_memoryfabricAllocFabricVa(FABRIC_VASPACE * pFabricVAS,OBJGPU * pGpu,RS_RES_ALLOC_PARAMS_INTERNAL * pParams,NV00F8_ALLOCATION_PARAMETERS * pAllocParams,VAS_ALLOC_FLAGS flags,NvU64 ** ppAddr,NvU32 * pNumAddr)479 _memoryfabricAllocFabricVa
480 (
481 FABRIC_VASPACE *pFabricVAS,
482 OBJGPU *pGpu,
483 RS_RES_ALLOC_PARAMS_INTERNAL *pParams,
484 NV00F8_ALLOCATION_PARAMETERS *pAllocParams,
485 VAS_ALLOC_FLAGS flags,
486 NvU64 **ppAddr,
487 NvU32 *pNumAddr
488 )
489 {
490 if (pFabricVAS->bRpcAlloc)
491 {
492 return _memoryfabricAllocFabricVa_VGPU(pGpu, pParams,
493 pAllocParams, ppAddr,
494 pNumAddr);
495 }
496 else
497 {
498 return fabricvaspaceAllocNonContiguous(pFabricVAS,
499 pAllocParams->allocSize,
500 pAllocParams->alignment,
501 fabricvaspaceGetUCFlaStart(pFabricVAS),
502 fabricvaspaceGetUCFlaLimit(pFabricVAS),
503 pAllocParams->pageSize, flags,
504 ppAddr, pNumAddr);
505 }
506 }
507
508 NV_STATUS
memoryfabricConstruct_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)509 memoryfabricConstruct_IMPL
510 (
511 MemoryFabric *pMemoryFabric,
512 CALL_CONTEXT *pCallContext,
513 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
514 )
515 {
516 Memory *pMemory = staticCast(pMemoryFabric, Memory);
517 OBJGPU *pGpu = pMemory->pGpu;
518 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
519 NV00F8_ALLOCATION_PARAMETERS *pAllocParams = pParams->pAllocParams;
520 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
521 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
522 MEMORY_DESCRIPTOR *pPhysMemDesc = NULL;
523 NV_STATUS status = NV_OK;
524 MEMORY_DESCRIPTOR *pMemDesc = NULL;
525 FABRIC_MEMDESC_DATA *pMemdescData = NULL;
526 MEM_DESC_DESTROY_CALLBACK *pCallback = NULL;
527 VAS_ALLOC_FLAGS flags = {0};
528 NvU64 *pAddr = NULL;
529 NvU32 numAddr = 0;
530 NvU32 pteKind = 0;
531 NvBool bReadOnly = NV_FALSE;
532 NvHandle hPhysMem;
533 NvBool bFlexible = NV_FALSE;
534 NvU32 mapFlags = 0;
535
536 if (RS_IS_COPY_CTOR(pParams))
537 {
538 return memoryfabricCopyConstruct_IMPL(pMemoryFabric,
539 pCallContext,
540 pParams);
541 }
542
543 hPhysMem = pAllocParams->map.hVidMem;
544
545 // Check if fabric vaspace is valid.
546 if (pFabricVAS == NULL)
547 {
548 NV_PRINTF(LEVEL_ERROR, "Fabric vaspace object not available\n");
549
550 return NV_ERR_NOT_SUPPORTED;
551 }
552
553 // initialize Fabric VAS Unicast range if not already setup
554 if (fabricvaspaceGetUCFlaLimit(pFabricVAS) == 0)
555 {
556 NV_PRINTF(LEVEL_ERROR,
557 "UC FLA ranges should be initialized by this time!\n");
558 return NV_ERR_INVALID_STATE;
559 }
560
561 if (
562 (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_256G) &&
563 (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_512M) &&
564 (pAllocParams->pageSize != NV_MEMORY_FABRIC_PAGE_SIZE_2M))
565 {
566 NV_PRINTF(LEVEL_ERROR, "Unsupported pageSize: 0x%llx\n",
567 pAllocParams->pageSize);
568
569 return NV_ERR_INVALID_ARGUMENT;
570 }
571
572 // Alignment should be pageSize aligned.
573 if (!NV_IS_ALIGNED64(pAllocParams->alignment, pAllocParams->pageSize))
574 {
575 NV_PRINTF(LEVEL_ERROR,
576 "Alignment should be pageSize aligned\n");
577
578 return NV_ERR_INVALID_ARGUMENT;
579 }
580
581 // AllocSize should be page size aligned.
582 if (!NV_IS_ALIGNED64(pAllocParams->allocSize, pAllocParams->pageSize))
583 {
584 NV_PRINTF(LEVEL_ERROR,
585 "AllocSize should be pageSize aligned\n");
586
587 return NV_ERR_INVALID_ARGUMENT;
588 }
589
590 bFlexible = !!(pAllocParams->allocFlags & NV00F8_ALLOC_FLAGS_FLEXIBLE_FLA);
591
592 if (pAllocParams->allocFlags & NV00F8_ALLOC_FLAGS_READ_ONLY)
593 {
594 #if !defined(DEVELOP) && !defined(DEBUG) && !RMCFG_FEATURE_MODS_FEATURES
595 NV_PRINTF(LEVEL_ERROR,
596 "RO mappings are only supported on non-release builds\n");
597
598 return NV_ERR_NOT_SUPPORTED;
599 #else
600 bReadOnly = NV_TRUE;
601 #endif
602 }
603
604 if (bFlexible && (hPhysMem != 0))
605 {
606 NV_PRINTF(LEVEL_ERROR,
607 "Physmem can't be provided during flexible object alloc\n");
608
609 return NV_ERR_NOT_SUPPORTED;
610 }
611 else if (!bFlexible)
612 {
613 status = _memoryfabricValidatePhysMem(pCallContext->pClient,
614 hPhysMem, pGpu, &pPhysMemDesc);
615 if (status != NV_OK)
616 return status;
617 }
618
619 // Set the vaspace alloc flags.
620 flags.bSkipTlbInvalidateOnFree = NV_TRUE;
621
622 flags.bForceContig = !!(pAllocParams->allocFlags &
623 NV00F8_ALLOC_FLAGS_FORCE_CONTIGUOUS);
624
625 flags.bForceNonContig = !!(pAllocParams->allocFlags &
626 NV00F8_ALLOC_FLAGS_FORCE_NONCONTIGUOUS);
627
628 status = _memoryfabricAllocFabricVa(pFabricVAS, pGpu,
629 pParams, pAllocParams,
630 flags, &pAddr, &numAddr);
631 if (status != NV_OK)
632 {
633 NV_PRINTF(LEVEL_ERROR,
634 "VA Space alloc failed! Status Code: 0x%x Size: 0x%llx "
635 "RangeLo: 0x%llx, RangeHi: 0x%llx, page size: 0x%llx\n",
636 status, pAllocParams->allocSize,
637 fabricvaspaceGetUCFlaStart(pFabricVAS),
638 fabricvaspaceGetUCFlaLimit(pFabricVAS),
639 pAllocParams->pageSize);
640
641 return status;
642 }
643
644 // Create a memdesc to associate with the above allocation.
645 status = memdescCreate(&pMemDesc, pGpu, pAllocParams->allocSize,
646 0, (numAddr == 1), ADDR_FABRIC_V2, NV_MEMORY_UNCACHED,
647 MEMDESC_FLAGS_NONE);
648 if (status != NV_OK)
649 {
650 NV_PRINTF(LEVEL_ERROR, "Failed to allocate memory descriptor\n");
651 goto freeVaspace;
652 }
653
654 // Associate the memdesc with the above FLA allocation.
655 if (numAddr == 1)
656 {
657 // For contiguous allocation, call memdescSetPte to set _pteArray[0].
658 memdescSetPte(pMemDesc, VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS),
659 0, pAddr[0]);
660 }
661 else
662 {
663 // For discontiguous allocations, call memdescFillPages to fill ptes.
664 memdescFillPages(pMemDesc, 0, pAddr, numAddr, pAllocParams->pageSize);
665 }
666
667 // Set the memdesc _pageSize.
668 memdescSetPageSize(pMemDesc, VAS_ADDRESS_TRANSLATION(pGpu->pFabricVAS),
669 pAllocParams->pageSize);
670
671 status = memConstructCommon(pMemory, NV_MEMORY_FABRIC, 0, pMemDesc, 0, NULL,
672 0, 0, 0, 0, NVOS32_MEM_TAG_NONE, NULL);
673 if (status != NV_OK)
674 {
675 NV_PRINTF(LEVEL_ERROR, "MemoryFabric memConstructCommon failed\n");
676 goto freeMemdesc;
677 }
678
679 status = memmgrGetFlaKind_HAL(pGpu, pMemoryManager, &pteKind);
680 if (status != NV_OK)
681 {
682 NV_PRINTF(LEVEL_ERROR, "Error getting kind attr for fabric memory\n");
683 goto freeMemCommon;
684 }
685
686 // Set PTE kind attribute for fabric memory.
687 memdescSetPteKind(pMemory->pMemDesc, pteKind);
688 memdescSetGpuCacheAttrib(pMemory->pMemDesc, NV_MEMORY_UNCACHED);
689
690 // Allocate memory for memory fabric memdesc private data.
691 pMemdescData = portMemAllocNonPaged(sizeof(FABRIC_MEMDESC_DATA));
692 if (pMemdescData == NULL)
693 {
694 status = NV_ERR_NO_MEMORY;
695 goto freeMemCommon;
696 }
697 portMemSet(pMemdescData, 0, sizeof(FABRIC_MEMDESC_DATA));
698
699 // Associate the memdesc data release callback function.
700 memdescSetMemData(pMemDesc, (void *)pMemdescData, NULL);
701
702 // Allocate memory for the memory descriptor destroy callback.
703 pCallback = portMemAllocNonPaged(sizeof(MEM_DESC_DESTROY_CALLBACK));
704 if (pCallback == NULL)
705 {
706 status = NV_ERR_NO_MEMORY;
707 goto freeMemdescMemData;
708 }
709 portMemSet(pCallback, 0, sizeof(MEM_DESC_DESTROY_CALLBACK));
710
711 // Associate the memdescDestroy callback function.
712 pCallback->pObject = (void *)pCallback;
713 pCallback->destroyCallback =
714 (MemDescDestroyCallBack*) &_memoryfabricMemDescDestroyCallback;
715
716 memdescAddDestroyCallback(pMemDesc, pCallback);
717
718 //
719 // In case of flexible mappings, we don't support:
720 //
721 // 1. Caching attributes of physical memory
722 // 2. FLA to GPA tracking to allow FLA object to be mapped as local memory
723 //
724 if (hPhysMem != 0)
725 {
726 // Dup the physical memory handle and cache it in memfabric memdesc.
727 status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient,
728 pFabricVAS->hDevice,
729 &pMemdescData->hDupedPhysMem,
730 pCallContext->pClient->hClient,
731 hPhysMem, 0);
732
733 if (status != NV_OK)
734 {
735 NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
736 goto freeCallback;
737 }
738
739 NV0041_CTRL_SURFACE_INFO surfaceInfo[2] = {0};
740 NV0041_CTRL_GET_SURFACE_INFO_PARAMS surfaceInfoParam = {0};
741
742 surfaceInfo[0].index = NV0041_CTRL_SURFACE_INFO_INDEX_ADDR_SPACE_TYPE;
743 surfaceInfo[1].index = NV0041_CTRL_SURFACE_INFO_INDEX_ATTRS;
744 surfaceInfoParam.surfaceInfoListSize = 2;
745 surfaceInfoParam.surfaceInfoList = NvP64_VALUE(&surfaceInfo);
746
747 status = pRmApi->Control(pRmApi,
748 pFabricVAS->hClient,
749 pMemdescData->hDupedPhysMem,
750 NV0041_CTRL_CMD_GET_SURFACE_INFO,
751 &surfaceInfoParam,
752 sizeof(surfaceInfoParam));
753 if (status != NV_OK)
754 {
755 NV_PRINTF(LEVEL_ERROR, "Failed to query physmem info\n");
756 goto freeDupedMem;
757 }
758
759 pMemdescData->physAttrs.addressSpace = surfaceInfo[0].data;
760 //
761 // TODO: Bug 4322867: use NV0041_CTRL_SURFACE_INFO_ATTRS_COMPR
762 // instead of NV0041_CTRL_SURFACE_INFO_INDEX_COMPR_COVERAGE.
763 // NV0041_CTRL_SURFACE_INFO_INDEX_COMPR_COVERAGE is buggy and
764 // will be removed soon.
765 //
766 pMemdescData->physAttrs.compressionCoverage =
767 (surfaceInfo[1].data & NV0041_CTRL_SURFACE_INFO_ATTRS_COMPR) ? 0x1 : 0x0;
768
769 mapFlags |= bReadOnly ? FABRIC_VASPACE_MAP_FLAGS_READ_ONLY : 0;
770
771 //
772 // Sticky FLA object should be mapped completely, so pass
773 // pAllocParams->allocSize.as mapLength.
774 //
775 status = fabricvaspaceMapPhysMemdesc(pFabricVAS,
776 pMemDesc, 0,
777 pPhysMemDesc,
778 pAllocParams->map.offset,
779 pAllocParams->allocSize,
780 mapFlags);
781 if (status != NV_OK)
782 {
783 NV_PRINTF(LEVEL_ERROR,
784 "Failed to map FLA at the given physmem offset\n");
785 goto freeDupedMem;
786 }
787
788 //
789 // No need to unmap on failure. Unmap happens implicitly when fabric VA
790 // would be freed.
791 //
792 status = fabricvaspaceVaToGpaMapInsert(pFabricVAS, pAddr[0],
793 pPhysMemDesc,
794 pAllocParams->map.offset);
795 if (status != NV_OK)
796 goto freeDupedMem;
797 }
798
799 pMemdescData->allocFlags = pAllocParams->allocFlags;
800 pMemory->bRpcAlloc = pFabricVAS->bRpcAlloc;
801
802 portMemFree(pAddr);
803
804 return NV_OK;
805
806 freeDupedMem:
807 // Free the duped physmem handle.
808 NV_ASSERT(pRmApi->Free(pRmApi, pFabricVAS->hClient,
809 pMemdescData->hDupedPhysMem) == NV_OK);
810
811 freeCallback:
812 // Destroy the memdesc destroy callback.
813 memdescRemoveDestroyCallback(pMemDesc, pCallback);
814 portMemFree(pCallback);
815 pCallback = NULL;
816
817 freeMemdescMemData:
818 // Free the memory fabric memdesc private data.
819 portMemFree(pMemdescData);
820 pMemdescData = NULL;
821
822 freeMemCommon:
823 memDestructCommon(pMemory);
824
825 freeMemdesc:
826 // Destroy the created memory descriptor.
827 memdescDestroy(pMemDesc);
828 pMemDesc = NULL;
829
830 freeVaspace:
831 _memoryfabricFreeFabricVa(pFabricVAS, pGpu,
832 pParams, pAddr, numAddr);
833
834 // Free memory allocated for vaspace allocations.
835 portMemFree(pAddr);
836
837 return status;
838 }
839
840 void
memoryfabricDestruct_IMPL(MemoryFabric * pMemoryFabric)841 memoryfabricDestruct_IMPL
842 (
843 MemoryFabric *pMemoryFabric
844 )
845 {
846 return;
847 }
848
849 NvBool
memoryfabricCanCopy_IMPL(MemoryFabric * pMemoryFabric)850 memoryfabricCanCopy_IMPL
851 (
852 MemoryFabric *pMemoryFabric
853 )
854 {
855 return NV_TRUE;
856 }
857
858 NV_STATUS
memoryfabricCopyConstruct_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)859 memoryfabricCopyConstruct_IMPL
860 (
861 MemoryFabric *pMemoryFabric,
862 CALL_CONTEXT *pCallContext,
863 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
864 )
865 {
866 //
867 // Memory fabric object must enforce the source (owner) GPU for duping.
868 // However, CUDA and UVM drivers have been using destination (mapping)
869 // GPU to dup memory objects in general. The changes involved in the
870 // UVM driver would need more time as they are a bit involved. Thus,
871 // for now RM is temporarily relaxing this restriction.
872 //
873 // The duping restriction will be added back once UVM bug 3367020
874 // is fixed.
875 //
876
877 return NV_OK;
878 }
879
880 NV_STATUS
memoryfabricControl_IMPL(MemoryFabric * pMemoryFabric,CALL_CONTEXT * pCallContext,RS_RES_CONTROL_PARAMS_INTERNAL * pParams)881 memoryfabricControl_IMPL
882 (
883 MemoryFabric *pMemoryFabric,
884 CALL_CONTEXT *pCallContext,
885 RS_RES_CONTROL_PARAMS_INTERNAL *pParams
886 )
887 {
888 NV_STATUS status;
889
890 if (REF_VAL(NVXXXX_CTRL_CMD_CLASS, pParams->cmd) != NV_MEMORY_FABRIC)
891 return NV_ERR_INVALID_ARGUMENT;
892
893 status = resControl_IMPL(staticCast(pMemoryFabric, RsResource),
894 pCallContext, pParams);
895
896 return status;
897 }
898
899 NV_STATUS
memoryfabricCtrlGetInfo_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_INFO_PARAMS * pParams)900 memoryfabricCtrlGetInfo_IMPL
901 (
902 MemoryFabric *pMemoryFabric,
903 NV00F8_CTRL_GET_INFO_PARAMS *pParams
904 )
905 {
906 Memory *pMemory = staticCast(pMemoryFabric, Memory);
907 FABRIC_MEMDESC_DATA *pMemdescData;
908
909 if (pMemory->pMemDesc == NULL)
910 return NV_ERR_INVALID_ARGUMENT;
911
912 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemory->pMemDesc);
913
914 pParams->size = memdescGetSize(pMemory->pMemDesc);
915 pParams->pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
916 pParams->allocFlags = pMemdescData->allocFlags;
917 pParams->physAttrs = pMemdescData->physAttrs;
918
919 return NV_OK;
920 }
921
922 NV_STATUS
memoryfabricCtrlCmdDescribe_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_DESCRIBE_PARAMS * pParams)923 memoryfabricCtrlCmdDescribe_IMPL
924 (
925 MemoryFabric *pMemoryFabric,
926 NV00F8_CTRL_DESCRIBE_PARAMS *pParams
927 )
928 {
929 NV_STATUS status;
930 Memory *pMemory = staticCast(pMemoryFabric, Memory);
931 NvU64 *pFabricArray;
932 NvU64 offset;
933 NvU64 pageSize;
934 NvU32 i;
935 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
936 NvHandle hClient = pCallContext->pClient->hClient;
937 FABRIC_MEMDESC_DATA *pMemdescData;
938 OBJGPU *pGpu;
939
940 if (
941 !rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) &&
942 !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel))
943 {
944 return NV_ERR_INSUFFICIENT_PERMISSIONS;
945 }
946
947 if (pMemory->pMemDesc == NULL)
948 return NV_ERR_INVALID_ARGUMENT;
949
950 pGpu = pMemory->pMemDesc->pGpu;
951 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pMemory->pMemDesc);
952
953 pParams->memFlags = pMemdescData->allocFlags;
954 pParams->physAttrs = pMemdescData->physAttrs;
955
956 pParams->attrs.pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
957 pParams->attrs.kind = memdescGetPteKind(pMemory->pMemDesc);
958 pParams->attrs.size = memdescGetSize(pMemory->pMemDesc);
959
960 if (gpuFabricProbeIsSupported(pGpu))
961 {
962 status = gpuFabricProbeGetFabricCliqueId(pGpu->pGpuFabricProbeInfoKernel,
963 &pParams->attrs.cliqueId);
964 if (status != NV_OK)
965 {
966 NV_PRINTF(LEVEL_ERROR, "unable to query cliqueId 0x%x\n", status);
967 return status;
968 }
969 }
970 else
971 {
972 pParams->attrs.cliqueId = 0;
973 }
974
975 pageSize = memdescGetPageSize(pMemory->pMemDesc, AT_GPU);
976
977 if (memdescGetContiguity(pMemory->pMemDesc, AT_GPU))
978 pParams->totalPfns = 1;
979 else
980 pParams->totalPfns = memdescGetSize(pMemory->pMemDesc) / pageSize;
981
982 if (pParams->offset >= pParams->totalPfns)
983 {
984 NV_PRINTF(LEVEL_ERROR, "offset: 0x%llx is out of range: 0x%llx\n",
985 pParams->offset, pParams->totalPfns);
986 return NV_ERR_OUT_OF_RANGE;
987 }
988
989 pParams->numPfns = NV_MIN(pParams->totalPfns - pParams->offset,
990 NV00F8_CTRL_DESCRIBE_PFN_ARRAY_SIZE);
991
992 pFabricArray = portMemAllocNonPaged(sizeof(NvU64) * pParams->numPfns);
993
994 if (pFabricArray == NULL)
995 return NV_ERR_NO_MEMORY;
996
997 offset = pParams->offset * pageSize;
998 memdescGetPhysAddrsForGpu(pMemory->pMemDesc, pGpu, AT_GPU, offset,
999 pageSize, pParams->numPfns, pFabricArray);
1000
1001 for (i = 0; i < pParams->numPfns; i++)
1002 {
1003 pParams->pfnArray[i] = (NvU32)(pFabricArray[i] >> RM_PAGE_SHIFT_HUGE);
1004 }
1005
1006 portMemFree(pFabricArray);
1007
1008 return NV_OK;
1009 }
1010
1011 NV_STATUS
memoryfabricCtrlAttachMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_ATTACH_MEM_PARAMS * pParams)1012 memoryfabricCtrlAttachMem_IMPL
1013 (
1014 MemoryFabric *pMemoryFabric,
1015 NV00F8_CTRL_ATTACH_MEM_PARAMS *pParams
1016 )
1017 {
1018 NvU32 i;
1019 NV_STATUS status;
1020
1021 pParams->numAttached = 0;
1022
1023 if (pParams->flags != 0)
1024 return NV_ERR_INVALID_ARGUMENT;
1025
1026 if ((pParams->numMemInfos == 0) ||
1027 (pParams->numMemInfos > NV00F8_MAX_ATTACHABLE_MEM_INFOS))
1028 return NV_ERR_INVALID_ARGUMENT;
1029
1030 for (i = 0; i < pParams->numMemInfos; i++)
1031 {
1032 status = _memoryFabricAttachMem(pMemoryFabric, &pParams->memInfos[i]);
1033 if (status != NV_OK)
1034 return status;
1035
1036 pParams->numAttached++;
1037 }
1038
1039 return NV_OK;
1040 }
1041
1042 NV_STATUS
memoryfabricCtrlDetachMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_DETACH_MEM_PARAMS * pParams)1043 memoryfabricCtrlDetachMem_IMPL
1044 (
1045 MemoryFabric *pMemoryFabric,
1046 NV00F8_CTRL_DETACH_MEM_PARAMS *pParams
1047 )
1048 {
1049 NvU32 i;
1050 NV_STATUS status;
1051 Memory *pMemory = staticCast(pMemoryFabric, Memory);
1052
1053 pParams->numDetached = 0;
1054
1055 if (pParams->flags != 0)
1056 return NV_ERR_INVALID_ARGUMENT;
1057
1058 if ((pParams->numOffsets == 0) ||
1059 pParams->numOffsets > NV00F8_MAX_DETACHABLE_OFFSETS)
1060 return NV_ERR_INVALID_ARGUMENT;
1061
1062 for (i = 0; i < pParams->numOffsets; i++)
1063 {
1064 status = _memoryFabricDetachMem(pMemory->pMemDesc, pParams->offsets[i]);
1065 if (status != NV_OK)
1066 return status;
1067
1068 pParams->numDetached++;
1069 }
1070
1071 return NV_OK;
1072 }
1073
1074 NV_STATUS
memoryfabricCtrlGetNumAttachedMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_NUM_ATTACHED_MEM_PARAMS * pParams)1075 memoryfabricCtrlGetNumAttachedMem_IMPL
1076 (
1077 MemoryFabric *pMemoryFabric,
1078 NV00F8_CTRL_GET_NUM_ATTACHED_MEM_PARAMS *pParams
1079 )
1080 {
1081 Memory *pMemory = staticCast(pMemoryFabric, Memory);
1082 MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1083 FABRIC_MEMDESC_DATA *pMemdescData = \
1084 (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1085 NODE *pNode = NULL;
1086
1087 pParams->numMemInfos = 0;
1088
1089 btreeEnumStart(pParams->offsetStart, &pNode,
1090 pMemdescData->pAttachMemInfoTree);
1091
1092 while ((pNode != NULL) && (pNode->keyStart <= pParams->offsetEnd))
1093 {
1094 pParams->numMemInfos++;
1095 btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
1096 }
1097
1098 return NV_OK;
1099 }
1100
1101 NV_STATUS
memoryfabricCtrlGetAttachedMem_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_ATTACHED_MEM_PARAMS * pParams)1102 memoryfabricCtrlGetAttachedMem_IMPL
1103 (
1104 MemoryFabric *pMemoryFabric,
1105 NV00F8_CTRL_GET_ATTACHED_MEM_PARAMS *pParams
1106 )
1107 {
1108 NV_STATUS status;
1109 Memory *pMemory = staticCast(pMemoryFabric, Memory);
1110 MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1111 FABRIC_MEMDESC_DATA *pMemdescData = \
1112 (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1113 OBJGPU *pGpu = pMemory->pGpu;
1114 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
1115 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1116 FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
1117 NODE *pNode = NULL;
1118 NvU16 i, count = 0;
1119
1120 if ((pParams->numMemInfos == 0) ||
1121 (pParams->numMemInfos > NV00F8_MAX_ATTACHED_MEM_INFOS))
1122 return NV_ERR_INVALID_ARGUMENT;
1123
1124 btreeEnumStart(pParams->offsetStart, &pNode,
1125 pMemdescData->pAttachMemInfoTree);
1126
1127 while (count < pParams->numMemInfos)
1128 {
1129 if (pNode == NULL)
1130 {
1131 status = NV_ERR_INVALID_ARGUMENT;
1132 goto cleanup;
1133 }
1134
1135 pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE *)pNode->Data;
1136
1137 status = pRmApi->DupObject(pRmApi, RES_GET_CLIENT_HANDLE(pMemory),
1138 RES_GET_HANDLE(pMemory->pDevice),
1139 &pParams->memInfos[count].hMemory,
1140 pFabricVAS->hClient,
1141 pAttachMemInfoNode->hDupedPhysMem, 0);
1142 if (status != NV_OK)
1143 {
1144 NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
1145 goto cleanup;
1146 }
1147
1148 pParams->memInfos[count].offset = pNode->keyStart;
1149 pParams->memInfos[count].mapOffset = pAttachMemInfoNode->physMapOffset;
1150 pParams->memInfos[count].mapLength = pAttachMemInfoNode->physMapLength;
1151
1152 btreeEnumNext(&pNode, pMemdescData->pAttachMemInfoTree);
1153 count++;
1154 }
1155
1156 return NV_OK;
1157
1158 cleanup:
1159 for (i = 0; i < count; i++)
1160 {
1161 NV_ASSERT_OK(pRmApi->Free(pRmApi, RES_GET_CLIENT_HANDLE(pMemory),
1162 pParams->memInfos[i].hMemory));
1163 pParams->memInfos[i].offset = 0;
1164 pParams->memInfos[i].mapOffset = 0;
1165 pParams->memInfos[i].mapLength = 0;
1166 }
1167
1168 return status;
1169 }
1170
1171 static NV_STATUS
_memoryfabricGetPhysAttrsUsingFabricMemdesc(OBJGPU * pGpu,FABRIC_VASPACE * pFabricVAS,MEMORY_DESCRIPTOR * pFabricMemDesc,NvU64 offset,NvU64 * pPhysPageSize)1172 _memoryfabricGetPhysAttrsUsingFabricMemdesc
1173 (
1174 OBJGPU *pGpu,
1175 FABRIC_VASPACE *pFabricVAS,
1176 MEMORY_DESCRIPTOR *pFabricMemDesc,
1177 NvU64 offset,
1178 NvU64 *pPhysPageSize
1179 )
1180 {
1181 NV_STATUS status;
1182 MEMORY_DESCRIPTOR *pPhysMemDesc;
1183 FABRIC_MEMDESC_DATA *pMemdescData;
1184 NODE *pNode = NULL;
1185 FABRIC_ATTCH_MEM_INFO_NODE *pAttachMemInfoNode;
1186
1187 if ((pFabricMemDesc == NULL) || (pPhysPageSize == NULL))
1188 {
1189 return NV_ERR_INVALID_ARGUMENT;
1190 }
1191
1192 status = fabricvaspaceGetGpaMemdesc(pFabricVAS, pFabricMemDesc, pGpu,
1193 &pPhysMemDesc);
1194 if (status == NV_OK)
1195 {
1196 *pPhysPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU);
1197 memdescDestroy(pPhysMemDesc);
1198 return NV_OK;
1199 }
1200
1201 if (status != NV_ERR_OBJECT_NOT_FOUND)
1202 return status;
1203
1204 // If the object is flexible, check the attach mem info tree.
1205 pMemdescData = (FABRIC_MEMDESC_DATA *)memdescGetMemData(pFabricMemDesc);
1206 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, btreeSearch(offset, &pNode,
1207 pMemdescData->pAttachMemInfoTree));
1208 pAttachMemInfoNode = (FABRIC_ATTCH_MEM_INFO_NODE*)pNode->Data;
1209
1210 *pPhysPageSize = memdescGetPageSize(pAttachMemInfoNode->pPhysMemDesc,
1211 AT_GPU);
1212
1213 return NV_OK;
1214 }
1215
1216 NV_STATUS
memoryfabricCtrlGetPageLevelInfo_IMPL(MemoryFabric * pMemoryFabric,NV00F8_CTRL_GET_PAGE_LEVEL_INFO_PARAMS * pParams)1217 memoryfabricCtrlGetPageLevelInfo_IMPL
1218 (
1219 MemoryFabric *pMemoryFabric,
1220 NV00F8_CTRL_GET_PAGE_LEVEL_INFO_PARAMS *pParams
1221 )
1222 {
1223 Memory *pMemory = staticCast(pMemoryFabric, Memory);
1224 OBJGPU *pGpu = pMemory->pGpu;
1225 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1226 NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS pageLevelInfoParams = {0};
1227 MEMORY_DESCRIPTOR *pFabricMemDesc = pMemory->pMemDesc;
1228 NvU64 fabricAddr;
1229 NvU64 mappingPageSize;
1230 NvU64 physPageSize;
1231 NvU32 i;
1232
1233 NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_ARGUMENT);
1234
1235 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1236 _memoryfabricGetPhysAttrsUsingFabricMemdesc(pGpu,
1237 pFabricVAS, pFabricMemDesc, pParams->offset,
1238 &physPageSize));
1239
1240 mappingPageSize = NV_MIN(physPageSize,
1241 memdescGetPageSize(pFabricMemDesc, AT_GPU));
1242
1243 NV_CHECK_OR_RETURN(LEVEL_ERROR,
1244 NV_IS_ALIGNED64(pParams->offset, mappingPageSize),
1245 NV_ERR_INVALID_ARGUMENT);
1246
1247 NV_CHECK_OR_RETURN(LEVEL_ERROR,
1248 pParams->offset < memdescGetSize(pFabricMemDesc),
1249 NV_ERR_INVALID_ARGUMENT);
1250
1251 memdescGetPhysAddrsForGpu(pFabricMemDesc, pGpu, AT_GPU, pParams->offset,
1252 mappingPageSize, 1, &fabricAddr);
1253
1254 pageLevelInfoParams.virtAddress = fabricAddr;
1255 pageLevelInfoParams.pageSize = mappingPageSize;
1256
1257 NV_ASSERT_OK_OR_RETURN(fabricvaspaceGetPageLevelInfo(pFabricVAS, pGpu,
1258 &pageLevelInfoParams));
1259
1260 pParams->numLevels = pageLevelInfoParams.numLevels;
1261 for (i = 0; i < pParams->numLevels; i++)
1262 {
1263 portMemCopy(&pParams->levels[i], sizeof(pParams->levels[i]),
1264 &pageLevelInfoParams.levels[i],
1265 sizeof(pageLevelInfoParams.levels[i]));
1266 }
1267
1268 return NV_OK;
1269 }
1270