1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 /******************************************************************************
25  *
26  *   Description:
27  *       This file contains the functions managing the memory multicast fabric
28  *
29  *****************************************************************************/
30 #define NVOC_MEM_MULTICAST_FABRIC_H_PRIVATE_ACCESS_ALLOWED
31 
32 /*
33  * Lock ordering
34  *
35  * RMAPI Lock
36  * |_Client Lock
37  *   |_GPU(s) Lock
38  *     |_MCFLA Module Lock
39  *       |_MCFLA Descriptor Lock
40  */
41 
42 #include "os/os.h"
43 #include "core/locks.h"
44 #include "nvport/nvport.h"
45 #include "rmapi/rs_utils.h"
46 #include "rmapi/rmapi_utils.h"
47 #include "compute/fabric.h"
48 #include "gpu/gpu.h"
49 #include "gpu/bus/kern_bus.h"
50 #include "gpu/mem_mgr/mem_desc.h"
51 #include "gpu/mem_mgr/mem_mgr.h"
52 #include "gpu/subdevice/subdevice.h"
53 #include "kernel/gpu/nvlink/kernel_nvlink.h"
54 #include "mem_mgr/fabric_vaspace.h"
55 #include "mem_mgr/mem_multicast_fabric.h"
56 #include "published/hopper/gh100/dev_mmu.h"
57 
58 #include "class/cl00f9.h"
59 #include "mem_mgr/mem_export.h"
60 
61 #include "gpu/gpu_fabric_probe.h"
62 
63 typedef struct mem_multicast_fabric_attach_mem_info_node
64 {
65     MEMORY_DESCRIPTOR *pPhysMemDesc;
66     NvHandle           hDupedPhysMem;
67     NvU64              physMapLength;
68     NODE               node;
69 } MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE;
70 
71 typedef struct mem_multicast_fabric_client_info
72 {
73     void   *pOsEvent;
74     Memory *pMemory;
75 } MEM_MULTICAST_FABRIC_CLIENT_INFO;
76 
77 typedef struct mem_multicast_fabric_gpu_info
78 {
79     void   *pGpuOsInfo;
80     OBJGPU *pGpu;
81     NvU64   gpuProbeHandle;
82     NvU32   cliqueId;
83     NvBool  bMcflaAlloc;
84 
85     //
86     // Unique import event ID. Valid only if the GPU was remotely attached to
87     // the prime MCFLA object
88     //
89     NvU64 attachEventId;
90 
91     //
92     // Tracks memory attached using NV00FD_CTRL_CMD_ATTACH_MEM
93     //
94     // GPU lock must be taken to protect this tree.
95     //
96     PNODE pAttachMemInfoTree;
97 } MEM_MULTICAST_FABRIC_GPU_INFO;
98 
99 typedef struct mem_multicast_fabric_remote_gpu_info
100 {
101     NvU32 cliqueId;
102     NvU64 key;
103 } MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO;
104 
105 MAKE_MULTIMAP(MemMulticastFabricRemoteGpuInfoMap,
106               MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO);
107 
108 MAKE_LIST(MemMulticastFabricClientInfoList, MEM_MULTICAST_FABRIC_CLIENT_INFO);
109 
110 MAKE_LIST(MemMulticastFabricGpuInfoList, MEM_MULTICAST_FABRIC_GPU_INFO);
111 
112 typedef enum
113 {
114     MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST = 0,
115     MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST,
116 } MEM_MULTICAST_FABRIC_REQUEST_TYPE;
117 
118 typedef struct mem_multicast_fabric_descriptor
119 {
120     // Refcount to keep this descriptor alive
121     NvU64 refCount;
122 
123     // List of clients waiting on this object to be ready
124     MemMulticastFabricClientInfoList waitingClientsList;
125 
126     // Mask representing the list of attached GPUs
127     NvU32 localAttachedGpusMask;
128 
129     // List of attached GPU info
130     MemMulticastFabricGpuInfoList gpuInfoList;
131 
132     // Boolean to be set when pMemDesc is installed
133     NvBool bMemdescInstalled;
134 
135     // Memory descriptor associated with the multicast object
136     MEMORY_DESCRIPTOR *pMemDesc;
137 
138     // Unique handle assigned for the multicast team by FM
139     NvU64 mcTeamHandle;
140 
141     // Status of the multicast team
142     NV_STATUS mcTeamStatus;
143 
144     //
145     // Boolean to be set when an Inband request has been sent to FM
146     // and is currently in progress
147     //
148     NvBool bInbandReqInProgress;
149 
150     //
151     // Request Id associated with the Inband request in progress when
152     // bInbandReqSent is set to true
153     //
154     NvU64 inbandReqId;
155 
156     // Alignment for the multicast FLA allocation
157     NvU64 alignment;
158 
159     // Multicast FLA allocation size
160     NvU64 allocSize;
161 
162     // Page size for the multicast FLA
163     NvU64 pageSize;
164 
165     // Multicast FLA allocation flags
166     NvU32 allocFlags;
167 
168     // Max. number of unique GPUs associated with the multicast object
169     NvU32 numMaxGpus;
170 
171     // No. of unique GPUs currently attached to the multicast object
172     NvU32 numAttachedGpus;
173 
174     // Export object information associated with this import descriptor.
175     NvU16   exportNodeId;
176     NvU16   index;
177 
178     // Same as packet.uuid, but uses NvUuid type.
179     NvUuid  expUuid;
180 
181     // Import cache key
182     NvU64   cacheKey;
183 
184     // Map of attached remote GPU info
185     MemMulticastFabricRemoteGpuInfoMap remoteGpuInfoMap;
186 
187     //
188     // The lock protects MEM_MULTICAST_FABRIC_DESCRIPTOR, the MCFLA descriptor.
189     //
190     // The lock should be taken only if an MCFLA descriptor is safe
191     // to access i.e., holding the module lock or the accessing thread
192     // has the MCFLA descriptor refcounted.
193     //
194     PORT_RWLOCK *pLock;
195 } MEM_MULTICAST_FABRIC_DESCRIPTOR;
196 
197 static NvBool
198 _memMulticastFabricIsPrime
199 (
200     NvU32 allocFlags
201 )
202 {
203     NvBool bPrime = NV_TRUE;
204 
205 #ifdef NV_MEMORY_MULTICAST_FABRIC_ALLOC_FLAGS_USE_EXPORT_PACKET
206     //
207     // If an MCFLA object is allocated using export packet (UUID), then it
208     // is a non-prime (imported) object. Such objects are just extension
209     // of the prime (exported) MCFLA objects.
210     //
211     bPrime = !(allocFlags &
212                NV_MEMORY_MULTICAST_FABRIC_ALLOC_FLAGS_USE_EXPORT_PACKET);
213 #endif
214 
215     return bPrime;
216 }
217 
218 static void
219 _memMulticastFabricInitAttachEvent
220 (
221     NvU64                        gpuFabricProbeHandle,
222     NvU64                        key,
223     NvU32                        cliqueId,
224     NvU16                        exportNodeId,
225     NvU16                        index,
226     NvUuid                      *pExportUuid,
227     NV00F1_CTRL_FABRIC_EVENT    *pEvent
228 )
229 {
230     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
231 
232     pEvent->imexChannel = 0;
233     pEvent->type = NV00F1_CTRL_FABRIC_EVENT_TYPE_REMOTE_GPU_ATTACH;
234     pEvent->id = fabricGenerateEventId(pFabric);
235 
236     pEvent->data.attach.gpuFabricProbeHandle = gpuFabricProbeHandle;
237     pEvent->data.attach.key = key;
238     pEvent->data.attach.cliqueId = cliqueId;
239     pEvent->data.attach.index = index;
240     pEvent->data.attach.exportNodeId = exportNodeId;
241     portMemCopy(pEvent->data.attach.exportUuid, NV_MEM_EXPORT_UUID_LEN,
242                 pExportUuid->uuid,              NV_MEM_EXPORT_UUID_LEN);
243 }
244 
245 static void
246 _memMulticastFabricInitUnimportEvent
247 (
248     NvU64                        attachEventId,
249     NvU16                        exportNodeId,
250     NV00F1_CTRL_FABRIC_EVENT    *pEvent
251 )
252 {
253     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
254 
255     pEvent->imexChannel = 0;
256     pEvent->type = NV00F1_CTRL_FABRIC_EVENT_TYPE_MEM_UNIMPORT;
257     pEvent->id = fabricGenerateEventId(pFabric);
258     pEvent->data.unimport.exportNodeId  = exportNodeId;
259     pEvent->data.unimport.importEventId = attachEventId;
260 }
261 
262 static
263 NV_STATUS
264 _memMulticastFabricValidateAllocParams
265 (
266     NV00FD_ALLOCATION_PARAMETERS *pAllocParams
267 )
268 {
269     // Nothing to verify in this case..
270     if (!_memMulticastFabricIsPrime(pAllocParams->allocFlags))
271         return NV_OK;
272 
273     // Only page size 512MB is supported
274     if (pAllocParams->pageSize != NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M)
275     {
276         NV_PRINTF(LEVEL_ERROR,
277             "Unsupported pageSize: 0x%x. Only 512MB pagesize is supported\n",
278             pAllocParams->pageSize);
279         return NV_ERR_INVALID_ARGUMENT;
280     }
281 
282     if (pAllocParams->alignment != pAllocParams->pageSize)
283     {
284         NV_PRINTF(LEVEL_ERROR,
285                   "Alignment must be pageSize for now\n");
286         return NV_ERR_INVALID_ARGUMENT;
287     }
288 
289     // AllocSize should be page size aligned
290     if (!NV_IS_ALIGNED64(pAllocParams->allocSize, pAllocParams->pageSize))
291     {
292         NV_PRINTF(LEVEL_ERROR,
293                   "AllocSize should be pageSize aligned\n");
294         return NV_ERR_INVALID_ARGUMENT;
295     }
296 
297     // NV_U32_MAX is reserved as a special value for non-prime MCFLA objects.
298     if ((pAllocParams->numGpus == 0) ||
299         (pAllocParams->numGpus == NV_U32_MAX))
300     {
301         NV_PRINTF(LEVEL_ERROR, "Invalid number of GPUs to attach\n");
302         return NV_ERR_INVALID_ARGUMENT;
303     }
304 
305     return NV_OK;
306 }
307 
308 static void
309 _memMulticastFabricDescriptorCleanup
310 (
311     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
312 )
313 {
314     if (pMulticastFabricDesc == NULL)
315         return;
316 
317     NV_ASSERT(listCount(&pMulticastFabricDesc->gpuInfoList) == 0);
318     listDestroy(&pMulticastFabricDesc->gpuInfoList);
319 
320     multimapDestroy(&pMulticastFabricDesc->remoteGpuInfoMap);
321 
322     NV_ASSERT(pMulticastFabricDesc->numAttachedGpus == 0);
323     NV_ASSERT(pMulticastFabricDesc->localAttachedGpusMask == 0);
324 
325     NV_ASSERT(listCount(&pMulticastFabricDesc->waitingClientsList) == 0);
326     listDestroy(&pMulticastFabricDesc->waitingClientsList);
327 
328     memdescDestroy(pMulticastFabricDesc->pMemDesc);
329 
330     if (pMulticastFabricDesc->pLock != NULL)
331         portSyncRwLockDestroy(pMulticastFabricDesc->pLock);
332 
333     portMemFree(pMulticastFabricDesc);
334 }
335 
336 static
337 MEM_MULTICAST_FABRIC_DESCRIPTOR*
338 _memMulticastFabricDescriptorAlloc
339 (
340     MemoryMulticastFabric        *pMemoryMulticastFabric,
341     NV00FD_ALLOCATION_PARAMETERS *pAllocParams
342 )
343 {
344     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
345     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
346     NV_STATUS status;
347 
348     pMulticastFabricDesc = portMemAllocNonPaged(
349                                 sizeof(MEM_MULTICAST_FABRIC_DESCRIPTOR));
350     if (pMulticastFabricDesc == NULL)
351         return NULL;
352 
353     portMemSet(pMulticastFabricDesc, 0,
354                sizeof(MEM_MULTICAST_FABRIC_DESCRIPTOR));
355 
356     listInit(&pMulticastFabricDesc->waitingClientsList,
357              portMemAllocatorGetGlobalNonPaged());
358 
359     listInit(&pMulticastFabricDesc->gpuInfoList,
360              portMemAllocatorGetGlobalNonPaged());
361 
362     pMulticastFabricDesc->refCount = 1;
363     pMulticastFabricDesc->mcTeamStatus = NV_ERR_NOT_READY;
364     pMulticastFabricDesc->localAttachedGpusMask = 0;
365     pMulticastFabricDesc->alignment  = pAllocParams->alignment;
366     pMulticastFabricDesc->allocSize  = pAllocParams->allocSize;
367     pMulticastFabricDesc->pageSize   = pAllocParams->pageSize;
368     pMulticastFabricDesc->allocFlags = pAllocParams->allocFlags;
369     pMulticastFabricDesc->numMaxGpus = pAllocParams->numGpus;
370     pMulticastFabricDesc->inbandReqId = osGetTimestamp();
371 
372     multimapInit(&pMulticastFabricDesc->remoteGpuInfoMap,
373                  portMemAllocatorGetGlobalNonPaged());
374     pMulticastFabricDesc->exportNodeId = NV_FABRIC_INVALID_NODE_ID;
375 
376     pMulticastFabricDesc->pLock =
377         portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
378     if (pMulticastFabricDesc->pLock == NULL)
379         goto fail;
380 
381     status = fabricMulticastSetupCacheInsert(pFabric,
382                                         pMulticastFabricDesc->inbandReqId,
383                                         pMulticastFabricDesc);
384     if (status != NV_OK)
385     {
386         NV_PRINTF(LEVEL_ERROR, "Failed to track memdesc 0x%x", status);
387         goto fail;
388     }
389 
390     return pMulticastFabricDesc;
391 
392 fail:
393     _memMulticastFabricDescriptorCleanup(pMulticastFabricDesc);
394 
395     return NULL;
396 }
397 
398 static void
399 _memMulticastFabricDescriptorFlushClients
400 (
401     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
402 )
403 {
404     MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode;
405 
406     while ((pNode =
407             listHead(&pMulticastFabricDesc->waitingClientsList)) != NULL)
408     {
409         if (pNode->pOsEvent != NULL)
410         {
411             osSetEvent(NULL, pNode->pOsEvent);
412             NV_ASSERT_OK(osDereferenceObjectCount(pNode->pOsEvent));
413         }
414 
415         listRemove(&pMulticastFabricDesc->waitingClientsList, pNode);
416     }
417 
418     return;
419 }
420 
421 static NV_STATUS
422 _memMulticastFabricDescriptorEnqueueWait
423 (
424     NvHandle                         hClient,
425     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc,
426     NvP64                            pOsEvent,
427     Memory                          *pMemory
428 )
429 {
430     MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode;
431     NvP64                             pValidatedOsEvent = NULL;
432     NV_STATUS                         status;
433 
434     if (pOsEvent != NULL)
435     {
436         status = osUserHandleToKernelPtr(hClient, pOsEvent,
437                                          &pValidatedOsEvent);
438         if (status != NV_OK)
439             return status;
440     }
441 
442     pNode = listAppendNew(&pMulticastFabricDesc->waitingClientsList);
443     if (pNode == NULL)
444     {
445         if (pOsEvent != NULL)
446             osDereferenceObjectCount(pValidatedOsEvent);
447 
448         return NV_ERR_NO_MEMORY;
449     }
450 
451     pNode->pOsEvent = pValidatedOsEvent;
452     pNode->pMemory = pMemory;
453 
454     //
455     // In case the multicast object's memdesc is ready, unblock
456     // clients waiting on it
457     //
458     if (pMulticastFabricDesc->bMemdescInstalled)
459     {
460         _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc);
461     }
462 
463     return NV_OK;
464 }
465 
466 static void
467 _memMulticastFabricDescriptorDequeueWait
468 (
469     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc,
470     Memory                          *pMemory
471 )
472 {
473     MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode;
474     MEM_MULTICAST_FABRIC_CLIENT_INFO *pNodeNext;
475 
476     pNode = listHead(&pMulticastFabricDesc->waitingClientsList);
477 
478     // There can be multiple events per memory object, so delete all.
479     while (pNode != NULL)
480     {
481         pNodeNext = listNext(&pMulticastFabricDesc->waitingClientsList, pNode);
482 
483         if (pNode->pMemory == pMemory)
484         {
485             if (pNode->pOsEvent != NULL)
486                 osDereferenceObjectCount(pNode->pOsEvent);
487 
488             listRemove(&pMulticastFabricDesc->waitingClientsList, pNode);
489         }
490 
491         pNode = pNodeNext;
492     }
493 }
494 
495 static NV_STATUS
496 _memMulticastFabricGpuInfoAdd
497 (
498     MemoryMulticastFabric          *pMemoryMulticastFabric,
499     RS_RES_CONTROL_PARAMS_INTERNAL *pParams
500 )
501 {
502     NV00FD_CTRL_ATTACH_GPU_PARAMS *pAttachParams = pParams->pParams;
503     Subdevice *pSubdevice = NULL;
504     MEM_MULTICAST_FABRIC_GPU_INFO *pNode;
505     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = \
506         pMemoryMulticastFabric->pMulticastFabricDesc;
507     OBJGPU *pGpu;
508     MEM_MULTICAST_FABRIC_GPU_INFO *pNodeItr;
509 
510     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
511         subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric),
512             pAttachParams->hSubdevice, &pSubdevice));
513 
514     pGpu = GPU_RES_GET_GPU(pSubdevice);
515 
516     if(!osMatchGpuOsInfo(pGpu, pParams->secInfo.gpuOsInfo))
517         return NV_ERR_INVALID_DEVICE;
518 
519     for (pNodeItr = listHead(&pMulticastFabricDesc->gpuInfoList);
520          pNodeItr != NULL;
521          pNodeItr = listNext(&pMulticastFabricDesc->gpuInfoList, pNodeItr))
522     {
523         if (pNodeItr->pGpu == pGpu)
524         {
525            NV_PRINTF(LEVEL_ERROR, "GPU %x has already attached\n",
526                                    pGpu->gpuInstance);
527            return NV_ERR_INSERT_DUPLICATE_NAME;
528         }
529     }
530 
531     pNode = listAppendNew(&pMulticastFabricDesc->gpuInfoList);
532     if (pNode == NULL)
533         return NV_ERR_NO_MEMORY;
534 
535     pNode->pGpuOsInfo = pParams->secInfo.gpuOsInfo;
536     pNode->pGpu = GPU_RES_GET_GPU(pSubdevice);
537     pNode->pAttachMemInfoTree = NULL;
538     pNode->bMcflaAlloc = NV_FALSE;
539 
540     return NV_OK;
541 }
542 
543 static void
544 _memMulticastFabricGpuInfoRemove
545 (
546     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
547 )
548 {
549     MEM_MULTICAST_FABRIC_GPU_INFO *pNode = NULL;
550     THREAD_STATE_NODE *pThreadNode = NULL;
551     THREAD_STATE_FREE_CALLBACK freeCallback;
552 
553     NV_ASSERT_OK(threadStateGetCurrent(&pThreadNode, NULL));
554 
555     while ((pNode = listHead(&pMulticastFabricDesc->gpuInfoList)) != NULL)
556     {
557         //
558         // Enqueue unimport event before the callback to release GPU.
559         // This ordering is important.
560         //
561         if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags))
562         {
563             Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
564             NV00F1_CTRL_FABRIC_EVENT unimportEvent;
565 
566             _memMulticastFabricInitUnimportEvent(pNode->attachEventId,
567                                     pMulticastFabricDesc->exportNodeId,
568                                     &unimportEvent);
569 
570             NV_CHECK(LEVEL_WARNING,
571                 fabricPostEventsV2(pFabric, &unimportEvent, 1) == NV_OK);
572         }
573 
574         freeCallback.pCb = osReleaseGpuOsInfo;
575         freeCallback.pCbData = (void *)pNode->pGpuOsInfo;
576 
577         NV_ASSERT_OK(threadStateEnqueueCallbackOnFree(pThreadNode,
578                                                       &freeCallback));
579         listRemove(&pMulticastFabricDesc->gpuInfoList, pNode);
580     }
581 
582     pMulticastFabricDesc->localAttachedGpusMask = 0;
583     pMulticastFabricDesc->numAttachedGpus = 0;
584 }
585 
586 NV_STATUS
587 _memMulticastFabricSendInbandTeamSetupRequestV2
588 (
589     OBJGPU                          *pGpu,
590     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
591 )
592 {
593     NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams;
594     nvlink_inband_mc_team_setup_req_v2_msg_t *pMcTeamSetupReqMsg = NULL;
595     nvlink_inband_mc_team_setup_req_v2_t *pMcTeamSetupReq = NULL;
596     MEM_MULTICAST_FABRIC_GPU_INFO *pNode;
597     NvU32 idx = 0;
598     NvU32 payloadSize;
599     NvU32 sendDataSize;
600     NV_STATUS status = NV_OK;
601     NvU16 numKeys = 1;
602     MemMulticastFabricRemoteGpuInfoMapSupermapIter smIter;
603 
604     sendDataParams =
605         (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *)
606         portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS));
607 
608     if (sendDataParams == NULL)
609         return NV_ERR_NO_MEMORY;
610 
611     pMcTeamSetupReqMsg =
612         (nvlink_inband_mc_team_setup_req_v2_msg_t *)&sendDataParams->buffer[0];
613 
614     pMcTeamSetupReq =
615         (nvlink_inband_mc_team_setup_req_v2_t *)&pMcTeamSetupReqMsg->mcTeamSetupReq;
616 
617     //
618     // Submap of remoteGpuInfoMap represent a node. As there is a key/node,
619     // count submap to calculate numKeys.
620     //
621     numKeys += multimapCountSubmaps(&pMulticastFabricDesc->remoteGpuInfoMap);
622 
623     payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_setup_req_v2_t) +
624                          (sizeof(pMcTeamSetupReq->gpuHandlesAndKeys[0]) *
625                                  (pMulticastFabricDesc->numMaxGpus + numKeys)));
626 
627     sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize);
628 
629     if ((NvU32)sendDataSize > sizeof(sendDataParams->buffer))
630     {
631         NV_ASSERT(0);
632         status = NV_ERR_INSUFFICIENT_RESOURCES;
633         goto done;
634     }
635 
636     portMemSet(sendDataParams, 0, sendDataSize);
637 
638     pMcTeamSetupReq->mcAllocSize = pMulticastFabricDesc->allocSize;
639     pMcTeamSetupReq->numGpuHandles = pMulticastFabricDesc->numMaxGpus;
640     pMcTeamSetupReq->numKeys = numKeys;
641 
642     // Fill local GPUs
643     for (pNode = listHead(&pMulticastFabricDesc->gpuInfoList);
644          pNode != NULL;
645          pNode = listNext(&pMulticastFabricDesc->gpuInfoList, pNode))
646         pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pNode->gpuProbeHandle;
647 
648     // Fill remote GPUs probe handles per submap (i.e per node)
649     smIter = multimapSubmapIterAll(&pMulticastFabricDesc->remoteGpuInfoMap);
650 
651     while (multimapSubmapIterNext(&smIter))
652     {
653         MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = smIter.pValue;
654         MemMulticastFabricRemoteGpuInfoMapIter iter;
655 
656         iter = multimapSubmapIterItems(&pMulticastFabricDesc->remoteGpuInfoMap,
657                                        pSubmap);
658 
659         while (multimapItemIterNext(&iter))
660         {
661             // Item key is the GPU probe handle
662             pMcTeamSetupReq->gpuHandlesAndKeys[idx++] =
663                     multimapItemKey(&pMulticastFabricDesc->remoteGpuInfoMap,
664                                     iter.pValue);
665         }
666     }
667 
668     // Fill local key
669     pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pMulticastFabricDesc->inbandReqId;
670 
671     // Fill remote keys per submap (i.e per node)
672     smIter = multimapSubmapIterAll(&pMulticastFabricDesc->remoteGpuInfoMap);
673 
674     while (multimapSubmapIterNext(&smIter))
675     {
676         MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = smIter.pValue;
677         MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteNode = NULL;
678         MemMulticastFabricRemoteGpuInfoMapIter iter;
679 
680         iter = multimapSubmapIterItems(&pMulticastFabricDesc->remoteGpuInfoMap,
681                                        pSubmap);
682 
683         if (multimapItemIterNext(&iter))
684         {
685             pRemoteNode = iter.pValue;
686             pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pRemoteNode->key;
687         }
688     }
689 
690     if (idx != (pMcTeamSetupReq->numGpuHandles + numKeys))
691     {
692         NV_ASSERT(0);
693         status = NV_ERR_INVALID_STATE;
694         goto done;
695     }
696 
697     sendDataParams->dataSize = sendDataSize;
698 
699     nvlinkInitInbandMsgHdr(&pMcTeamSetupReqMsg->msgHdr,
700                            NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ_V2,
701                            payloadSize, pMulticastFabricDesc->inbandReqId);
702 
703     status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu),
704                                    sendDataParams);
705     if (status != NV_OK)
706         goto done;
707 
708     pMulticastFabricDesc->bInbandReqInProgress = NV_TRUE;
709 
710 done:
711     portMemFree(sendDataParams);
712 
713     return status;
714 }
715 
716 NV_STATUS
717 _memMulticastFabricSendInbandTeamSetupRequestV1
718 (
719     OBJGPU                          *pGpu,
720     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
721 )
722 {
723     NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams;
724     nvlink_inband_mc_team_setup_req_msg_t *pMcTeamSetupReqMsg = NULL;
725     nvlink_inband_mc_team_setup_req_t *pMcTeamSetupReq = NULL;
726     MEM_MULTICAST_FABRIC_GPU_INFO *pNode;
727     NvU32 idx = 0;
728     NvU32 payloadSize;
729     NvU32 sendDataSize;
730     NV_STATUS status = NV_OK;
731 
732     sendDataParams = \
733         (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *)
734         portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS));
735 
736     if (sendDataParams == NULL)
737         return NV_ERR_NO_MEMORY;
738 
739     pMcTeamSetupReqMsg = \
740         (nvlink_inband_mc_team_setup_req_msg_t *)&sendDataParams->buffer[0];
741 
742     pMcTeamSetupReq = \
743         (nvlink_inband_mc_team_setup_req_t *)&pMcTeamSetupReqMsg->mcTeamSetupReq;
744 
745     payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_setup_req_t) + \
746                          (sizeof(pMcTeamSetupReq->gpuHandles[0]) * \
747                                  pMulticastFabricDesc->numMaxGpus));
748 
749     sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize);
750 
751     NV_ASSERT((NvU32)sendDataSize <= sizeof(sendDataParams->buffer));
752 
753     portMemSet(sendDataParams, 0, sendDataSize);
754 
755     pMcTeamSetupReq->mcAllocSize = pMulticastFabricDesc->allocSize;
756     pMcTeamSetupReq->numGpuHandles = pMulticastFabricDesc->numMaxGpus;
757 
758     for (pNode = listHead(&pMulticastFabricDesc->gpuInfoList);
759          pNode != NULL;
760          pNode = listNext(&pMulticastFabricDesc->gpuInfoList, pNode))
761         pMcTeamSetupReq->gpuHandles[idx++] = pNode->gpuProbeHandle;
762 
763     NV_ASSERT(idx == pMcTeamSetupReq->numGpuHandles);
764 
765     sendDataParams->dataSize = sendDataSize;
766 
767     nvlinkInitInbandMsgHdr(&pMcTeamSetupReqMsg->msgHdr,
768                            NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ,
769                            payloadSize, pMulticastFabricDesc->inbandReqId);
770 
771     status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu),
772                                    sendDataParams);
773     if (status != NV_OK)
774         goto done;
775 
776     pMulticastFabricDesc->bInbandReqInProgress = NV_TRUE;
777 
778 done:
779     portMemFree(sendDataParams);
780 
781     return status;
782 }
783 
784 NV_STATUS
785 _memMulticastFabricSendInbandTeamReleaseRequestV1
786 (
787     OBJGPU *pGpu,
788     NvU64   mcTeamHandle
789 )
790 {
791     NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams;
792     nvlink_inband_mc_team_release_req_msg_t *pMcTeamReleaseReqMsg = NULL;
793     nvlink_inband_mc_team_release_req_t *pMcTeamReleaseReq = NULL;
794     NvU32 payloadSize;
795     NvU32 sendDataSize;
796     NV_STATUS status;
797 
798     sendDataParams = (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *)
799         portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS));
800 
801     if (sendDataParams == NULL)
802         return NV_ERR_NO_MEMORY;
803 
804     pMcTeamReleaseReqMsg = (nvlink_inband_mc_team_release_req_msg_t *) \
805                            &sendDataParams->buffer[0];
806 
807     pMcTeamReleaseReq = (nvlink_inband_mc_team_release_req_t *) \
808                         &pMcTeamReleaseReqMsg->mcTeamReleaseReq;
809 
810     payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_release_req_t));
811 
812     sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize);
813 
814     portMemSet(sendDataParams, 0, sendDataSize);
815 
816     pMcTeamReleaseReq->mcTeamHandle = mcTeamHandle;
817 
818     nvlinkInitInbandMsgHdr(&pMcTeamReleaseReqMsg->msgHdr,
819                            NVLINK_INBAND_MSG_TYPE_MC_TEAM_RELEASE_REQ,
820                            payloadSize, osGetTimestamp());
821 
822     sendDataParams->dataSize = sendDataSize;
823 
824     status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu),
825                                    sendDataParams);
826 
827     portMemFree(sendDataParams);
828 
829     return status;
830 }
831 
832 NV_STATUS
833 _memMulticastFabricSendInbandTeamSetupRequest
834 (
835     OBJGPU                          *pGpu,
836     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
837 )
838 {
839     NvU64 fmCaps;
840     NV_STATUS status = NV_OK;
841 
842     status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel,
843                                      &fmCaps);
844     if (status != NV_OK)
845         return status;
846 
847     if (fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V2)
848     {
849         return _memMulticastFabricSendInbandTeamSetupRequestV2(pGpu,
850                                                         pMulticastFabricDesc);
851     }
852     else if (fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V1)
853     {
854         return _memMulticastFabricSendInbandTeamSetupRequestV1(pGpu,
855                                                         pMulticastFabricDesc);
856     }
857     else
858     {
859         return NV_ERR_NOT_SUPPORTED;
860     }
861 }
862 
863 NV_STATUS
864 _memMulticastFabricSendInbandTeamReleaseRequest
865 (
866     OBJGPU *pGpu,
867     NvU64   mcTeamHandle
868 )
869 {
870     NvU64 fmCaps;
871     NV_STATUS status = NV_OK;
872 
873     // If mcTeamHandle is zero, nothing to do...
874     if (mcTeamHandle == 0)
875         return NV_OK;
876 
877     status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel,
878                                      &fmCaps);
879     if (status != NV_OK)
880         return status;
881 
882     if (!(fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_RELEASE_V1))
883         return NV_ERR_NOT_SUPPORTED;
884 
885     return _memMulticastFabricSendInbandTeamReleaseRequestV1(pGpu,
886                                                              mcTeamHandle);
887 }
888 
889 static NV_STATUS
890 _memMulticastFabricSendInbandRequest
891 (
892     OBJGPU                            *pGpu,
893     MEM_MULTICAST_FABRIC_DESCRIPTOR   *pMulticastFabricDesc,
894     MEM_MULTICAST_FABRIC_REQUEST_TYPE  requestType
895 )
896 {
897     NV_STATUS status = NV_OK;
898     NvU32 gpuMask;
899 
900     if (pGpu == NULL)
901         return NV_ERR_NOT_SUPPORTED;
902 
903     gpuMask = NVBIT(gpuGetInstance(pGpu));
904     if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask))
905     {
906         NV_ASSERT(0);
907         return NV_ERR_INVALID_STATE;
908     }
909 
910     switch (requestType)
911     {
912         case MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST:
913             status = _memMulticastFabricSendInbandTeamSetupRequest(pGpu,
914                                             pMulticastFabricDesc);
915             break;
916         case MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST:
917             status = _memMulticastFabricSendInbandTeamReleaseRequest(pGpu,
918                                             pMulticastFabricDesc->mcTeamHandle);
919             break;
920         default:
921             status = NV_ERR_NOT_SUPPORTED;
922             break;
923     }
924 
925     return status;
926 }
927 
928 //
929 // This function may be called with RO pMulticastFabricDesc->pLock. Don't modify
930 // pMulticastFabricDesc.
931 //
932 static void
933 _memorymulticastfabricDetachMem
934 (
935     FABRIC_VASPACE     *pFabricVAS,
936     MEMORY_DESCRIPTOR  *pFabricMemDesc,
937     NODE               *pMemNode
938 )
939 {
940     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
941     MEMORY_DESCRIPTOR *pPhysMemDesc;
942     MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *pAttachMemInfoNode;
943 
944     pAttachMemInfoNode =
945         (MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *)pMemNode->Data;
946     pPhysMemDesc = pAttachMemInfoNode->pPhysMemDesc;
947 
948     fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc,
949                                   pMemNode->keyStart,
950                                   pPhysMemDesc,
951                                   pAttachMemInfoNode->physMapLength);
952 
953     NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient,
954                               pAttachMemInfoNode->hDupedPhysMem));
955 }
956 
957 static void
958 _memorymulticastfabricBatchDetachMem
959 (
960     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
961 )
962 {
963     MEMORY_DESCRIPTOR *pFabricMemDesc;
964     MEM_MULTICAST_FABRIC_GPU_INFO *pGpuNode;
965     NODE *pMemNode;
966     FABRIC_VASPACE *pFabricVAS;
967     NvU32 gpuMask;
968     OBJGPU *pGpu;
969 
970     pFabricMemDesc = pMulticastFabricDesc->pMemDesc;
971     NV_ASSERT_OR_RETURN_VOID(pFabricMemDesc != NULL);
972 
973     for (pGpuNode = listHead(&pMulticastFabricDesc->gpuInfoList);
974          pGpuNode != NULL;
975          pGpuNode = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuNode))
976     {
977         pGpu = pGpuNode->pGpu;
978         gpuMask = NVBIT(gpuGetInstance(pGpu));
979 
980         NV_ASSERT(rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask));
981 
982         pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
983         if (pFabricVAS == NULL)
984         {
985             NV_ASSERT(0);
986             continue;
987         }
988 
989         btreeEnumStart(0, &pMemNode, pGpuNode->pAttachMemInfoTree);
990         while (pMemNode != NULL)
991         {
992             _memorymulticastfabricDetachMem(pFabricVAS, pFabricMemDesc,
993                                             pMemNode);
994 
995             btreeUnlink(pMemNode, &pGpuNode->pAttachMemInfoTree);
996             portMemFree(pMemNode->Data);
997 
998             btreeEnumStart(0, &pMemNode, pGpuNode->pAttachMemInfoTree);
999         }
1000 
1001         // Everything is detached during object cleanup, free MCFLA now.
1002         if (pGpuNode->bMcflaAlloc)
1003         {
1004             fabricvaspaceBatchFree(pFabricVAS, &pFabricMemDesc->_pteArray[0],
1005                                    1, 1);
1006             pGpuNode->bMcflaAlloc = NV_FALSE;
1007         }
1008     }
1009 }
1010 
1011 static void
1012 _memMulticastFabricDescriptorFree
1013 (
1014     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc
1015 )
1016 {
1017     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
1018     NvU32 allocFlags;
1019     NvU32 gpuMask;
1020     NvBool bGpuLockTaken = NV_FALSE;
1021 
1022     if (pMulticastFabricDesc == NULL)
1023         return;
1024 
1025     //
1026     // Take pMulticastFabricModuleLock to synchronize with
1027     // memorymulticastfabricTeamSetupResponseCallback() and
1028     // _memMulticastFabricDescriptorAllocUsingExpPacket().
1029     // We don't want to delete pMulticastFabricDesc under them.
1030     //
1031     portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock);
1032     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
1033 
1034     pMulticastFabricDesc->refCount--;
1035 
1036     if (pMulticastFabricDesc->refCount != 0)
1037     {
1038         portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1039         portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock);
1040 
1041         return;
1042     }
1043 
1044     allocFlags = pMulticastFabricDesc->allocFlags;
1045 
1046     //
1047     // Empty caches so new calls to
1048     // memorymulticastfabricTeamSetupResponseCallback() and
1049     // _memMulticastFabricDescriptorAllocUsingExpPacket() fail.
1050     //
1051     if (!_memMulticastFabricIsPrime(allocFlags))
1052     {
1053         NV_ASSERT(pMulticastFabricDesc->cacheKey != 0);
1054         fabricImportCacheDelete(pFabric, &pMulticastFabricDesc->expUuid,
1055                                 pMulticastFabricDesc->cacheKey);
1056     }
1057 
1058     fabricMulticastSetupCacheDelete(pFabric,
1059                                     pMulticastFabricDesc->inbandReqId);
1060 
1061     // Now I am the only one holding the pMulticastFabricDesc, drop the locks.
1062     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1063     portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock);
1064 
1065     //
1066     // TODO: Make this a per-GPU lock in future. For now both dup() and free()
1067     // forces all GPU lock.
1068     //
1069 
1070     if (!rmGpuLockIsOwner())
1071     {
1072         gpuMask = GPUS_LOCK_ALL;
1073 
1074         if (rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE,
1075                                   RM_LOCK_MODULES_MEM, &gpuMask) == NV_OK)
1076         {
1077             bGpuLockTaken = NV_TRUE;
1078         }
1079         else
1080         {
1081             NV_ASSERT(0);
1082         }
1083     }
1084 
1085     if (pMulticastFabricDesc->pMemDesc != NULL)
1086     {
1087         MEM_MULTICAST_FABRIC_GPU_INFO *pNode =
1088                                 listHead(&pMulticastFabricDesc->gpuInfoList);
1089 
1090         NV_ASSERT(pMulticastFabricDesc->bMemdescInstalled);
1091         NV_ASSERT(pNode != NULL);
1092 
1093         _memorymulticastfabricBatchDetachMem(pMulticastFabricDesc);
1094 
1095         if (_memMulticastFabricIsPrime(allocFlags))
1096         {
1097             _memMulticastFabricSendInbandRequest(pNode->pGpu,
1098                                 pMulticastFabricDesc,
1099                                 MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST);
1100         }
1101     }
1102 
1103     if (pMulticastFabricDesc->bInbandReqInProgress)
1104     {
1105         OS_WAIT_QUEUE *pWq;
1106         THREAD_STATE_NODE *pThreadNode = NULL;
1107         THREAD_STATE_FREE_CALLBACK freeCallback;
1108 
1109         NV_ASSERT_OK(osAllocWaitQueue(&pWq));
1110 
1111         if (pWq != NULL)
1112         {
1113             NV_ASSERT_OK(fabricMulticastCleanupCacheInsert(pFabric,
1114                                         pMulticastFabricDesc->inbandReqId,
1115                                         pWq));
1116 
1117             NV_ASSERT_OK(threadStateGetCurrent(&pThreadNode, NULL));
1118 
1119             freeCallback.pCb = fabricMulticastWaitOnTeamCleanupCallback;
1120             freeCallback.pCbData = (void *)pMulticastFabricDesc->inbandReqId;
1121 
1122             NV_ASSERT_OK(threadStateEnqueueCallbackOnFree(pThreadNode,
1123                                                           &freeCallback));
1124         }
1125     }
1126 
1127     _memMulticastFabricGpuInfoRemove(pMulticastFabricDesc);
1128 
1129     if (bGpuLockTaken)
1130     {
1131         rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
1132         bGpuLockTaken = NV_FALSE;
1133     }
1134 
1135     _memMulticastFabricDescriptorCleanup(pMulticastFabricDesc);
1136 }
1137 
1138 static
1139 MEM_MULTICAST_FABRIC_DESCRIPTOR*
1140 _memMulticastFabricDescriptorAllocUsingExpPacket
1141 (
1142     MemoryMulticastFabric        *pMemoryMulticastFabric,
1143     NV00FD_ALLOCATION_PARAMETERS *pAllocParams
1144 )
1145 {
1146     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
1147     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
1148     NvU64 cacheKey;
1149     NvUuid expUuid;
1150     NV_EXPORT_MEM_PACKET *pExportPacket = &pAllocParams->expPacket;
1151     NV_STATUS status = NV_OK;
1152 
1153     ct_assert(NV_MEM_EXPORT_UUID_LEN == NV_UUID_LEN);
1154     portMemCopy(expUuid.uuid, NV_UUID_LEN, pExportPacket->uuid,
1155                 NV_MEM_EXPORT_UUID_LEN);
1156 
1157     //
1158     // To reuse import cache with the UCFLA class (0xf9), we create
1159     // unique import cache keys for MCFLA using NV00F9_IMPORT_ID_MAX, which
1160     // are never used by UCFLA.
1161     //
1162     cacheKey =
1163         (NV00F9_IMPORT_ID_MAX << NV00F9_IMPORT_ID_SHIFT) | pAllocParams->index;
1164 
1165     //
1166     // Take pMulticastFabricModuleLock to synchronize multiple constructors
1167     // to share the cached pMulticastFabricDesc. Without this lock, two or
1168     // more constructors may see the import cache empty at the same time.
1169     //
1170     // We want the following sequence happen atomically:
1171     //
1172     //    pMulticastFabricDesc = fabricImportCacheGet(..);
1173     //    if (pMulticastFabricDesc == NULL)
1174     //    {
1175     //        pMulticastFabricDesc = alloc();
1176     //        fabricImportCacheInsert(pMulticastFabricDesc);
1177     //        ...
1178     //    }
1179     //
1180     // Also, it is important to synchronize the constructor with the destructor.
1181     // The constructor looks up and refcounts pMulticastFabricDesc
1182     // non-atomically. Thus, pMulticastFabricDesc may be destroyed before
1183     // it could be refcounted.
1184     //
1185     //  pMulticastFabricDesc = fabricImportCacheGet(..);
1186     //  if (pMulticastFabricDesc != NULL)
1187     //  {
1188     //      pMulticastFabricDesc->lock();
1189     //      pMulticastFabricDesc->refCount++;
1190     //     ...
1191     //   }
1192     //
1193 
1194     portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock);
1195 
1196     pMulticastFabricDesc = fabricImportCacheGet(pFabric, &expUuid, cacheKey);
1197     if (pMulticastFabricDesc != NULL)
1198     {
1199         portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
1200 
1201         NV_ASSERT(!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags));
1202 
1203         pMulticastFabricDesc->refCount++;
1204 
1205         portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1206 
1207         goto done;
1208     }
1209 
1210     pMulticastFabricDesc =
1211         _memMulticastFabricDescriptorAlloc(pMemoryMulticastFabric,
1212                                            pAllocParams);
1213 
1214     if (pMulticastFabricDesc == NULL)
1215     {
1216         status = NV_ERR_NO_MEMORY;
1217         goto done;
1218     }
1219 
1220     //
1221     // For the imported object, set numMaxGpus to NV_U32_MAX for two reasons.
1222     // a. It doesn't track how many GPUs the team should have. That's tracked
1223     //    by the exporter.
1224     // b. This object should never send MCFLA team setup request to the FM.
1225     //    Setting numMaxGpus to NV_U32_MAX, makes sure that implicitly.
1226     //
1227     pMulticastFabricDesc->numMaxGpus = NV_U32_MAX;
1228 
1229     //
1230     // allocSize is set to zero. GFM will provide that in the team setup
1231     // response.
1232     //
1233     pMulticastFabricDesc->allocSize = 0;
1234 
1235     //
1236     // For now only pageSize support is 512MB. This needs to be revisited
1237     // in case we support more pagesizes.
1238     //
1239     pMulticastFabricDesc->pageSize = NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M;
1240     pMulticastFabricDesc->alignment = NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M;
1241 
1242     pMulticastFabricDesc->exportNodeId = memoryExportGetNodeId(pExportPacket);
1243     pMulticastFabricDesc->expUuid = expUuid;
1244     pMulticastFabricDesc->cacheKey = cacheKey;
1245     pMulticastFabricDesc->index = pAllocParams->index;
1246 
1247     // insert into cache once ready...
1248     status = fabricImportCacheInsert(pFabric, &expUuid, cacheKey,
1249                                      pMulticastFabricDesc);
1250 
1251 done:
1252     portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock);
1253 
1254     if (status != NV_OK)
1255     {
1256         _memMulticastFabricDescriptorFree(pMulticastFabricDesc);
1257         pMulticastFabricDesc = NULL;
1258     }
1259 
1260     return pMulticastFabricDesc;
1261 }
1262 
1263 NV_STATUS
1264 _memMulticastFabricConstruct
1265 (
1266     MemoryMulticastFabric        *pMemoryMulticastFabric,
1267     CALL_CONTEXT                 *pCallContext,
1268     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
1269 )
1270 {
1271     Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory);
1272     NV00FD_ALLOCATION_PARAMETERS *pAllocParams = pParams->pAllocParams;
1273     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
1274     NV_STATUS status = NV_OK;
1275 
1276     if (!_memMulticastFabricIsPrime(pAllocParams->allocFlags))
1277     {
1278         NV_EXPORT_MEM_PACKET *pExportPacket = &pAllocParams->expPacket;
1279 
1280         // If this a single-node UUID?
1281         if (memoryExportGetNodeId(pExportPacket) == NV_FABRIC_INVALID_NODE_ID)
1282             return NV_ERR_NOT_SUPPORTED;
1283 
1284         pMulticastFabricDesc =
1285             _memMulticastFabricDescriptorAllocUsingExpPacket(
1286                                                 pMemoryMulticastFabric,
1287                                                 pAllocParams);
1288     }
1289     else
1290     {
1291         pMulticastFabricDesc =
1292             _memMulticastFabricDescriptorAlloc(pMemoryMulticastFabric,
1293                                                pAllocParams);
1294     }
1295 
1296     if (pMulticastFabricDesc == NULL)
1297         return NV_ERR_NO_MEMORY;
1298 
1299     //
1300     // At this point the pMulticastFabricDesc can be fresh or in-use, but will
1301     // be refcounted for safe access. If in-use, it can get modified before
1302     // pMulticastFabricDesc->pLock() is acquired.
1303     //
1304 
1305     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
1306 
1307     status = _memMulticastFabricDescriptorEnqueueWait(pParams->hClient,
1308                                                       pMulticastFabricDesc,
1309                                                       pAllocParams->pOsEvent,
1310                                                       pMemory);
1311     if (status != NV_OK)
1312         goto fail;
1313 
1314     pMemoryMulticastFabric->pMulticastFabricDesc = pMulticastFabricDesc;
1315 
1316     pMemoryMulticastFabric->expNodeId = pMulticastFabricDesc->exportNodeId;
1317     pMemoryMulticastFabric->bImported =
1318                !_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags);
1319 
1320     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1321 
1322     return NV_OK;
1323 
1324 fail:
1325     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1326 
1327     _memMulticastFabricDescriptorFree(pMulticastFabricDesc);
1328 
1329     return status;
1330 }
1331 
1332 NV_STATUS
1333 _memMulticastFabricCreateMemDesc
1334 (
1335     MEM_MULTICAST_FABRIC_DESCRIPTOR  *pMulticastFabricDesc,
1336     NvU64                             mcAddressBase,
1337     MEMORY_DESCRIPTOR               **ppMemDesc
1338 )
1339 {
1340     NV_STATUS status;
1341     MEMORY_DESCRIPTOR *pTempMemDesc = NULL;
1342 
1343     status = memdescCreate(&pTempMemDesc, NULL,
1344                            pMulticastFabricDesc->allocSize,
1345                            0, NV_TRUE, ADDR_FABRIC_MC,
1346                            NV_MEMORY_UNCACHED,
1347                            MEMDESC_FLAGS_NONE);
1348     if (status != NV_OK)
1349     {
1350         NV_PRINTF(LEVEL_ERROR,
1351             "Failed to allocate memory descriptor for multicast object\n");
1352         return status;
1353     }
1354 
1355     memdescSetPte(pTempMemDesc, AT_GPU, 0, mcAddressBase);
1356 
1357     memdescSetPageSize(pTempMemDesc, AT_GPU,
1358                        pMulticastFabricDesc->pageSize);
1359 
1360     pTempMemDesc->_pteKind = NV_MMU_PTE_KIND_SMSKED_MESSAGE;
1361 
1362     memdescSetFlag(pTempMemDesc, MEMDESC_FLAGS_SET_KIND, NV_TRUE);
1363 
1364     memdescSetGpuCacheAttrib(pTempMemDesc, NV_MEMORY_UNCACHED);
1365 
1366     *ppMemDesc = pTempMemDesc;
1367 
1368     return NV_OK;
1369 }
1370 
1371 void
1372 _memMulticastFabricInstallMemDesc
1373 (
1374     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc,
1375     MEMORY_DESCRIPTOR               *pMemDesc,
1376     NvU64                            mcTeamHandle,
1377     NV_STATUS                        status
1378 )
1379 {
1380     NV_ASSERT(pMulticastFabricDesc->pMemDesc == NULL);
1381 
1382     pMulticastFabricDesc->pMemDesc = pMemDesc;
1383     pMulticastFabricDesc->bMemdescInstalled = NV_TRUE;
1384     pMulticastFabricDesc->mcTeamHandle = mcTeamHandle;
1385     pMulticastFabricDesc->mcTeamStatus = status;
1386 
1387     _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc);
1388 }
1389 
1390 static NV_STATUS
1391 _memorymulticastFabricAllocVas
1392 (
1393     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc,
1394     MEMORY_DESCRIPTOR               *pFabricMemDesc
1395 )
1396 {
1397     NV_STATUS status = NV_OK;
1398     FABRIC_VASPACE *pFabricVAS;
1399     MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo;
1400     VAS_ALLOC_FLAGS flags = { 0 };
1401     NvU64 gpuProbeHandle;
1402 
1403     for (pGpuInfo = listHead(&pMulticastFabricDesc->gpuInfoList);
1404          pGpuInfo != NULL;
1405          pGpuInfo = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuInfo))
1406     {
1407         OBJGPU *pGpu = pGpuInfo->pGpu;
1408         NvU32 gpuMask = NVBIT(gpuGetInstance(pGpu));
1409 
1410         if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask))
1411         {
1412             NV_ASSERT(0);
1413             status = NV_ERR_INVALID_STATE;
1414             goto cleanup;
1415         }
1416 
1417         pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1418         if (pFabricVAS == NULL)
1419         {
1420             status = NV_ERR_INVALID_STATE;
1421             goto cleanup;
1422         }
1423 
1424         //
1425         // The fabric handle might not be available or have changed, if fabric
1426         // state was ever invalidated while MCFLA allocation was in progress.
1427         //
1428         status = gpuFabricProbeGetGpuFabricHandle(pGpu->pGpuFabricProbeInfoKernel,
1429                                                   &gpuProbeHandle);
1430         if ((status != NV_OK) || (pGpuInfo->gpuProbeHandle != gpuProbeHandle))
1431         {
1432             NV_PRINTF(LEVEL_ERROR, "Attached GPU's probe handle is stale\n");
1433             status = NV_ERR_INVALID_DEVICE;
1434             goto cleanup;
1435         }
1436 
1437         status = fabricvaspaceAllocMulticast(pFabricVAS,
1438                                     memdescGetPageSize(pFabricMemDesc, AT_GPU),
1439                                     pMulticastFabricDesc->alignment,
1440                                     flags, pFabricMemDesc->_pteArray[0],
1441                                     pMulticastFabricDesc->allocSize);
1442         if (status != NV_OK)
1443         {
1444             NV_PRINTF(LEVEL_ERROR,
1445                       "Fabric VA space alloc failed for GPU %d\n",
1446                       pGpuInfo->pGpu->gpuInstance);
1447             goto cleanup;
1448         }
1449 
1450         pGpuInfo->bMcflaAlloc = NV_TRUE;
1451     }
1452 
1453     return NV_OK;
1454 
1455 cleanup:
1456     for (pGpuInfo = listHead(&pMulticastFabricDesc->gpuInfoList);
1457          pGpuInfo != NULL;
1458          pGpuInfo = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuInfo))
1459     {
1460         if (pGpuInfo->bMcflaAlloc)
1461         {
1462             pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE);
1463 
1464             fabricvaspaceBatchFree(pFabricVAS, &pFabricMemDesc->_pteArray[0],
1465                                    1, 1);
1466 
1467             pGpuInfo->bMcflaAlloc = NV_FALSE;
1468         }
1469     }
1470 
1471     return status;
1472 }
1473 
1474 static void
1475 _memMulticastFabricAttachGpuPostProcessor
1476 (
1477     OBJGPU                          *pGpu,
1478     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc,
1479     NV_STATUS                        mcTeamStatus,
1480     NvU64                            mcTeamHandle,
1481     NvU64                            mcAddressBase,
1482     NvU64                            mcAddressSize
1483 )
1484 {
1485     NV_STATUS status = mcTeamStatus;
1486     MEMORY_DESCRIPTOR *pMemDesc = NULL;
1487 
1488     //
1489     // FM is never expected to return NV_ERR_NOT_READY
1490     // as part of the inband response.
1491     //
1492     NV_ASSERT(mcTeamStatus != NV_ERR_NOT_READY);
1493 
1494     //
1495     // There is a failure case on multinode systems, where a buggy client
1496     // may cause an error the prime object (e.g. attaching a GPU even after all
1497     // the required GPUs were attached) after MCFLA team request is sent to the
1498     // GFM. This should never happen under normal case, but in case it happens,
1499     // fail loudly.
1500     //
1501     // Initial value is not NV_ERR_NOT_READY, means a failure was observed.
1502     if (pMulticastFabricDesc->mcTeamStatus != NV_ERR_NOT_READY)
1503     {
1504         status = pMulticastFabricDesc->mcTeamStatus;
1505         NV_ASSERT((status != NV_OK));
1506         goto installMemDesc;
1507     }
1508 
1509     if (mcTeamStatus != NV_OK)
1510         goto installMemDesc;
1511 
1512     if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags))
1513         pMulticastFabricDesc->allocSize = mcAddressSize;
1514 
1515     if (mcAddressSize < pMulticastFabricDesc->allocSize)
1516     {
1517         NV_PRINTF(LEVEL_ERROR,
1518                   "Insufficient mcAddressSize returned from Fabric Manager\n");
1519         status = NV_ERR_INSUFFICIENT_RESOURCES;
1520         goto installMemDesc;
1521     }
1522 
1523     if (!NV_IS_ALIGNED64(mcAddressBase,
1524                          NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M))
1525     {
1526         NV_PRINTF(LEVEL_ERROR,
1527                   "Insufficient mcAddressSize returned from Fabric Manager\n");
1528         status = NV_ERR_INVALID_ADDRESS;
1529         goto installMemDesc;
1530     }
1531 
1532     status = _memMulticastFabricCreateMemDesc(pMulticastFabricDesc,
1533                                               mcAddressBase, &pMemDesc);
1534     if (status != NV_OK)
1535     {
1536         NV_PRINTF(LEVEL_ERROR, "Failed to allocate fabric memdesc\n");
1537         goto installMemDesc;
1538     }
1539 
1540     status = _memorymulticastFabricAllocVas(pMulticastFabricDesc, pMemDesc);
1541     if (status != NV_OK)
1542     {
1543         NV_PRINTF(LEVEL_ERROR, "Failed to allocate fabric VAS\n");
1544         memdescDestroy(pMemDesc);
1545         pMemDesc = NULL;
1546         goto installMemDesc;
1547     }
1548 
1549 installMemDesc:
1550     _memMulticastFabricInstallMemDesc(pMulticastFabricDesc, pMemDesc,
1551                                       mcTeamHandle, status);
1552 
1553      if ((status != NV_OK) && (mcTeamStatus == NV_OK))
1554          _memMulticastFabricSendInbandRequest(pGpu, pMulticastFabricDesc,
1555                                 MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST);
1556 }
1557 
1558 NV_STATUS
1559 memorymulticastfabricTeamSetupResponseCallback
1560 (
1561     NvU32                                           gpuInstance,
1562     NvU64                                          *pNotifyGfidMask,
1563     NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams
1564 )
1565 {
1566     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
1567     nvlink_inband_mc_team_setup_rsp_msg_t *pMcTeamSetupRspMsg;
1568     nvlink_inband_mc_team_setup_rsp_t *pMcTeamSetupRsp;
1569     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
1570     NvU64 requestId;
1571     NV_STATUS mcTeamStatus;
1572     NvU64 mcTeamHandle = 0;
1573     NvU64 mcAddressBase = 0;
1574     NvU64 mcAddressSize = 0;
1575     OBJGPU *pGpu;
1576 
1577     NV_ASSERT(pInbandRcvParams != NULL);
1578     NV_ASSERT(rmGpuLockIsOwner());
1579 
1580     if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL)
1581     {
1582         NV_ASSERT_FAILED("Invalid GPU instance");
1583         return NV_ERR_INVALID_ARGUMENT;
1584     }
1585 
1586     pMcTeamSetupRspMsg =
1587         (nvlink_inband_mc_team_setup_rsp_msg_t *)&pInbandRcvParams->data[0];
1588 
1589     pMcTeamSetupRsp =
1590         (nvlink_inband_mc_team_setup_rsp_t *)&pMcTeamSetupRspMsg->mcTeamSetupRsp;
1591 
1592     requestId = pMcTeamSetupRspMsg->msgHdr.requestId;
1593 
1594     mcTeamStatus = pMcTeamSetupRspMsg->msgHdr.status;
1595 
1596     if (mcTeamStatus == NV_OK)
1597     {
1598         mcTeamHandle = pMcTeamSetupRsp->mcTeamHandle;
1599         mcAddressBase = pMcTeamSetupRsp->mcAddressBase;
1600         mcAddressSize = pMcTeamSetupRsp->mcAddressSize;
1601 
1602 #if defined(DEBUG) || defined(DEVELOP)
1603         {
1604             // Make sure that the reserved fields are initialized to 0
1605             NvU8 *pRsvd = &pMcTeamSetupRsp->reserved[0];
1606 
1607             NV_ASSERT((pRsvd[0] == 0) && portMemCmp(pRsvd, pRsvd + 1,
1608                       (sizeof(pMcTeamSetupRsp->reserved) - 1)) == 0);
1609         }
1610 #endif
1611     }
1612 
1613     //
1614     // Acquire pMulticastFabricModuleLock here, to make sure
1615     // pMulticastFabricDesc is not removed underneath us.
1616     // The thread doesn't hold refcount on pMulticastFabricDesc.
1617     //
1618     portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock);
1619 
1620     pMulticastFabricDesc = fabricMulticastSetupCacheGet(pFabric, requestId);
1621 
1622     if (pMulticastFabricDesc != NULL)
1623     {
1624         fabricMulticastSetupCacheDelete(pFabric, requestId);
1625 
1626         //
1627         // We have now safely acquired pMulticastFabricDesc->lock, which
1628         // should block the destructor from removing pMulticastFabricDesc
1629         // under us even if pMulticastFabricModuleLock is dropped.
1630         //
1631         portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
1632 
1633         //
1634         // Drop pMulticastFabricModuleLock here as out of order, now we
1635         // shouldn't need it ever.
1636         //
1637         portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock);
1638 
1639         pMulticastFabricDesc->bInbandReqInProgress = NV_FALSE;
1640 
1641         _memMulticastFabricAttachGpuPostProcessor(pGpu,
1642                                                   pMulticastFabricDesc,
1643                                                   mcTeamStatus,
1644                                                   mcTeamHandle,
1645                                                   mcAddressBase,
1646                                                   mcAddressSize);
1647 
1648         portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1649     }
1650     else
1651     {
1652         //
1653         // As pMulticastFabricDesc is not found, drop the lock right
1654         // away. GPU locks are already taken to submit the team release
1655         // request.
1656         //
1657         portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock);
1658 
1659         if (mcTeamStatus == NV_OK)
1660             (void)_memMulticastFabricSendInbandTeamReleaseRequest(pGpu,
1661                                                                   mcTeamHandle);
1662 
1663         //
1664         // Check if there is any thread waiting for team release and
1665         // wake it up.
1666         //
1667         // The multicast fabric descriptor could have undergone the
1668         // destruct sequence while an inband team setup request was in
1669         // progress with FM.
1670         //
1671         // In such a scenario the last thread to free the multicast
1672         // descriptor is put to sleep until the team setup response
1673         // is received and a subsequent team release request is sent.
1674         //
1675         fabricMulticastCleanupCacheInvokeCallback(pFabric, requestId,
1676                                                   fabricWakeUpThreadCallback);
1677     }
1678 
1679     return NV_OK;
1680 }
1681 
1682 NV_STATUS
1683 memorymulticastfabricConstruct_IMPL
1684 (
1685     MemoryMulticastFabric        *pMemoryMulticastFabric,
1686     CALL_CONTEXT                 *pCallContext,
1687     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
1688 )
1689 {
1690     NV00FD_ALLOCATION_PARAMETERS *pAllocParams = pParams->pAllocParams;
1691 
1692     if (RS_IS_COPY_CTOR(pParams))
1693     {
1694         return memorymulticastfabricCopyConstruct(pMemoryMulticastFabric,
1695                                                   pCallContext, pParams);
1696     }
1697 
1698     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
1699                 _memMulticastFabricValidateAllocParams(pAllocParams));
1700 
1701     return _memMulticastFabricConstruct(pMemoryMulticastFabric,
1702                                         pCallContext, pParams);
1703 }
1704 
1705 //
1706 // Note this function is not always called with the GPU lock. Be careful
1707 // while accessing GPU state.
1708 //
1709 static NV_STATUS
1710 _memorymulticastfabricCtrlAttachGpu
1711 (
1712     OBJGPU                        *pGpu,
1713     MemoryMulticastFabric         *pMemoryMulticastFabric,
1714     NV00FD_CTRL_ATTACH_GPU_PARAMS *pParams
1715 )
1716 {
1717     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
1718                                 pMemoryMulticastFabric->pMulticastFabricDesc;
1719     NV_STATUS status;
1720     FABRIC_VASPACE *pFabricVAS;
1721     MEM_MULTICAST_FABRIC_GPU_INFO *pNode;
1722     MEM_MULTICAST_FABRIC_GPU_INFO *pHead;
1723     MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteHead;
1724     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
1725 
1726     if (pParams->flags != 0)
1727     {
1728         NV_PRINTF(LEVEL_ERROR, "flags passed for attach mem must be zero\n");
1729         return NV_ERR_INVALID_ARGUMENT;
1730     }
1731 
1732     // Check if the Multicast FLA object has any additional slots for GPUs
1733     if (pMulticastFabricDesc->numAttachedGpus ==
1734                                 pMulticastFabricDesc->numMaxGpus)
1735     {
1736         NV_PRINTF(LEVEL_ERROR, "Max no. of GPUs have already attached!\n");
1737         return NV_ERR_INVALID_OPERATION;
1738     }
1739 
1740     if (pMulticastFabricDesc->bMemdescInstalled)
1741     {
1742         NV_PRINTF(LEVEL_ERROR, "The object is already ready.\n");
1743         return NV_ERR_STATE_IN_USE;
1744     }
1745 
1746     if (RMCFG_FEATURE_PLATFORM_WINDOWS ||
1747         gpuIsCCFeatureEnabled(pGpu) ||
1748         IS_VIRTUAL(pGpu))
1749     {
1750         NV_PRINTF(LEVEL_ERROR,
1751                   "Multicast attach not supported on Windows/CC/vGPU modes\n");
1752         return NV_ERR_NOT_SUPPORTED;
1753     }
1754 
1755     status = _memMulticastFabricGpuInfoAdd(pMemoryMulticastFabric,
1756                                            pCallContext->pControlParams);
1757     if (status != NV_OK)
1758     {
1759         NV_PRINTF(LEVEL_ERROR, "Failed to populate GPU info\n");
1760         return status;
1761     }
1762 
1763     pNode = listTail(&pMulticastFabricDesc->gpuInfoList);
1764 
1765     status = gpuFabricProbeGetGpuFabricHandle(pGpu->pGpuFabricProbeInfoKernel,
1766                                               &pNode->gpuProbeHandle);
1767     if (status != NV_OK)
1768     {
1769         NV_PRINTF(LEVEL_ERROR,
1770                   "Attaching GPU does not have a valid probe handle\n");
1771         goto fail;
1772     }
1773 
1774     status = gpuFabricProbeGetFabricCliqueId(pGpu->pGpuFabricProbeInfoKernel,
1775                                              &pNode->cliqueId);
1776     if (status != NV_OK)
1777     {
1778         NV_PRINTF(LEVEL_ERROR,
1779                   "Attaching GPU does not have a valid clique ID\n");
1780         goto fail;
1781     }
1782 
1783     pHead = listHead(&pMulticastFabricDesc->gpuInfoList);
1784     pRemoteHead = multimapFirstItem(&pMulticastFabricDesc->remoteGpuInfoMap);
1785 
1786     if ((pHead != NULL) && (pHead->cliqueId != pNode->cliqueId))
1787     {
1788         NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n");
1789         status = NV_ERR_INVALID_DEVICE;
1790         goto fail;
1791     }
1792 
1793     if ((pRemoteHead != NULL) && (pRemoteHead->cliqueId != pNode->cliqueId))
1794     {
1795         NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n");
1796         status = NV_ERR_INVALID_DEVICE;
1797         goto fail;
1798     }
1799 
1800     pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE);
1801     if (pFabricVAS == NULL)
1802     {
1803         NV_PRINTF(LEVEL_ERROR,
1804                   "Fabric vaspace object not available for GPU %x\n",
1805                   pGpu->gpuInstance);
1806         status = NV_ERR_NOT_SUPPORTED;
1807         goto fail;
1808     }
1809 
1810     if ((pMulticastFabricDesc->numAttachedGpus + 1)
1811                                     == pMulticastFabricDesc->numMaxGpus)
1812     {
1813         NV_ASSERT(_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags));
1814 
1815         status = _memMulticastFabricSendInbandRequest(pGpu,
1816                                     pMulticastFabricDesc,
1817                                     MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST);
1818         if (status != NV_OK)
1819         {
1820             NV_PRINTF(LEVEL_ERROR,
1821                       "Inband request Multicast Team Setup failed!\n");
1822             goto fail;
1823         }
1824     }
1825     // Invoke remote GPU attach event..
1826     else if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags))
1827     {
1828         NvU64 fmCaps;
1829         Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
1830         NV00F1_CTRL_FABRIC_EVENT event;
1831 
1832         status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel,
1833                                          &fmCaps);
1834         if (status != NV_OK)
1835         {
1836             NV_PRINTF(LEVEL_ERROR, "Failed to query IMEX FM caps\n");
1837             goto fail;
1838         }
1839 
1840         if (!(fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V2))
1841         {
1842             NV_PRINTF(LEVEL_ERROR, "Remote attach is supported from V2+\n");
1843             status = NV_ERR_NOT_SUPPORTED;
1844             goto fail;
1845         }
1846 
1847         _memMulticastFabricInitAttachEvent(pNode->gpuProbeHandle,
1848                                            pMulticastFabricDesc->inbandReqId,
1849                                            pNode->cliqueId,
1850                                            pMulticastFabricDesc->exportNodeId,
1851                                            pMulticastFabricDesc->index,
1852                                            &pMulticastFabricDesc->expUuid,
1853                                            &event);
1854 
1855         pNode->attachEventId = event.id;
1856 
1857         status = fabricPostEventsV2(pFabric, &event, 1);
1858         if (status != NV_OK)
1859         {
1860             NV_PRINTF(LEVEL_ERROR,
1861                       "Failed to notify IMEX daemon of import event\n");
1862             goto fail;
1863         }
1864     }
1865 
1866     //
1867     // DO NOT FAIL after the call to fabricPostEventsV2 or
1868     // _memMulticastFabricSendInbandRequest().
1869     //
1870     // These functions are used to communicate with the external
1871     // entities like FM/IMEX daemons. It is recommended that we
1872     // do not fail the control call after these to avoid
1873     // complicated cleanups.
1874     //
1875 
1876     pMulticastFabricDesc->numAttachedGpus++;
1877     pMulticastFabricDesc->localAttachedGpusMask |= NVBIT32(pGpu->gpuInstance);
1878 
1879     return NV_OK;
1880 
1881 fail:
1882     // Remove GPU OS info added in the prologue.
1883     listRemove(&pMulticastFabricDesc->gpuInfoList, pNode);
1884 
1885     return status;
1886 }
1887 
1888 NV_STATUS
1889 memorymulticastfabricCtrlAttachGpu_IMPL
1890 (
1891     MemoryMulticastFabric         *pMemoryMulticastFabric,
1892     NV00FD_CTRL_ATTACH_GPU_PARAMS *pParams
1893 )
1894 {
1895     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
1896                                 pMemoryMulticastFabric->pMulticastFabricDesc;
1897     NV_STATUS status;
1898     NvBool bLastAttach;
1899     NvBool bLastAttachRecheck;
1900     NvBool bGpuLockTaken = NV_FALSE;
1901     Subdevice *pSubdevice;
1902     OBJGPU *pGpu;
1903     NvU32 gpuMask;
1904 
1905     status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric),
1906                                   pParams->hSubdevice, &pSubdevice);
1907     if (status == NV_ERR_OBJECT_NOT_FOUND)
1908         status = NV_ERR_INVALID_DEVICE;
1909 
1910     if (status != NV_OK)
1911         return status;
1912 
1913     pGpu = GPU_RES_GET_GPU(pSubdevice);
1914     gpuMask = NVBIT(gpuGetInstance(pGpu));
1915 
1916 retry:
1917     //
1918     // Find if I am the last attach. If yes, take a GPU lock to submit
1919     // inband request. Otherwise, skip taking GPU lock.
1920     //
1921     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
1922 
1923     bLastAttach = ((pMulticastFabricDesc->numMaxGpus -
1924                     pMulticastFabricDesc->numAttachedGpus) == 1);
1925 
1926     // Drop the lock to avoid lock inversion with the GPU lock...
1927     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
1928 
1929     if (bLastAttach && (gpuMask != 0))
1930     {
1931         status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK,
1932                                        GPUS_LOCK_FLAGS_NONE,
1933                                        RM_LOCK_MODULES_MEM, &gpuMask);
1934         if (status != NV_OK)
1935         {
1936             NV_ASSERT(0);
1937             return status;
1938         }
1939 
1940         bGpuLockTaken = NV_TRUE;
1941     }
1942 
1943     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
1944 
1945     //
1946     // Recheck to avoid TOCTOU, make sure bLastAttach state didn't change..
1947     //
1948     // This is a deterministic check as once the GPU is attached to
1949     // this object it can't be detached unless the object is destroyed.
1950     //
1951     bLastAttachRecheck = ((pMulticastFabricDesc->numMaxGpus -
1952                            pMulticastFabricDesc->numAttachedGpus) == 1);
1953 
1954     if (bLastAttachRecheck != bLastAttach)
1955     {
1956         portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1957 
1958         if (bGpuLockTaken)
1959         {
1960             rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
1961             bGpuLockTaken = NV_FALSE;
1962         }
1963 
1964         goto retry;
1965     }
1966 
1967     status = _memorymulticastfabricCtrlAttachGpu(pGpu, pMemoryMulticastFabric,
1968                                                  pParams);
1969 
1970     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
1971 
1972     if (bGpuLockTaken)
1973         rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
1974 
1975     return status;
1976 }
1977 
1978 #ifdef NV00FD_CTRL_CMD_SET_FAILURE
1979 static NV_STATUS
1980 _memorymulticastfabricCtrlSetFailure
1981 (
1982     MemoryMulticastFabric          *pMemoryMulticastFabric,
1983     NV00FD_CTRL_SET_FAILURE_PARAMS *pParams
1984 )
1985 {
1986     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
1987     NvHandle hClient = pCallContext->pClient->hClient;
1988     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
1989                                 pMemoryMulticastFabric->pMulticastFabricDesc;
1990 
1991     if (!rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) &&
1992         !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel))
1993         return NV_ERR_INSUFFICIENT_PERMISSIONS;
1994 
1995     if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags))
1996     {
1997         NV_PRINTF(LEVEL_ERROR, "Only supported on prime MCLFA object\n");
1998         return NV_ERR_NOT_SUPPORTED;
1999     }
2000 
2001     if ((pParams->status == NV_OK) || (pParams->status == NV_ERR_NOT_READY))
2002         return NV_ERR_INVALID_ARGUMENT;
2003 
2004     pMulticastFabricDesc->mcTeamStatus = pParams->status;
2005 
2006     // Notify the callers about the failure.
2007     _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc);
2008 
2009     return NV_OK;
2010 }
2011 
2012 NV_STATUS memorymulticastfabricCtrlSetFailure_IMPL
2013 (
2014     MemoryMulticastFabric          *pMemoryMulticastFabric,
2015     NV00FD_CTRL_SET_FAILURE_PARAMS *pParams
2016 )
2017 {
2018     NV_STATUS status;
2019     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2020                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2021 
2022     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
2023 
2024     status = _memorymulticastfabricCtrlSetFailure(pMemoryMulticastFabric,
2025                                                   pParams);
2026 
2027     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2028 
2029     return status;
2030 }
2031 #endif
2032 
2033 #ifdef NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU
2034 //
2035 // Note this function is not always called with the GPU lock. Be careful
2036 // while accessing GPU state.
2037 //
2038 static NV_STATUS
2039 _memorymulticastfabricCtrlAttachRemoteGpu
2040 (
2041     OBJGPU                               *pGpu,
2042     MemoryMulticastFabric                *pMemoryMulticastFabric,
2043     NV00FD_CTRL_ATTACH_REMOTE_GPU_PARAMS *pParams
2044 )
2045 {
2046     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2047                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2048     MEM_MULTICAST_FABRIC_GPU_INFO *pHead =
2049                         listHead(&pMulticastFabricDesc->gpuInfoList);
2050     MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteHead =
2051                     multimapFirstItem(&pMulticastFabricDesc->remoteGpuInfoMap);
2052     MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pNode;
2053     MEM_MULTICAST_FABRIC_GPU_INFO *pIter;
2054     MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = NULL;
2055     CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
2056     NvHandle hClient = pCallContext->pClient->hClient;
2057     NV00FD_CTRL_SET_FAILURE_PARAMS params;
2058     NV_STATUS status;
2059 
2060     if (!rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) &&
2061         !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel))
2062     {
2063         // Don't set failure as non-priv client can hit this failure too.
2064         return NV_ERR_INSUFFICIENT_PERMISSIONS;
2065     }
2066 
2067     if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags))
2068     {
2069         NV_PRINTF(LEVEL_ERROR, "Only supported on prime MCLFA object\n");
2070         status = NV_ERR_NOT_SUPPORTED;
2071         goto fail;
2072     }
2073 
2074     // Check if the Multicast FLA object has any additional slots for GPUs
2075     if (pMulticastFabricDesc->numAttachedGpus ==
2076                                 pMulticastFabricDesc->numMaxGpus)
2077     {
2078         NV_PRINTF(LEVEL_ERROR, "Max no. of GPUs have already attached!\n");
2079         status = NV_ERR_INVALID_OPERATION;
2080         goto fail;
2081     }
2082 
2083     if ((pHead != NULL) && (pHead->cliqueId != pParams->cliqueId))
2084     {
2085         NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n");
2086         status = NV_ERR_INVALID_DEVICE;
2087         goto fail;
2088     }
2089 
2090     if ((pRemoteHead != NULL) && (pRemoteHead->cliqueId != pParams->cliqueId))
2091     {
2092         NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n");
2093         status = NV_ERR_INVALID_DEVICE;
2094         goto fail;
2095     }
2096 
2097     if (pParams->nodeId == NV_FABRIC_INVALID_NODE_ID)
2098     {
2099         NV_PRINTF(LEVEL_ERROR, "Invalid node ID\n");
2100         status = NV_ERR_INVALID_ARGUMENT;
2101         goto fail;
2102     }
2103 
2104     for (pIter = listHead(&pMulticastFabricDesc->gpuInfoList);
2105          pIter != NULL;
2106          pIter = listNext(&pMulticastFabricDesc->gpuInfoList, pIter))
2107     {
2108         if (pIter->gpuProbeHandle == pParams->gpuFabricProbeHandle)
2109         {
2110            NV_PRINTF(LEVEL_ERROR, "GPU is already attached\n");
2111            status = NV_ERR_INSERT_DUPLICATE_NAME;
2112            goto fail;
2113         }
2114     }
2115 
2116     if (multimapFindSubmap(&pMulticastFabricDesc->remoteGpuInfoMap,
2117                            pParams->nodeId) == NULL)
2118     {
2119         pSubmap = multimapInsertSubmap(&pMulticastFabricDesc->remoteGpuInfoMap,
2120                                        pParams->nodeId);
2121         if (pSubmap == NULL)
2122         {
2123             status = NV_ERR_NO_MEMORY;
2124             goto fail;
2125         }
2126     }
2127 
2128     pNode = multimapInsertItemNew(&pMulticastFabricDesc->remoteGpuInfoMap,
2129                                   pParams->nodeId,
2130                                   pParams->gpuFabricProbeHandle);
2131     if (pNode == NULL)
2132     {
2133         // This could also fail due to NO_MEMORY error..
2134         NV_PRINTF(LEVEL_ERROR, "Failed to track remote GPU info\n");
2135         status = NV_ERR_INSERT_DUPLICATE_NAME;
2136         goto cleanup_submap;
2137     }
2138 
2139     portMemSet(pNode, 0, sizeof(*pNode));
2140 
2141     pNode->key = pParams->key;
2142     pNode->cliqueId = pParams->cliqueId;
2143 
2144     if ((pMulticastFabricDesc->numAttachedGpus + 1)
2145                                     == pMulticastFabricDesc->numMaxGpus)
2146     {
2147         status = _memMulticastFabricSendInbandRequest(pGpu,
2148                                     pMulticastFabricDesc,
2149                                     MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST);
2150         if (status != NV_OK)
2151         {
2152             NV_PRINTF(LEVEL_ERROR,
2153                       "Inband request Multicast Team Setup failed!\n");
2154             goto unlink_node;
2155         }
2156     }
2157 
2158     //
2159     // DO NOT FAIL after the call to _memMulticastFabricSendInbandRequest().
2160     //
2161     // This function is used to communicate with the external
2162     // entities like FM daemon. It is recommended that we do not
2163     // fail the control call after this to avoid complicated state
2164     // cleanups.
2165     //
2166 
2167     pMulticastFabricDesc->numAttachedGpus++;
2168 
2169     return NV_OK;
2170 
2171 unlink_node:
2172     multimapRemoveItemByKey(&pMulticastFabricDesc->remoteGpuInfoMap,
2173                             pParams->nodeId, pParams->gpuFabricProbeHandle);
2174 
2175 cleanup_submap:
2176     if (pSubmap != NULL)
2177         multimapRemoveSubmap(&pMulticastFabricDesc->remoteGpuInfoMap, pSubmap);
2178 
2179 fail:
2180     //
2181     // NV_ERR_NOT_READY means the object is not ready, which is the initial
2182     // state. Thus, in case any API returns NV_ERR_NOT_READY, overwrite it.
2183     //
2184     if (status == NV_ERR_NOT_READY)
2185         status = NV_ERR_GENERIC;
2186 
2187     params.status = status;
2188 
2189     NV_ASSERT_OK(_memorymulticastfabricCtrlSetFailure(pMemoryMulticastFabric,
2190                                                       &params));
2191 
2192     return status;
2193 }
2194 
2195 NV_STATUS
2196 memorymulticastfabricCtrlAttachRemoteGpu_IMPL
2197 (
2198     MemoryMulticastFabric                *pMemoryMulticastFabric,
2199     NV00FD_CTRL_ATTACH_REMOTE_GPU_PARAMS *pParams
2200 )
2201 {
2202     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2203                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2204     NV_STATUS status;
2205     OBJGPU *pGpu;
2206     MEM_MULTICAST_FABRIC_GPU_INFO *pNode;
2207     NvU32 gpuMask;
2208     NvBool bLastAttach;
2209     NvBool bLastAttachRecheck;
2210     NvBool bGpuLockTaken = NV_FALSE;
2211 
2212 retry:
2213     //
2214     // Find if I am the last attach. If yes, take a GPU lock to submit
2215     // inband request. Otherwise, skip taking GPU lock.
2216     //
2217     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2218 
2219     bLastAttach = ((pMulticastFabricDesc->numMaxGpus -
2220                     pMulticastFabricDesc->numAttachedGpus) == 1);
2221 
2222     pNode = listHead(&pMulticastFabricDesc->gpuInfoList);
2223     pGpu = (pNode != NULL) ? pNode->pGpu : NULL;
2224     gpuMask = (pGpu != NULL) ? NVBIT(gpuGetInstance(pGpu)) : 0;
2225 
2226     // Drop the lock to avoid lock inversion with the GPU lock...
2227     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2228 
2229     if (bLastAttach && (gpuMask != 0))
2230     {
2231         status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK,
2232                                        GPUS_LOCK_FLAGS_NONE,
2233                                        RM_LOCK_MODULES_MEM, &gpuMask);
2234         if (status != NV_OK)
2235         {
2236             NV_ASSERT(0);
2237             return status;
2238         }
2239 
2240         bGpuLockTaken = NV_TRUE;
2241     }
2242 
2243     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
2244 
2245     //
2246     // Recheck to avoid TOCTOU, make sure bLastAttach state didn't change..
2247     //
2248     // This is a deterministic check as once the GPU is attached to
2249     // this object it can't be detached unless the object is destroyed.
2250     //
2251     bLastAttachRecheck = ((pMulticastFabricDesc->numMaxGpus -
2252                            pMulticastFabricDesc->numAttachedGpus) == 1);
2253 
2254     if (bLastAttachRecheck != bLastAttach)
2255     {
2256         portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2257 
2258         if (bGpuLockTaken)
2259         {
2260             rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
2261             bGpuLockTaken = NV_FALSE;
2262         }
2263 
2264         goto retry;
2265     }
2266 
2267     status = _memorymulticastfabricCtrlAttachRemoteGpu(pGpu,
2268                                                        pMemoryMulticastFabric,
2269                                                        pParams);
2270 
2271     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2272 
2273     if (bGpuLockTaken)
2274         rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
2275 
2276     return status;
2277 }
2278 #endif
2279 
2280 static MEM_MULTICAST_FABRIC_GPU_INFO*
2281 _memorymulticastfabricGetAttchedGpuInfo
2282 (
2283     MemoryMulticastFabric  *pMemoryMulticastFabric,
2284     Subdevice              *pSubdevice
2285 )
2286 {
2287     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2288                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2289     MEM_MULTICAST_FABRIC_GPU_INFO *pNodeItr;
2290 
2291     for (pNodeItr = listHead(&pMulticastFabricDesc->gpuInfoList);
2292          pNodeItr != NULL;
2293          pNodeItr = listNext(&pMulticastFabricDesc->gpuInfoList, pNodeItr))
2294     {
2295         if (pNodeItr->pGpu == GPU_RES_GET_GPU(pSubdevice))
2296             return pNodeItr;
2297     }
2298 
2299     return NULL;
2300 }
2301 
2302 //
2303 // This function is called with RO pMulticastFabricDesc->pLock. Don't modify
2304 // pMulticastFabricDesc.
2305 //
2306 static NV_STATUS
2307 _memorymulticastfabricCtrlDetachMem
2308 (
2309     Subdevice                     *pSubdevice,
2310     MemoryMulticastFabric         *pMemoryMulticastFabric,
2311     NV00FD_CTRL_DETACH_MEM_PARAMS *pParams
2312 )
2313 {
2314     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2315                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2316     MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo;
2317     NODE *pNode;
2318     MEMORY_DESCRIPTOR *pFabricMemDesc;
2319     FABRIC_VASPACE *pFabricVAS;
2320     NV_STATUS status;
2321     OBJGPU *pGpu;
2322     NvU32 gpuMask;
2323 
2324     if (pParams->flags != 0)
2325         return NV_ERR_INVALID_ARGUMENT;
2326 
2327     pGpuInfo = _memorymulticastfabricGetAttchedGpuInfo(pMemoryMulticastFabric,
2328                                                        pSubdevice);
2329     if (pGpuInfo == NULL)
2330         return NV_ERR_INVALID_DEVICE;
2331 
2332     pGpu = pGpuInfo->pGpu;
2333     gpuMask = NVBIT(gpuGetInstance(pGpu));
2334 
2335     if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask))
2336         return NV_ERR_INVALID_LOCK_STATE;
2337 
2338     status = btreeSearch(pParams->offset, &pNode, pGpuInfo->pAttachMemInfoTree);
2339     if (status != NV_OK)
2340         return status;
2341 
2342     pFabricMemDesc = pMulticastFabricDesc->pMemDesc;
2343     NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_STATE);
2344 
2345     pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE);
2346     NV_ASSERT_OR_RETURN(pFabricVAS != NULL, NV_ERR_INVALID_STATE);
2347 
2348     _memorymulticastfabricDetachMem(pFabricVAS, pFabricMemDesc, pNode);
2349 
2350     btreeUnlink(pNode, &pGpuInfo->pAttachMemInfoTree);
2351     portMemFree(pNode->Data);
2352 
2353     return NV_OK;
2354 }
2355 
2356 NV_STATUS
2357 memorymulticastfabricCtrlDetachMem_IMPL
2358 (
2359     MemoryMulticastFabric         *pMemoryMulticastFabric,
2360     NV00FD_CTRL_DETACH_MEM_PARAMS *pParams
2361 )
2362 {
2363     NV_STATUS status;
2364     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2365                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2366     Subdevice *pSubdevice;
2367     NvU32 gpuMask;
2368 
2369     status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric),
2370                                   pParams->hSubdevice, &pSubdevice);
2371     if (status == NV_ERR_OBJECT_NOT_FOUND)
2372         status = NV_ERR_INVALID_DEVICE;
2373 
2374     if (status != NV_OK)
2375         return status;
2376 
2377     //
2378     // TODO: Make this a per-GPU lock in future. For now both dup() and free()
2379     // forces all GPU lock.
2380     //
2381     gpuMask = GPUS_LOCK_ALL;
2382 
2383     //
2384     // Take per-GPU lock as we ensure source and destination devices are the
2385     // same.
2386     //
2387     status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE,
2388                                    RM_LOCK_MODULES_MEM, &gpuMask);
2389     if (status != NV_OK)
2390     {
2391         NV_ASSERT(0);
2392         return status;
2393     }
2394 
2395     //
2396     // pMulticastFabricDesc->pLock read locking is sufficient as rest of the
2397     // state in being modified is per-GPU, which is protected by the GPU lock.
2398     //
2399     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2400 
2401     status = _memorymulticastfabricCtrlDetachMem(pSubdevice,
2402                                                  pMemoryMulticastFabric,
2403                                                  pParams);
2404 
2405     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2406 
2407     rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
2408 
2409     return status;
2410 }
2411 
2412 static NV_STATUS
2413 _memorymulticastfabricValidatePhysMem
2414 (
2415     MemoryMulticastFabric *pMemoryMulticastFabric,
2416     NvHandle               hPhysMem,
2417     OBJGPU                *pAttachedGpu,
2418     MEMORY_DESCRIPTOR    **ppPhysMemDesc
2419 )
2420 {
2421     MEMORY_DESCRIPTOR *pPhysMemDesc;
2422     NvU64 physPageSize;
2423     NV_STATUS status;
2424     Memory *pMemory;
2425 
2426     status = memGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric),
2427                             hPhysMem, &pMemory);
2428     if (status != NV_OK)
2429     {
2430         NV_PRINTF(LEVEL_ERROR, "Invalid object handle passed\n");
2431         return status;
2432     }
2433 
2434     pPhysMemDesc = pMemory->pMemDesc;
2435 
2436     if ((pAttachedGpu != pPhysMemDesc->pGpu) ||
2437         !memmgrIsMemDescSupportedByFla_HAL(pAttachedGpu,
2438                                            GPU_GET_MEMORY_MANAGER(pAttachedGpu),
2439                                            pPhysMemDesc))
2440     {
2441         NV_PRINTF(LEVEL_ERROR, "Invalid physmem handle passed\n");
2442 
2443         return NV_ERR_INVALID_ARGUMENT;
2444     }
2445 
2446     physPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU);
2447     if ((physPageSize != RM_PAGE_SIZE_HUGE) &&
2448         (physPageSize != RM_PAGE_SIZE_512M))
2449     {
2450         NV_PRINTF(LEVEL_ERROR, "Physmem page size should be 2MB\n");
2451 
2452         return NV_ERR_INVALID_ARGUMENT;
2453     }
2454 
2455     *ppPhysMemDesc = pPhysMemDesc;
2456 
2457     return NV_OK;
2458 }
2459 
2460 //
2461 // This function is called with RO pMulticastFabricDesc->pLock. Don't modify
2462 // pMulticastFabricDesc.
2463 //
2464 static NV_STATUS
2465 _memorymulticastfabricCtrlAttachMem
2466 (
2467     Subdevice                     *pSubdevice,
2468     MemoryMulticastFabric         *pMemoryMulticastFabric,
2469     NV00FD_CTRL_ATTACH_MEM_PARAMS *pParams
2470 )
2471 {
2472     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2473                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2474     MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo;
2475     NV_STATUS status;
2476     MEMORY_DESCRIPTOR *pPhysMemDesc;
2477     MEMORY_DESCRIPTOR *pFabricMemDesc;
2478     NvHandle hDupedPhysMem = 0;
2479     RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
2480     FABRIC_VASPACE *pFabricVAS;
2481     MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *pNode;
2482     OBJGPU *pGpu;
2483     NvU32 gpuMask;
2484 
2485     if (pParams->flags != 0)
2486         return NV_ERR_INVALID_ARGUMENT;
2487 
2488     pGpuInfo = _memorymulticastfabricGetAttchedGpuInfo(pMemoryMulticastFabric,
2489                                                        pSubdevice);
2490     if (pGpuInfo == NULL)
2491         return NV_ERR_INVALID_DEVICE;
2492 
2493     pGpu = pGpuInfo->pGpu;
2494     gpuMask = NVBIT(gpuGetInstance(pGpu));
2495 
2496     if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask))
2497         return NV_ERR_INVALID_LOCK_STATE;
2498 
2499     status = _memorymulticastfabricValidatePhysMem(pMemoryMulticastFabric,
2500                                                    pParams->hMemory,
2501                                                    pGpuInfo->pGpu,
2502                                                    &pPhysMemDesc);
2503     if (status != NV_OK)
2504     {
2505         NV_PRINTF(LEVEL_ERROR, "Failed to validate physmem handle\n");
2506         return status;
2507     }
2508 
2509     pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE);
2510     NV_ASSERT_OR_RETURN(pFabricVAS != NULL, NV_ERR_INVALID_STATE);
2511 
2512     pFabricMemDesc = pMulticastFabricDesc->pMemDesc;
2513     NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_STATE);
2514 
2515     NV_ASSERT_OR_RETURN(pGpuInfo->bMcflaAlloc, NV_ERR_INVALID_STATE);
2516 
2517     status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient,
2518                                pFabricVAS->hDevice, &hDupedPhysMem,
2519                                RES_GET_CLIENT_HANDLE(pMemoryMulticastFabric),
2520                                pParams->hMemory, 0);
2521     if (status != NV_OK)
2522     {
2523         NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n");
2524         return status;
2525     }
2526 
2527     status = fabricvaspaceMapPhysMemdesc(pFabricVAS,
2528                                          pFabricMemDesc,
2529                                          pParams->offset,
2530                                          pPhysMemDesc,
2531                                          pParams->mapOffset,
2532                                          pParams->mapLength,
2533                                          0);
2534     if (status != NV_OK)
2535     {
2536         NV_PRINTF(LEVEL_ERROR, "Failed to map FLA\n");
2537         goto freeDupedMem;
2538     }
2539 
2540     pNode = portMemAllocNonPaged(sizeof(*pNode));
2541     if (pNode == NULL)
2542     {
2543         status = NV_ERR_NO_MEMORY;
2544         goto unmapVas;
2545     }
2546 
2547     portMemSet(pNode, 0, sizeof(*pNode));
2548 
2549     pNode->node.keyStart = pParams->offset;
2550     pNode->node.keyEnd   = pParams->offset;
2551     pNode->physMapLength = pParams->mapLength;
2552     pNode->pPhysMemDesc  = pPhysMemDesc;
2553     pNode->hDupedPhysMem = hDupedPhysMem;
2554     pNode->node.Data     = pNode;
2555 
2556     status = btreeInsert(&pNode->node, &pGpuInfo->pAttachMemInfoTree);
2557     if (status != NV_OK)
2558     {
2559         NV_PRINTF(LEVEL_ERROR, "Failed to track attach mem info\n");
2560         goto freeNode;
2561     }
2562 
2563     return NV_OK;
2564 
2565 freeNode:
2566     portMemFree(pNode);
2567 
2568 unmapVas:
2569     fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, pParams->offset,
2570                                   pPhysMemDesc, pParams->mapLength);
2571 
2572 freeDupedMem:
2573     NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient, hDupedPhysMem));
2574 
2575     return status;
2576 }
2577 
2578 NV_STATUS
2579 memorymulticastfabricCtrlAttachMem_IMPL
2580 (
2581     MemoryMulticastFabric         *pMemoryMulticastFabric,
2582     NV00FD_CTRL_ATTACH_MEM_PARAMS *pParams
2583 )
2584 {
2585     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2586                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2587     NV_STATUS status;
2588     Subdevice *pSubdevice;
2589     NvU32 gpuMask;
2590 
2591     status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric),
2592                                   pParams->hSubdevice, &pSubdevice);
2593     if (status == NV_ERR_OBJECT_NOT_FOUND)
2594         status = NV_ERR_INVALID_DEVICE;
2595 
2596     if (status != NV_OK)
2597         return status;
2598 
2599     //
2600     // TODO: Make this a per-GPU lock in future. For now both dup() and free()
2601     // forces all GPU lock.
2602     //
2603     gpuMask = GPUS_LOCK_ALL;
2604 
2605     //
2606     // Take per-GPU lock as we ensure source and destination devices are the
2607     // same.
2608     //
2609     status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE,
2610                                    RM_LOCK_MODULES_MEM, &gpuMask);
2611     if (status != NV_OK)
2612     {
2613         NV_ASSERT(0);
2614         return status;
2615     }
2616 
2617     //
2618     // pMulticastFabricDesc->pLock read locking is sufficient as rest of the
2619     // state in being modified is per-GPU, which is protected by the GPU lock.
2620     //
2621     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2622 
2623     status = _memorymulticastfabricCtrlAttachMem(pSubdevice,
2624                                                  pMemoryMulticastFabric,
2625                                                  pParams);
2626 
2627     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2628 
2629     rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
2630 
2631     return status;
2632 }
2633 
2634 void
2635 memorymulticastfabricDestruct_IMPL
2636 (
2637     MemoryMulticastFabric *pMemoryMulticastFabric
2638 )
2639 {
2640     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2641                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2642     Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory);
2643 
2644     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
2645 
2646     memDestructCommon(pMemory);
2647 
2648     _memMulticastFabricDescriptorDequeueWait(pMulticastFabricDesc, pMemory);
2649 
2650     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2651 
2652     _memMulticastFabricDescriptorFree(pMulticastFabricDesc);
2653 }
2654 
2655 NvBool
2656 memorymulticastfabricCanCopy_IMPL
2657 (
2658     MemoryMulticastFabric *pMemoryMulticastFabric
2659 )
2660 {
2661     return NV_TRUE;
2662 }
2663 
2664 NV_STATUS
2665 memorymulticastfabricCopyConstruct_IMPL
2666 (
2667     MemoryMulticastFabric        *pMemoryMulticastFabric,
2668     CALL_CONTEXT                 *pCallContext,
2669     RS_RES_ALLOC_PARAMS_INTERNAL *pParams
2670 )
2671 {
2672     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
2673 
2674     MemoryMulticastFabric *pSourceMemoryMulticastFabric =
2675         dynamicCast(pParams->pSrcRef->pResource, MemoryMulticastFabric);
2676 
2677     pMulticastFabricDesc = pSourceMemoryMulticastFabric->pMulticastFabricDesc;
2678 
2679     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
2680 
2681     pMulticastFabricDesc->refCount++;
2682 
2683     pMemoryMulticastFabric->pMulticastFabricDesc = pMulticastFabricDesc;
2684 
2685     pMemoryMulticastFabric->expNodeId = pMulticastFabricDesc->exportNodeId;
2686     pMemoryMulticastFabric->bImported =
2687                !_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags);
2688 
2689     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2690 
2691     return NV_OK;
2692 }
2693 
2694 static NV_STATUS
2695 _memorymulticastfabricCtrlGetInfo
2696 (
2697     MemoryMulticastFabric       *pMemoryMulticastFabric,
2698     NV00FD_CTRL_GET_INFO_PARAMS *pParams
2699 )
2700 {
2701     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
2702 
2703     pMulticastFabricDesc = pMemoryMulticastFabric->pMulticastFabricDesc;
2704 
2705     pParams->alignment       = pMulticastFabricDesc->alignment;
2706     pParams->allocSize       = pMulticastFabricDesc->allocSize;
2707     pParams->pageSize        = pMulticastFabricDesc->pageSize;
2708     pParams->numMaxGpus      = pMulticastFabricDesc->numMaxGpus;
2709     pParams->numAttachedGpus = pMulticastFabricDesc->numAttachedGpus;
2710 
2711     return NV_OK;
2712 }
2713 
2714 NV_STATUS
2715 memorymulticastfabricCtrlGetInfo_IMPL
2716 (
2717     MemoryMulticastFabric       *pMemoryMulticastFabric,
2718     NV00FD_CTRL_GET_INFO_PARAMS *pParams
2719 )
2720 {
2721     NV_STATUS status = NV_OK;
2722     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc;
2723 
2724     pMulticastFabricDesc = pMemoryMulticastFabric->pMulticastFabricDesc;
2725 
2726     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2727 
2728     status = _memorymulticastfabricCtrlGetInfo(pMemoryMulticastFabric,
2729                                                pParams);
2730 
2731     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2732 
2733     return status;
2734 }
2735 
2736 NV_STATUS
2737 memorymulticastfabricIsReady_IMPL
2738 (
2739     MemoryMulticastFabric *pMemoryMulticastFabric,
2740     NvBool                 bCopyConstructorContext
2741 )
2742 {
2743     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2744                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2745     Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory);
2746     NV_STATUS mcTeamStatus;
2747 
2748     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2749 
2750     mcTeamStatus = pMulticastFabricDesc->mcTeamStatus;
2751 
2752     if (bCopyConstructorContext && (mcTeamStatus == NV_ERR_NOT_READY))
2753     {
2754         portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2755         return NV_OK;
2756     }
2757 
2758     if (pMemory->pMemDesc != pMulticastFabricDesc->pMemDesc)
2759     {
2760         // This function only initializes pMemory so it should never fail.
2761         NV_ASSERT_OK(memConstructCommon(pMemory,
2762                                         NV_MEMORY_MULTICAST_FABRIC,
2763                                         0, pMulticastFabricDesc->pMemDesc,
2764                                         0, NULL, 0, 0, 0, 0,
2765                                         NVOS32_MEM_TAG_NONE, NULL));
2766     }
2767 
2768     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2769 
2770     return mcTeamStatus;
2771 }
2772 
2773 static NV_STATUS
2774 _memorymulticastfabricCtrlRegisterEvent
2775 (
2776     MemoryMulticastFabric             *pMemoryMulticastFabric,
2777     NV00FD_CTRL_REGISTER_EVENT_PARAMS *pParams
2778 )
2779 {
2780     Memory    *pMemory = staticCast(pMemoryMulticastFabric, Memory);
2781     NvHandle   hClient = RES_GET_CLIENT_HANDLE(pMemoryMulticastFabric);
2782 
2783     return _memMulticastFabricDescriptorEnqueueWait(hClient,
2784                             pMemoryMulticastFabric->pMulticastFabricDesc,
2785                             pParams->pOsEvent, pMemory);
2786 }
2787 
2788 
2789 NV_STATUS
2790 memorymulticastfabricCtrlRegisterEvent_IMPL
2791 (
2792     MemoryMulticastFabric             *pMemoryMulticastFabric,
2793     NV00FD_CTRL_REGISTER_EVENT_PARAMS *pParams
2794 )
2795 {
2796     NV_STATUS status;
2797     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2798                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2799 
2800     portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock);
2801 
2802     status = _memorymulticastfabricCtrlRegisterEvent(pMemoryMulticastFabric,
2803                                                      pParams);
2804 
2805     portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock);
2806 
2807     return status;
2808 }
2809 
2810 NV_STATUS
2811 memorymulticastfabricControl_IMPL
2812 (
2813     MemoryMulticastFabric          *pMemoryMulticastFabric,
2814     CALL_CONTEXT                   *pCallContext,
2815     RS_RES_CONTROL_PARAMS_INTERNAL *pParams
2816 )
2817 {
2818     NV_STATUS status = NV_OK;
2819 
2820     if (
2821 #ifdef NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU
2822         (pParams->cmd != NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU) &&
2823 #endif
2824 #ifdef NV00FD_CTRL_CMD_SET_FAILURE
2825         (pParams->cmd != NV00FD_CTRL_CMD_SET_FAILURE) &&
2826 #endif
2827         (pParams->cmd != NV00FD_CTRL_CMD_ATTACH_GPU))
2828     {
2829         status = memorymulticastfabricIsReady(pMemoryMulticastFabric,
2830                                               NV_FALSE);
2831     }
2832 
2833     //
2834     // If clients try to register when the multicast object
2835     // is ready, then there is nothing left to do as the memory
2836     // descriptor is already installed.
2837     //
2838     // If the status is NV_ERR_NOT_READY then we are yet to
2839     // receive the inband response and we register the event.
2840     //
2841     if (pParams->cmd == NV00FD_CTRL_CMD_REGISTER_EVENT)
2842     {
2843         if (status == NV_OK)
2844             return NV_WARN_NOTHING_TO_DO;
2845 
2846         if (status != NV_ERR_NOT_READY)
2847             return status;
2848     }
2849     else
2850     {
2851         // Clients may busy-loop on this error status, don't log error.
2852         if (status == NV_ERR_NOT_READY)
2853             return status;
2854 
2855         NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status);
2856     }
2857 
2858     //
2859     // Note: GPU lock(s) is required for some control calls. Thus, it is
2860     // incorrect to take the leaf lock here. resControl_IMPL() attempts to
2861     // acquire the GPU locks before it calls the control call body.
2862     //
2863     return resControl_IMPL(staticCast(pMemoryMulticastFabric, RsResource),
2864                            pCallContext, pParams);
2865 }
2866 
2867 NvBool
2868 memorymulticastfabricIsExportAllowed_IMPL
2869 (
2870     MemoryMulticastFabric *pMemoryMulticastFabric
2871 )
2872 {
2873     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2874                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2875     NvBool bAllowed;
2876 
2877     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2878 
2879     bAllowed = _memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags);
2880 
2881     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2882 
2883     return bAllowed;
2884 }
2885 
2886 NvBool
2887 memorymulticastfabricIsGpuMapAllowed_IMPL
2888 (
2889     MemoryMulticastFabric *pMemoryMulticastFabric,
2890     OBJGPU                *pGpu
2891 )
2892 {
2893     NvU32 localAttachedGpusMask;
2894     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2895                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2896 
2897     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2898 
2899     localAttachedGpusMask = pMulticastFabricDesc->localAttachedGpusMask;
2900 
2901     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2902 
2903     return ((localAttachedGpusMask & NVBIT32(pGpu->gpuInstance)) != 0U);
2904 }
2905 
2906 NV_STATUS
2907 memorymulticastfabricGetMapAddrSpace_IMPL
2908 (
2909     MemoryMulticastFabric *pMemoryMulticastFabric,
2910     CALL_CONTEXT          *pCallContext,
2911     NvU32                  mapFlags,
2912     NV_ADDRESS_SPACE      *pAddrSpace
2913 )
2914 {
2915     *pAddrSpace = ADDR_FABRIC_MC;
2916     return NV_OK;
2917 }
2918 
2919 void
2920 memorymulticastfabricRemoveFromCache_IMPL
2921 (
2922     MemoryMulticastFabric *pMemoryMulticastFabric
2923 )
2924 {
2925     Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
2926     MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc =
2927                                 pMemoryMulticastFabric->pMulticastFabricDesc;
2928 
2929     if (!pMemoryMulticastFabric->bImported)
2930         return;
2931 
2932     portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock);
2933 
2934     fabricImportCacheDelete(pFabric, &pMulticastFabricDesc->expUuid,
2935                             pMulticastFabricDesc->cacheKey);
2936 
2937     portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock);
2938 }
2939