1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2019-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /*!
25 * @file
26 * @brief This file contains the functions managing the NVLink fabric
27 */
28 #define NVOC_FABRIC_H_PRIVATE_ACCESS_ALLOWED
29
30 #include "os/os.h"
31 #include "compute/fabric.h"
32
33 static NV_STATUS
_fabricCacheInsert(FabricCache * pCache,NvU64 key1,NvU64 key2,NvU64 key3,void * pData)34 _fabricCacheInsert
35 (
36 FabricCache *pCache,
37 NvU64 key1,
38 NvU64 key2,
39 NvU64 key3,
40 void *pData
41 )
42 {
43 FabricCacheSubmap *pInsertedSubmap = NULL;
44 FabricCacheEntry *pInsertedEntry = NULL;
45 FabricCacheEntry *pEntry;
46 FabricCacheMapEntry *pMapEntry;
47
48 pEntry = multimapFindItem(pCache, key1, key2);
49 if (pEntry != NULL)
50 goto insert;
51
52 if (multimapFindSubmap(pCache, key1) == NULL)
53 {
54 pInsertedSubmap = multimapInsertSubmap(pCache, key1);
55 if (pInsertedSubmap == NULL)
56 goto fail;
57 }
58
59 pInsertedEntry = multimapInsertItemNew(pCache, key1, key2);
60 if (pInsertedEntry == NULL)
61 goto fail;
62
63 mapInit(&pInsertedEntry->map, portMemAllocatorGetGlobalNonPaged());
64 pEntry = pInsertedEntry;
65
66 insert:
67 pMapEntry = mapInsertNew(&pEntry->map, key3);
68 if (pMapEntry == NULL)
69 goto fail;
70
71 pMapEntry->pData = pData;
72
73 return NV_OK;
74
75 fail:
76 if (pInsertedEntry != NULL)
77 {
78 mapDestroy(&pInsertedEntry->map);
79 multimapRemoveItem(pCache, pInsertedEntry);
80 }
81
82 if (pInsertedSubmap != NULL)
83 multimapRemoveSubmap(pCache, pInsertedSubmap);
84
85 return NV_ERR_INVALID_STATE;
86 }
87
88 static void
_fabricCacheDelete(FabricCache * pCache,NvU64 key1,NvU64 key2,NvU64 key3)89 _fabricCacheDelete
90 (
91 FabricCache *pCache,
92 NvU64 key1,
93 NvU64 key2,
94 NvU64 key3
95 )
96 {
97 FabricCacheSubmap *pSubmap;
98 FabricCacheEntry *pEntry;
99
100 pEntry = multimapFindItem(pCache, key1, key2);
101 if (pEntry == NULL)
102 return;
103
104 mapRemoveByKey(&pEntry->map, key3);
105 if (mapCount(&pEntry->map) > 0)
106 return;
107
108 mapDestroy(&pEntry->map);
109 multimapRemoveItem(pCache, pEntry);
110
111 pSubmap = multimapFindSubmap(pCache, key1);
112 NV_ASSERT_OR_RETURN_VOID(pSubmap != NULL);
113
114 if (multimapCountSubmapItems(pCache, pSubmap) > 0)
115 return;
116
117 multimapRemoveSubmap(pCache, pSubmap);
118 }
119
120 static FabricCacheMapEntry*
_fabricCacheFind(FabricCache * pCache,NvU64 key1,NvU64 key2,NvU64 key3)121 _fabricCacheFind
122 (
123 FabricCache *pCache,
124 NvU64 key1,
125 NvU64 key2,
126 NvU64 key3
127 )
128 {
129 FabricCacheEntry *pEntry;
130 FabricCacheMapEntry *pMapEntry;
131
132 pEntry = multimapFindItem(pCache, key1, key2);
133 if (pEntry == NULL)
134 return NULL;
135
136 pMapEntry = mapFind(&pEntry->map, key3);
137 if (pMapEntry == NULL)
138 return NULL;
139
140 return pMapEntry;
141 }
142
143 void
fabricWakeUpThreadCallback(void * pData)144 fabricWakeUpThreadCallback
145 (
146 void *pData
147 )
148 {
149 osWakeUp((OS_WAIT_QUEUE*)pData);
150 }
151
152 static NvBool
_fabricCacheInvokeCallback(FabricCache * pCache,NvU64 key1,NvU64 key2,NvU64 key3,void (* pCb)(void * pData))153 _fabricCacheInvokeCallback
154 (
155 FabricCache *pCache,
156 NvU64 key1,
157 NvU64 key2,
158 NvU64 key3,
159 void (*pCb)(void *pData)
160 )
161 {
162 FabricCacheMapEntry *pMapEntry;
163
164 pMapEntry = _fabricCacheFind(pCache, key1, key2, key3);
165 if (pMapEntry == NULL)
166 return NV_FALSE;
167
168 (*pCb)(pMapEntry->pData);
169
170 return NV_TRUE;
171 }
172
173 static void
_fabricWaitOnUnimportCallback(void * pCbData)174 _fabricWaitOnUnimportCallback
175 (
176 void *pCbData
177 )
178 {
179 NvU64 unimportEventId = (NvU64)pCbData;
180 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
181 OS_WAIT_QUEUE *pWq;
182
183 if ((pWq = fabricUnimportCacheGet(pFabric, unimportEventId)) != NULL)
184 {
185 osWaitInterruptible(pWq);
186 fabricUnimportCacheDelete(pFabric, unimportEventId);
187 osFreeWaitQueue(pWq);
188 }
189 }
190
191 static void
_fabricUnsetUnimportCallback(NV00F1_CTRL_FABRIC_EVENT * pEvent)192 _fabricUnsetUnimportCallback
193 (
194 NV00F1_CTRL_FABRIC_EVENT *pEvent
195 )
196 {
197 THREAD_STATE_NODE *pThreadNode;
198 THREAD_STATE_FREE_CALLBACK callback;
199 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
200 OS_WAIT_QUEUE *pWq;
201 NV_STATUS status;
202
203 // Nothing to do
204 if (pEvent->type != NV00F1_CTRL_FABRIC_EVENT_TYPE_MEM_UNIMPORT)
205 return;
206
207 status = threadStateGetCurrent(&pThreadNode, NULL);
208 NV_ASSERT_OR_RETURN_VOID(status == NV_OK);
209
210 callback.pCb = _fabricWaitOnUnimportCallback;
211 callback.pCbData = (void*)pEvent->id;
212
213 threadStateRemoveCallbackOnFree(pThreadNode, &callback);
214
215 pWq = fabricUnimportCacheGet(pFabric, pEvent->id);
216 if (pWq != NULL)
217 {
218 fabricUnimportCacheDelete(pFabric, pEvent->id);
219 osFreeWaitQueue(pWq);
220 }
221 }
222
223 static NV_STATUS
_fabricSetUnimportCallback(NV00F1_CTRL_FABRIC_EVENT * pEvent)224 _fabricSetUnimportCallback
225 (
226 NV00F1_CTRL_FABRIC_EVENT *pEvent
227 )
228 {
229 THREAD_STATE_NODE *pThreadNode = NULL;
230 THREAD_STATE_FREE_CALLBACK callback;
231 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
232 NV_STATUS status;
233 OS_WAIT_QUEUE *pWq = NULL;
234
235 // Nothing to do
236 if (pEvent->type != NV00F1_CTRL_FABRIC_EVENT_TYPE_MEM_UNIMPORT)
237 return NV_OK;
238
239 status = threadStateGetCurrent(&pThreadNode, NULL);
240 if (status != NV_OK)
241 return status;
242
243 callback.pCb = _fabricWaitOnUnimportCallback;
244 callback.pCbData = (void*)pEvent->id;
245
246 status = osAllocWaitQueue(&pWq);
247 if (status != NV_OK)
248 return status;
249
250 status = fabricUnimportCacheInsert(pFabric, pEvent->id, pWq);
251 if (status != NV_OK)
252 goto fail;
253
254 status = threadStateEnqueueCallbackOnFree(pThreadNode, &callback);
255 if (status != NV_OK)
256 goto fail;
257
258 return NV_OK;
259
260 fail:
261 fabricUnimportCacheDelete(pFabric, pEvent->id);
262
263 if (pWq != NULL)
264 osFreeWaitQueue(pWq);
265
266 return status;
267 }
268
269 static NV_STATUS
_fabricNotifyEvent(Fabric * pFabric)270 _fabricNotifyEvent(Fabric *pFabric)
271 {
272 if (pFabric->pOsImexEvent == NULL)
273 {
274 NV_PRINTF(LEVEL_ERROR, "Unable to notify ImexSessionApi\n");
275 return NV_ERR_NOT_READY;
276 }
277
278 osSetEvent(NULL, pFabric->pOsImexEvent);
279
280 return NV_OK;
281 }
282
283 NV_STATUS
fabricSetImexEvent_IMPL(Fabric * pFabric,NvP64 pOsEvent)284 fabricSetImexEvent_IMPL
285 (
286 Fabric *pFabric,
287 NvP64 pOsEvent
288 )
289 {
290 //
291 // RS-TODO: Thread-safety of the cached pOsImexEvent is
292 // guaranteed by the RMAPI lock. If the semantics of the RMAPI
293 // lock change in the future then we would need to revisit this.
294 //
295 if ((pOsEvent != NULL) && (pFabric->pOsImexEvent != NULL))
296 return NV_ERR_IN_USE;
297
298 pFabric->pOsImexEvent = pOsEvent;
299
300 return NV_OK;
301 }
302
303 NvP64
fabricGetImexEvent_IMPL(Fabric * pFabric)304 fabricGetImexEvent_IMPL
305 (
306 Fabric *pFabric
307 )
308 {
309 return pFabric->pOsImexEvent;
310 }
311
312 void
fabricSetNodeId_IMPL(Fabric * pFabric,NvU16 nodeId)313 fabricSetNodeId_IMPL
314 (
315 Fabric *pFabric,
316 NvU16 nodeId
317 )
318 {
319 pFabric->nodeId = nodeId;
320 }
321
322 NvU16
fabricGetNodeId_IMPL(Fabric * pFabric)323 fabricGetNodeId_IMPL
324 (
325 Fabric *pFabric
326 )
327 {
328 return pFabric->nodeId;
329 }
330
331 NvBool
fabricIsMemAllocDisabled_IMPL(Fabric * pFabric)332 fabricIsMemAllocDisabled_IMPL
333 (
334 Fabric *pFabric
335 )
336 {
337 return !pFabric->bAllowFabricMemAlloc;
338 }
339
340 void
fabricDisableMemAlloc_IMPL(Fabric * pFabric)341 fabricDisableMemAlloc_IMPL
342 (
343 Fabric *pFabric
344 )
345 {
346 pFabric->bAllowFabricMemAlloc = NV_FALSE;
347 }
348
349 void
fabricEnableMemAlloc_IMPL(Fabric * pFabric)350 fabricEnableMemAlloc_IMPL
351 (
352 Fabric *pFabric
353 )
354 {
355 pFabric->bAllowFabricMemAlloc = NV_TRUE;
356 }
357
358 NV_STATUS
fabricPostEventsV2_IMPL(Fabric * pFabric,NV00F1_CTRL_FABRIC_EVENT * pEvents,NvU32 numEvents)359 fabricPostEventsV2_IMPL
360 (
361 Fabric *pFabric,
362 NV00F1_CTRL_FABRIC_EVENT *pEvents,
363 NvU32 numEvents
364 )
365 {
366 NV_STATUS status = NV_OK;
367 NV00F1_CTRL_FABRIC_EVENT *pNode;
368 NvU32 i, j;
369
370 if (numEvents == 0)
371 return NV_OK;
372
373 portSyncRwLockAcquireWrite(pFabric->pListLock);
374
375 for (i = 0; i < numEvents; i++)
376 {
377 pNode = listAppendNew(&pFabric->fabricEventListV2);
378 if (pNode == NULL)
379 {
380 status = NV_ERR_NO_MEMORY;
381 goto done;
382 }
383
384 status = _fabricSetUnimportCallback(&pEvents[i]);
385 if (status != NV_OK)
386 goto done;
387
388 portMemCopy(pNode, sizeof(*pNode), &pEvents[i], sizeof(pEvents[i]));
389 }
390
391 status = _fabricNotifyEvent(pFabric);
392
393 done:
394 if (status != NV_OK)
395 {
396 for (j = 0; j < i; j++)
397 {
398 pNode = listTail(&pFabric->fabricEventListV2);
399 _fabricUnsetUnimportCallback(pNode);
400 listRemove(&pFabric->fabricEventListV2, pNode);
401 }
402 }
403
404 portSyncRwLockReleaseWrite(pFabric->pListLock);
405 return status;
406 }
407
408 NvBool
fabricExtractEventsV2_IMPL(Fabric * pFabric,NV00F1_CTRL_FABRIC_EVENT * pEventArray,NvU32 * pNumEvents)409 fabricExtractEventsV2_IMPL
410 (
411 Fabric *pFabric,
412 NV00F1_CTRL_FABRIC_EVENT *pEventArray,
413 NvU32 *pNumEvents
414 )
415 {
416 NV00F1_CTRL_FABRIC_EVENT *pEvent;
417 NvU32 count = 0;
418 NvBool bMoreEvents;
419
420 portSyncRwLockAcquireWrite(pFabric->pListLock);
421
422 while ((count < *pNumEvents) &&
423 ((pEvent = listHead(&pFabric->fabricEventListV2)) != NULL))
424 {
425 portMemCopy(&pEventArray[count], sizeof(*pEvent),
426 pEvent, sizeof(*pEvent));
427
428 listRemove(&pFabric->fabricEventListV2, pEvent);
429
430 count++;
431 }
432
433 *pNumEvents = count;
434
435 bMoreEvents = listCount(&pFabric->fabricEventListV2) > 0;
436
437 portSyncRwLockReleaseWrite(pFabric->pListLock);
438
439 return bMoreEvents;
440 }
441
442 void
fabricFlushUnhandledEvents_IMPL(Fabric * pFabric)443 fabricFlushUnhandledEvents_IMPL
444 (
445 Fabric *pFabric
446 )
447 {
448 void *pEvent;
449
450 portSyncRwLockAcquireWrite(pFabric->pListLock);
451
452 while ((pEvent = listHead(&pFabric->fabricEventListV2)) != NULL)
453 listRemove(&pFabric->fabricEventListV2, pEvent);
454
455 portSyncRwLockReleaseWrite(pFabric->pListLock);
456 }
457
458 NvU64
fabricGenerateEventId_IMPL(Fabric * pFabric)459 fabricGenerateEventId_IMPL
460 (
461 Fabric *pFabric
462 )
463 {
464 return portAtomicExIncrementU64(&pFabric->eventId);
465 }
466
467 static void
_fabricCacheIterateAll(FabricCache * pCache,void (* pCb)(void * pData))468 _fabricCacheIterateAll
469 (
470 FabricCache *pCache,
471 void (*pCb)(void *pData)
472 )
473 {
474 FabricCacheIter mmIter;
475 FabricCacheEntry *pEntry;
476 FabricCacheMapIter mIter;
477 FabricCacheMapEntry *pMapEntry;
478
479 mmIter = multimapItemIterAll(pCache);
480
481 while (multimapItemIterNext(&mmIter))
482 {
483 pEntry = mmIter.pValue;
484 mIter = mapIterAll(&pEntry->map);
485
486 while (mapIterNext(&mIter))
487 {
488 pMapEntry = mIter.pValue;
489 (*pCb)(pMapEntry->pData);
490 }
491 }
492 }
493
494 static void
_fabricCacheClearAll(FabricCache * pCache)495 _fabricCacheClearAll
496 (
497 FabricCache *pCache
498 )
499 {
500 FabricCacheIter mmIter;
501 FabricCacheEntry *pEntry;
502
503 mmIter = multimapItemIterAll(pCache);
504
505 while (multimapItemIterNext(&mmIter))
506 {
507 pEntry = mmIter.pValue;
508 mapClear(&pEntry->map);
509 }
510
511 multimapClear(pCache);
512 }
513
514 NV_STATUS
fabricImportCacheInsert_IMPL(Fabric * pFabric,const NvUuid * pExportUuid,NvU64 key,void * pData)515 fabricImportCacheInsert_IMPL
516 (
517 Fabric *pFabric,
518 const NvUuid *pExportUuid,
519 NvU64 key,
520 void *pData
521 )
522 {
523 NV_STATUS status;
524
525 portSyncRwLockAcquireWrite(pFabric->pImportCacheLock);
526
527 status = _fabricCacheInsert(&pFabric->importCache,
528 NV_UUID_HI(pExportUuid),
529 NV_UUID_LO(pExportUuid),
530 key, pData);
531
532 portSyncRwLockReleaseWrite(pFabric->pImportCacheLock);
533
534 return status;
535 }
536
537 void
fabricImportCacheDelete_IMPL(Fabric * pFabric,const NvUuid * pExportUuid,NvU64 key)538 fabricImportCacheDelete_IMPL
539 (
540 Fabric *pFabric,
541 const NvUuid *pExportUuid,
542 NvU64 key
543 )
544 {
545 portSyncRwLockAcquireWrite(pFabric->pImportCacheLock);
546
547 _fabricCacheDelete(&pFabric->importCache,
548 NV_UUID_HI(pExportUuid),
549 NV_UUID_LO(pExportUuid),
550 key);
551
552 portSyncRwLockReleaseWrite(pFabric->pImportCacheLock);
553 }
554
555 void
fabricImportCacheClear_IMPL(Fabric * pFabric)556 fabricImportCacheClear_IMPL
557 (
558 Fabric *pFabric
559 )
560 {
561 portSyncRwLockAcquireWrite(pFabric->pImportCacheLock);
562
563 _fabricCacheClearAll(&pFabric->importCache);
564
565 portSyncRwLockReleaseWrite(pFabric->pImportCacheLock);
566 }
567
568 void*
fabricImportCacheGet_IMPL(Fabric * pFabric,const NvUuid * pExportUuid,NvU64 key)569 fabricImportCacheGet_IMPL
570 (
571 Fabric *pFabric,
572 const NvUuid *pExportUuid,
573 NvU64 key
574 )
575 {
576 FabricCacheMapEntry *pMapEntry;
577
578 portSyncRwLockAcquireRead(pFabric->pImportCacheLock);
579
580 pMapEntry = _fabricCacheFind(&pFabric->importCache,
581 NV_UUID_HI(pExportUuid),
582 NV_UUID_LO(pExportUuid),
583 key);
584
585 portSyncRwLockReleaseRead(pFabric->pImportCacheLock);
586
587 return (pMapEntry == NULL ? NULL : pMapEntry->pData);
588 }
589
590 NV_STATUS
fabricUnimportCacheInsert_IMPL(Fabric * pFabric,NvU64 unimportEventId,void * pData)591 fabricUnimportCacheInsert_IMPL
592 (
593 Fabric *pFabric,
594 NvU64 unimportEventId,
595 void *pData
596 )
597 {
598 NV_STATUS status;
599
600 portSyncRwLockAcquireWrite(pFabric->pUnimportCacheLock);
601
602 status = _fabricCacheInsert(&pFabric->unimportCache,
603 unimportEventId, 0, 0, pData);
604
605 portSyncRwLockReleaseWrite(pFabric->pUnimportCacheLock);
606
607 return status;
608 }
609
610 void
fabricUnimportCacheDelete_IMPL(Fabric * pFabric,NvU64 unimportEventId)611 fabricUnimportCacheDelete_IMPL
612 (
613 Fabric *pFabric,
614 NvU64 unimportEventId
615 )
616 {
617 portSyncRwLockAcquireWrite(pFabric->pUnimportCacheLock);
618
619 _fabricCacheDelete(&pFabric->unimportCache, unimportEventId, 0, 0);
620
621 portSyncRwLockReleaseWrite(pFabric->pUnimportCacheLock);
622 }
623
624 void*
fabricUnimportCacheGet_IMPL(Fabric * pFabric,NvU64 unimportEventId)625 fabricUnimportCacheGet_IMPL
626 (
627 Fabric *pFabric,
628 NvU64 unimportEventId
629 )
630 {
631 FabricCacheMapEntry *pMapEntry;
632
633 portSyncRwLockAcquireRead(pFabric->pUnimportCacheLock);
634
635 pMapEntry = _fabricCacheFind(&pFabric->unimportCache,
636 unimportEventId, 0, 0);
637
638 portSyncRwLockReleaseRead(pFabric->pUnimportCacheLock);
639
640 return (pMapEntry == NULL ? NULL : pMapEntry->pData);
641 }
642
643 NvBool
fabricUnimportCacheInvokeCallback_IMPL(Fabric * pFabric,NvU64 unimportEventId,void (* pCb)(void * pData))644 fabricUnimportCacheInvokeCallback_IMPL
645 (
646 Fabric *pFabric,
647 NvU64 unimportEventId,
648 void (*pCb)(void *pData)
649 )
650 {
651 NvBool result;
652
653 portSyncRwLockAcquireWrite(pFabric->pUnimportCacheLock);
654
655 result = _fabricCacheInvokeCallback(&pFabric->unimportCache,
656 unimportEventId, 0, 0, pCb);
657
658 portSyncRwLockReleaseWrite(pFabric->pUnimportCacheLock);
659
660 return result;
661 }
662
663 void
fabricUnimportCacheIterateAll_IMPL(Fabric * pFabric,void (* pCb)(void * pData))664 fabricUnimportCacheIterateAll_IMPL
665 (
666 Fabric *pFabric,
667 void (*pCb)(void *pData)
668 )
669 {
670 portSyncRwLockAcquireWrite(pFabric->pUnimportCacheLock);
671
672 _fabricCacheIterateAll(&pFabric->unimportCache, pCb);
673
674 portSyncRwLockReleaseWrite(pFabric->pUnimportCacheLock);
675 }
676
677 NV_STATUS
fabricMulticastSetupCacheInsert_IMPL(Fabric * pFabric,NvU64 requestId,void * pData)678 fabricMulticastSetupCacheInsert_IMPL
679 (
680 Fabric *pFabric,
681 NvU64 requestId,
682 void *pData
683 )
684 {
685 NV_STATUS status;
686
687 portSyncRwLockAcquireWrite(pFabric->pMulticastFabriCacheLock);
688
689 status = _fabricCacheInsert(&pFabric->fabricMulticastCache,
690 0, requestId, 0, pData);
691
692 portSyncRwLockReleaseWrite(pFabric->pMulticastFabriCacheLock);
693
694 return status;
695 }
696
697 void
fabricMulticastSetupCacheDelete_IMPL(Fabric * pFabric,NvU64 requestId)698 fabricMulticastSetupCacheDelete_IMPL
699 (
700 Fabric *pFabric,
701 NvU64 requestId
702 )
703 {
704 portSyncRwLockAcquireWrite(pFabric->pMulticastFabriCacheLock);
705
706 _fabricCacheDelete(&pFabric->fabricMulticastCache,
707 0, requestId, 0);
708
709 portSyncRwLockReleaseWrite(pFabric->pMulticastFabriCacheLock);
710 }
711
712 void*
fabricMulticastSetupCacheGet_IMPL(Fabric * pFabric,NvU64 requestId)713 fabricMulticastSetupCacheGet_IMPL
714 (
715 Fabric *pFabric,
716 NvU64 requestId
717 )
718 {
719 FabricCacheMapEntry *pMapEntry;
720
721 portSyncRwLockAcquireRead(pFabric->pMulticastFabriCacheLock);
722
723 pMapEntry = _fabricCacheFind(&pFabric->fabricMulticastCache,
724 0, requestId, 0);
725
726 portSyncRwLockReleaseRead(pFabric->pMulticastFabriCacheLock);
727
728 return (pMapEntry == NULL ? NULL : pMapEntry->pData);
729 }
730
731 NV_STATUS
fabricMulticastCleanupCacheInsert_IMPL(Fabric * pFabric,NvU64 requestId,void * pData)732 fabricMulticastCleanupCacheInsert_IMPL
733 (
734 Fabric *pFabric,
735 NvU64 requestId,
736 void *pData
737 )
738 {
739 NV_STATUS status;
740
741 portSyncRwLockAcquireWrite(pFabric->pMulticastFabriCacheLock);
742
743 status = _fabricCacheInsert(&pFabric->fabricMulticastCache,
744 1, requestId, 0, pData);
745
746 portSyncRwLockReleaseWrite(pFabric->pMulticastFabriCacheLock);
747
748 return status;
749 }
750
751 void
fabricMulticastCleanupCacheDelete_IMPL(Fabric * pFabric,NvU64 requestId)752 fabricMulticastCleanupCacheDelete_IMPL
753 (
754 Fabric *pFabric,
755 NvU64 requestId
756 )
757 {
758 portSyncRwLockAcquireWrite(pFabric->pMulticastFabriCacheLock);
759
760 _fabricCacheDelete(&pFabric->fabricMulticastCache,
761 1, requestId, 0);
762
763 portSyncRwLockReleaseWrite(pFabric->pMulticastFabriCacheLock);
764 }
765
766 void*
fabricMulticastCleanupCacheGet_IMPL(Fabric * pFabric,NvU64 requestId)767 fabricMulticastCleanupCacheGet_IMPL
768 (
769 Fabric *pFabric,
770 NvU64 requestId
771 )
772 {
773 FabricCacheMapEntry *pMapEntry;
774
775 portSyncRwLockAcquireRead(pFabric->pMulticastFabriCacheLock);
776
777 pMapEntry = _fabricCacheFind(&pFabric->fabricMulticastCache,
778 1, requestId, 0);
779
780 portSyncRwLockReleaseRead(pFabric->pMulticastFabriCacheLock);
781
782 return (pMapEntry == NULL ? NULL : pMapEntry->pData);
783 }
784
785 void
fabricMulticastCleanupCacheInvokeCallback_IMPL(Fabric * pFabric,NvU64 requestId,void (* pCb)(void * pData))786 fabricMulticastCleanupCacheInvokeCallback_IMPL
787 (
788 Fabric *pFabric,
789 NvU64 requestId,
790 void (*pCb)(void *pData)
791 )
792 {
793 portSyncRwLockAcquireWrite(pFabric->pMulticastFabriCacheLock);
794
795 (void)_fabricCacheInvokeCallback(&pFabric->fabricMulticastCache,
796 1, requestId, 0, pCb);
797
798 portSyncRwLockReleaseWrite(pFabric->pMulticastFabriCacheLock);
799 }
800
801 void
fabricSetFmSessionFlags_IMPL(Fabric * pFabric,NvU32 flags)802 fabricSetFmSessionFlags_IMPL
803 (
804 Fabric *pFabric,
805 NvU32 flags
806 )
807 {
808 pFabric->flags = flags;
809 }
810
811 NvU32
fabricGetFmSessionFlags_IMPL(Fabric * pFabric)812 fabricGetFmSessionFlags_IMPL
813 (
814 Fabric *pFabric
815 )
816 {
817 return pFabric->flags;
818 }
819
820 NV_STATUS
fabricConstruct_IMPL(Fabric * pFabric)821 fabricConstruct_IMPL
822 (
823 Fabric *pFabric
824 )
825 {
826 NV_STATUS status = NV_OK;
827
828 listInit(&pFabric->fabricEventListV2, portMemAllocatorGetGlobalNonPaged());
829 multimapInit(&pFabric->importCache, portMemAllocatorGetGlobalNonPaged());
830 multimapInit(&pFabric->unimportCache, portMemAllocatorGetGlobalNonPaged());
831
832 multimapInit(&pFabric->fabricMulticastCache,
833 portMemAllocatorGetGlobalNonPaged());
834
835 pFabric->pListLock =
836 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
837
838 if (pFabric->pListLock == NULL)
839 {
840 status = NV_ERR_NO_MEMORY;
841 goto fail;
842 }
843
844 pFabric->pImportCacheLock =
845 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
846
847 if (pFabric->pImportCacheLock == NULL)
848 {
849 status = NV_ERR_NO_MEMORY;
850 goto fail;
851 }
852
853 pFabric->pUnimportCacheLock =
854 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
855
856 if (pFabric->pUnimportCacheLock == NULL)
857 {
858 status = NV_ERR_NO_MEMORY;
859 goto fail;
860 }
861
862 pFabric->pFabricImportModuleLock =
863 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
864
865 if (pFabric->pFabricImportModuleLock == NULL)
866 {
867 status = NV_ERR_NO_MEMORY;
868 goto fail;
869 }
870
871 pFabric->nodeId = NV_FABRIC_INVALID_NODE_ID;
872
873 pFabric->bAllowFabricMemAlloc = NV_TRUE;
874
875 pFabric->pMulticastFabricModuleLock =
876 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
877
878 if (pFabric->pMulticastFabricModuleLock == NULL)
879 {
880 status = NV_ERR_NO_MEMORY;
881 goto fail;
882 }
883
884 pFabric->pMulticastFabriCacheLock =
885 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
886
887 if (pFabric->pMulticastFabriCacheLock == NULL)
888 {
889 status = NV_ERR_NO_MEMORY;
890 goto fail;
891 }
892
893 return NV_OK;
894
895 //TODO: Remove the WAR to suppress unused label warning
896 goto fail;
897 fail:
898 fabricDestruct_IMPL(pFabric);
899 return status;
900 }
901
902 void
fabricDestruct_IMPL(Fabric * pFabric)903 fabricDestruct_IMPL
904 (
905 Fabric *pFabric
906 )
907 {
908 NV_ASSERT(multimapCountItems(&pFabric->fabricMulticastCache) == 0);
909
910 multimapDestroy(&pFabric->fabricMulticastCache);
911
912 if (pFabric->pMulticastFabricModuleLock != NULL)
913 portSyncRwLockDestroy(pFabric->pMulticastFabricModuleLock);
914
915 if (pFabric->pMulticastFabriCacheLock != NULL)
916 portSyncRwLockDestroy(pFabric->pMulticastFabriCacheLock);
917
918 NV_ASSERT(listCount(&pFabric->fabricEventListV2) == 0);
919
920 NV_ASSERT(multimapCountItems(&pFabric->unimportCache) == 0);
921 NV_ASSERT(multimapCountItems(&pFabric->importCache) == 0);
922
923 NV_ASSERT(pFabric->nodeId == NV_FABRIC_INVALID_NODE_ID);
924
925 if (pFabric->pFabricImportModuleLock != NULL)
926 portSyncRwLockDestroy(pFabric->pFabricImportModuleLock);
927
928 if (pFabric->pUnimportCacheLock != NULL)
929 portSyncRwLockDestroy(pFabric->pUnimportCacheLock);
930
931 if (pFabric->pImportCacheLock != NULL)
932 portSyncRwLockDestroy(pFabric->pImportCacheLock);
933
934 if (pFabric->pListLock != NULL)
935 portSyncRwLockDestroy(pFabric->pListLock);
936
937 multimapDestroy(&pFabric->unimportCache);
938 multimapDestroy(&pFabric->importCache);
939
940 listDestroy(&pFabric->fabricEventListV2);
941 }
942
943 void
fabricMulticastWaitOnTeamCleanupCallback(void * pCbData)944 fabricMulticastWaitOnTeamCleanupCallback
945 (
946 void *pCbData
947 )
948 {
949 NvU64 inbandReqId = (NvU64)pCbData;
950 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE());
951 OS_WAIT_QUEUE *pWq;
952
953 pWq = (OS_WAIT_QUEUE *)fabricMulticastCleanupCacheGet(pFabric, inbandReqId);
954 if (pWq == NULL)
955 return;
956
957 osWaitInterruptible(pWq);
958 fabricMulticastCleanupCacheDelete(pFabric, inbandReqId);
959 osFreeWaitQueue(pWq);
960 }
961