1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /****************************************************************************** 25 * 26 * Description: 27 * This file contains the functions managing the memory multicast fabric 28 * 29 *****************************************************************************/ 30 #define NVOC_MEM_MULTICAST_FABRIC_H_PRIVATE_ACCESS_ALLOWED 31 32 /* 33 * Lock ordering 34 * 35 * RMAPI Lock 36 * |_Client Lock 37 * |_GPU(s) Lock 38 * |_MCFLA Module Lock 39 * |_MCFLA Descriptor Lock 40 */ 41 42 #include "os/os.h" 43 #include "core/locks.h" 44 #include "nvport/nvport.h" 45 #include "rmapi/rs_utils.h" 46 #include "rmapi/rmapi_utils.h" 47 #include "compute/fabric.h" 48 #include "gpu/gpu.h" 49 #include "gpu/bus/kern_bus.h" 50 #include "gpu/mem_mgr/mem_desc.h" 51 #include "gpu/mem_mgr/mem_mgr.h" 52 #include "gpu/subdevice/subdevice.h" 53 #include "kernel/gpu/nvlink/kernel_nvlink.h" 54 #include "mem_mgr/fabric_vaspace.h" 55 #include "mem_mgr/mem_multicast_fabric.h" 56 #include "published/hopper/gh100/dev_mmu.h" 57 58 #include "class/cl00f9.h" 59 #include "mem_mgr/mem_export.h" 60 61 #include "gpu/gpu_fabric_probe.h" 62 63 typedef struct mem_multicast_fabric_attach_mem_info_node 64 { 65 MEMORY_DESCRIPTOR *pPhysMemDesc; 66 NvHandle hDupedPhysMem; 67 NvU64 physMapLength; 68 NODE node; 69 } MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE; 70 71 typedef struct mem_multicast_fabric_client_info 72 { 73 void *pOsEvent; 74 Memory *pMemory; 75 } MEM_MULTICAST_FABRIC_CLIENT_INFO; 76 77 typedef struct mem_multicast_fabric_gpu_info 78 { 79 void *pGpuOsInfo; 80 OBJGPU *pGpu; 81 NvU64 gpuProbeHandle; 82 NvU32 cliqueId; 83 NvBool bMcflaAlloc; 84 85 // 86 // Unique import event ID. Valid only if the GPU was remotely attached to 87 // the prime MCFLA object 88 // 89 NvU64 attachEventId; 90 91 // 92 // Tracks memory attached using NV00FD_CTRL_CMD_ATTACH_MEM 93 // 94 // GPU lock must be taken to protect this tree. 95 // 96 PNODE pAttachMemInfoTree; 97 } MEM_MULTICAST_FABRIC_GPU_INFO; 98 99 typedef struct mem_multicast_fabric_remote_gpu_info 100 { 101 NvU32 cliqueId; 102 NvU64 key; 103 } MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO; 104 105 MAKE_MULTIMAP(MemMulticastFabricRemoteGpuInfoMap, 106 MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO); 107 108 MAKE_LIST(MemMulticastFabricClientInfoList, MEM_MULTICAST_FABRIC_CLIENT_INFO); 109 110 MAKE_LIST(MemMulticastFabricGpuInfoList, MEM_MULTICAST_FABRIC_GPU_INFO); 111 112 typedef enum 113 { 114 MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST = 0, 115 MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST, 116 } MEM_MULTICAST_FABRIC_REQUEST_TYPE; 117 118 typedef struct mem_multicast_fabric_descriptor 119 { 120 // Refcount to keep this descriptor alive 121 NvU64 refCount; 122 123 // List of clients waiting on this object to be ready 124 MemMulticastFabricClientInfoList waitingClientsList; 125 126 // Mask representing the list of attached GPUs 127 NvU32 localAttachedGpusMask; 128 129 // List of attached GPU info 130 MemMulticastFabricGpuInfoList gpuInfoList; 131 132 // Boolean to be set when pMemDesc is installed 133 NvBool bMemdescInstalled; 134 135 // Memory descriptor associated with the multicast object 136 MEMORY_DESCRIPTOR *pMemDesc; 137 138 // Unique handle assigned for the multicast team by FM 139 NvU64 mcTeamHandle; 140 141 // Status of the multicast team 142 NV_STATUS mcTeamStatus; 143 144 // 145 // Boolean to be set when an Inband request has been sent to FM 146 // and is currently in progress 147 // 148 NvBool bInbandReqInProgress; 149 150 // 151 // Request Id associated with the Inband request in progress when 152 // bInbandReqSent is set to true 153 // 154 NvU64 inbandReqId; 155 156 // Alignment for the multicast FLA allocation 157 NvU64 alignment; 158 159 // Multicast FLA allocation size 160 NvU64 allocSize; 161 162 // Page size for the multicast FLA 163 NvU64 pageSize; 164 165 // Multicast FLA allocation flags 166 NvU32 allocFlags; 167 168 // Max. number of unique GPUs associated with the multicast object 169 NvU32 numMaxGpus; 170 171 // No. of unique GPUs currently attached to the multicast object 172 NvU32 numAttachedGpus; 173 174 // Export object information associated with this import descriptor. 175 NvU16 exportNodeId; 176 NvU16 index; 177 178 // Same as packet.uuid, but uses NvUuid type. 179 NvUuid expUuid; 180 181 // Import cache key 182 NvU64 cacheKey; 183 184 // Map of attached remote GPU info 185 MemMulticastFabricRemoteGpuInfoMap remoteGpuInfoMap; 186 187 // 188 // The lock protects MEM_MULTICAST_FABRIC_DESCRIPTOR, the MCFLA descriptor. 189 // 190 // The lock should be taken only if an MCFLA descriptor is safe 191 // to access i.e., holding the module lock or the accessing thread 192 // has the MCFLA descriptor refcounted. 193 // 194 PORT_RWLOCK *pLock; 195 } MEM_MULTICAST_FABRIC_DESCRIPTOR; 196 197 static NvBool 198 _memMulticastFabricIsPrime 199 ( 200 NvU32 allocFlags 201 ) 202 { 203 NvBool bPrime = NV_TRUE; 204 205 #ifdef NV_MEMORY_MULTICAST_FABRIC_ALLOC_FLAGS_USE_EXPORT_PACKET 206 // 207 // If an MCFLA object is allocated using export packet (UUID), then it 208 // is a non-prime (imported) object. Such objects are just extension 209 // of the prime (exported) MCFLA objects. 210 // 211 bPrime = !(allocFlags & 212 NV_MEMORY_MULTICAST_FABRIC_ALLOC_FLAGS_USE_EXPORT_PACKET); 213 #endif 214 215 return bPrime; 216 } 217 218 static void 219 _memMulticastFabricInitAttachEvent 220 ( 221 NvU64 gpuFabricProbeHandle, 222 NvU64 key, 223 NvU32 cliqueId, 224 NvU16 exportNodeId, 225 NvU16 index, 226 NvUuid *pExportUuid, 227 NV00F1_CTRL_FABRIC_EVENT *pEvent 228 ) 229 { 230 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 231 232 pEvent->imexChannel = 0; 233 pEvent->type = NV00F1_CTRL_FABRIC_EVENT_TYPE_REMOTE_GPU_ATTACH; 234 pEvent->id = fabricGenerateEventId(pFabric); 235 236 pEvent->data.attach.gpuFabricProbeHandle = gpuFabricProbeHandle; 237 pEvent->data.attach.key = key; 238 pEvent->data.attach.cliqueId = cliqueId; 239 pEvent->data.attach.index = index; 240 pEvent->data.attach.exportNodeId = exportNodeId; 241 portMemCopy(pEvent->data.attach.exportUuid, NV_MEM_EXPORT_UUID_LEN, 242 pExportUuid->uuid, NV_MEM_EXPORT_UUID_LEN); 243 } 244 245 static void 246 _memMulticastFabricInitUnimportEvent 247 ( 248 NvU64 attachEventId, 249 NvU16 exportNodeId, 250 NV00F1_CTRL_FABRIC_EVENT *pEvent 251 ) 252 { 253 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 254 255 pEvent->imexChannel = 0; 256 pEvent->type = NV00F1_CTRL_FABRIC_EVENT_TYPE_MEM_UNIMPORT; 257 pEvent->id = fabricGenerateEventId(pFabric); 258 pEvent->data.unimport.exportNodeId = exportNodeId; 259 pEvent->data.unimport.importEventId = attachEventId; 260 } 261 262 static 263 NV_STATUS 264 _memMulticastFabricValidateAllocParams 265 ( 266 NV00FD_ALLOCATION_PARAMETERS *pAllocParams 267 ) 268 { 269 // Nothing to verify in this case.. 270 if (!_memMulticastFabricIsPrime(pAllocParams->allocFlags)) 271 return NV_OK; 272 273 // Only page size 512MB is supported 274 if (pAllocParams->pageSize != NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M) 275 { 276 NV_PRINTF(LEVEL_ERROR, 277 "Unsupported pageSize: 0x%x. Only 512MB pagesize is supported\n", 278 pAllocParams->pageSize); 279 return NV_ERR_INVALID_ARGUMENT; 280 } 281 282 if (pAllocParams->alignment != pAllocParams->pageSize) 283 { 284 NV_PRINTF(LEVEL_ERROR, 285 "Alignment must be pageSize for now\n"); 286 return NV_ERR_INVALID_ARGUMENT; 287 } 288 289 // AllocSize should be page size aligned 290 if (!NV_IS_ALIGNED64(pAllocParams->allocSize, pAllocParams->pageSize)) 291 { 292 NV_PRINTF(LEVEL_ERROR, 293 "AllocSize should be pageSize aligned\n"); 294 return NV_ERR_INVALID_ARGUMENT; 295 } 296 297 // NV_U32_MAX is reserved as a special value for non-prime MCFLA objects. 298 if ((pAllocParams->numGpus == 0) || 299 (pAllocParams->numGpus == NV_U32_MAX)) 300 { 301 NV_PRINTF(LEVEL_ERROR, "Invalid number of GPUs to attach\n"); 302 return NV_ERR_INVALID_ARGUMENT; 303 } 304 305 return NV_OK; 306 } 307 308 static void 309 _memMulticastFabricDescriptorCleanup 310 ( 311 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 312 ) 313 { 314 if (pMulticastFabricDesc == NULL) 315 return; 316 317 NV_ASSERT(listCount(&pMulticastFabricDesc->gpuInfoList) == 0); 318 listDestroy(&pMulticastFabricDesc->gpuInfoList); 319 320 multimapDestroy(&pMulticastFabricDesc->remoteGpuInfoMap); 321 322 NV_ASSERT(pMulticastFabricDesc->numAttachedGpus == 0); 323 NV_ASSERT(pMulticastFabricDesc->localAttachedGpusMask == 0); 324 325 NV_ASSERT(listCount(&pMulticastFabricDesc->waitingClientsList) == 0); 326 listDestroy(&pMulticastFabricDesc->waitingClientsList); 327 328 memdescDestroy(pMulticastFabricDesc->pMemDesc); 329 330 if (pMulticastFabricDesc->pLock != NULL) 331 portSyncRwLockDestroy(pMulticastFabricDesc->pLock); 332 333 portMemFree(pMulticastFabricDesc); 334 } 335 336 static 337 MEM_MULTICAST_FABRIC_DESCRIPTOR* 338 _memMulticastFabricDescriptorAlloc 339 ( 340 MemoryMulticastFabric *pMemoryMulticastFabric, 341 NV00FD_ALLOCATION_PARAMETERS *pAllocParams 342 ) 343 { 344 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 345 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 346 NV_STATUS status; 347 348 pMulticastFabricDesc = portMemAllocNonPaged( 349 sizeof(MEM_MULTICAST_FABRIC_DESCRIPTOR)); 350 if (pMulticastFabricDesc == NULL) 351 return NULL; 352 353 portMemSet(pMulticastFabricDesc, 0, 354 sizeof(MEM_MULTICAST_FABRIC_DESCRIPTOR)); 355 356 listInit(&pMulticastFabricDesc->waitingClientsList, 357 portMemAllocatorGetGlobalNonPaged()); 358 359 listInit(&pMulticastFabricDesc->gpuInfoList, 360 portMemAllocatorGetGlobalNonPaged()); 361 362 pMulticastFabricDesc->refCount = 1; 363 pMulticastFabricDesc->mcTeamStatus = NV_ERR_NOT_READY; 364 pMulticastFabricDesc->localAttachedGpusMask = 0; 365 pMulticastFabricDesc->alignment = pAllocParams->alignment; 366 pMulticastFabricDesc->allocSize = pAllocParams->allocSize; 367 pMulticastFabricDesc->pageSize = pAllocParams->pageSize; 368 pMulticastFabricDesc->allocFlags = pAllocParams->allocFlags; 369 pMulticastFabricDesc->numMaxGpus = pAllocParams->numGpus; 370 pMulticastFabricDesc->inbandReqId = osGetTimestamp(); 371 372 multimapInit(&pMulticastFabricDesc->remoteGpuInfoMap, 373 portMemAllocatorGetGlobalNonPaged()); 374 pMulticastFabricDesc->exportNodeId = NV_FABRIC_INVALID_NODE_ID; 375 376 pMulticastFabricDesc->pLock = 377 portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged()); 378 if (pMulticastFabricDesc->pLock == NULL) 379 goto fail; 380 381 status = fabricMulticastSetupCacheInsert(pFabric, 382 pMulticastFabricDesc->inbandReqId, 383 pMulticastFabricDesc); 384 if (status != NV_OK) 385 { 386 NV_PRINTF(LEVEL_ERROR, "Failed to track memdesc 0x%x", status); 387 goto fail; 388 } 389 390 return pMulticastFabricDesc; 391 392 fail: 393 _memMulticastFabricDescriptorCleanup(pMulticastFabricDesc); 394 395 return NULL; 396 } 397 398 static void 399 _memMulticastFabricDescriptorFlushClients 400 ( 401 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 402 ) 403 { 404 MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode; 405 406 while ((pNode = 407 listHead(&pMulticastFabricDesc->waitingClientsList)) != NULL) 408 { 409 if (pNode->pOsEvent != NULL) 410 { 411 osSetEvent(NULL, pNode->pOsEvent); 412 NV_ASSERT_OK(osDereferenceObjectCount(pNode->pOsEvent)); 413 } 414 415 listRemove(&pMulticastFabricDesc->waitingClientsList, pNode); 416 } 417 418 return; 419 } 420 421 static NV_STATUS 422 _memMulticastFabricDescriptorEnqueueWait 423 ( 424 NvHandle hClient, 425 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 426 NvP64 pOsEvent, 427 Memory *pMemory 428 ) 429 { 430 MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode; 431 NvP64 pValidatedOsEvent = NULL; 432 NV_STATUS status; 433 434 if (pOsEvent != NULL) 435 { 436 status = osUserHandleToKernelPtr(hClient, pOsEvent, 437 &pValidatedOsEvent); 438 if (status != NV_OK) 439 return status; 440 } 441 442 pNode = listAppendNew(&pMulticastFabricDesc->waitingClientsList); 443 if (pNode == NULL) 444 { 445 if (pOsEvent != NULL) 446 osDereferenceObjectCount(pValidatedOsEvent); 447 448 return NV_ERR_NO_MEMORY; 449 } 450 451 pNode->pOsEvent = pValidatedOsEvent; 452 pNode->pMemory = pMemory; 453 454 // 455 // In case the multicast object's memdesc is ready, unblock 456 // clients waiting on it 457 // 458 if (pMulticastFabricDesc->bMemdescInstalled) 459 { 460 _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc); 461 } 462 463 return NV_OK; 464 } 465 466 static void 467 _memMulticastFabricDescriptorDequeueWait 468 ( 469 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 470 Memory *pMemory 471 ) 472 { 473 MEM_MULTICAST_FABRIC_CLIENT_INFO *pNode; 474 MEM_MULTICAST_FABRIC_CLIENT_INFO *pNodeNext; 475 476 pNode = listHead(&pMulticastFabricDesc->waitingClientsList); 477 478 // There can be multiple events per memory object, so delete all. 479 while (pNode != NULL) 480 { 481 pNodeNext = listNext(&pMulticastFabricDesc->waitingClientsList, pNode); 482 483 if (pNode->pMemory == pMemory) 484 { 485 if (pNode->pOsEvent != NULL) 486 osDereferenceObjectCount(pNode->pOsEvent); 487 488 listRemove(&pMulticastFabricDesc->waitingClientsList, pNode); 489 } 490 491 pNode = pNodeNext; 492 } 493 } 494 495 static NV_STATUS 496 _memMulticastFabricGpuInfoAdd 497 ( 498 MemoryMulticastFabric *pMemoryMulticastFabric, 499 RS_RES_CONTROL_PARAMS_INTERNAL *pParams 500 ) 501 { 502 NV00FD_CTRL_ATTACH_GPU_PARAMS *pAttachParams = pParams->pParams; 503 Subdevice *pSubdevice = NULL; 504 MEM_MULTICAST_FABRIC_GPU_INFO *pNode; 505 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = \ 506 pMemoryMulticastFabric->pMulticastFabricDesc; 507 OBJGPU *pGpu; 508 MEM_MULTICAST_FABRIC_GPU_INFO *pNodeItr; 509 510 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 511 subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric), 512 pAttachParams->hSubdevice, &pSubdevice)); 513 514 pGpu = GPU_RES_GET_GPU(pSubdevice); 515 516 if(!osMatchGpuOsInfo(pGpu, pParams->secInfo.gpuOsInfo)) 517 return NV_ERR_INVALID_DEVICE; 518 519 for (pNodeItr = listHead(&pMulticastFabricDesc->gpuInfoList); 520 pNodeItr != NULL; 521 pNodeItr = listNext(&pMulticastFabricDesc->gpuInfoList, pNodeItr)) 522 { 523 if (pNodeItr->pGpu == pGpu) 524 { 525 NV_PRINTF(LEVEL_ERROR, "GPU %x has already attached\n", 526 pGpu->gpuInstance); 527 return NV_ERR_INSERT_DUPLICATE_NAME; 528 } 529 } 530 531 pNode = listAppendNew(&pMulticastFabricDesc->gpuInfoList); 532 if (pNode == NULL) 533 return NV_ERR_NO_MEMORY; 534 535 pNode->pGpuOsInfo = pParams->secInfo.gpuOsInfo; 536 pNode->pGpu = GPU_RES_GET_GPU(pSubdevice); 537 pNode->pAttachMemInfoTree = NULL; 538 pNode->bMcflaAlloc = NV_FALSE; 539 540 return NV_OK; 541 } 542 543 static void 544 _memMulticastFabricGpuInfoRemove 545 ( 546 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 547 ) 548 { 549 MEM_MULTICAST_FABRIC_GPU_INFO *pNode = NULL; 550 THREAD_STATE_NODE *pThreadNode = NULL; 551 THREAD_STATE_FREE_CALLBACK freeCallback; 552 553 NV_ASSERT_OK(threadStateGetCurrent(&pThreadNode, NULL)); 554 555 while ((pNode = listHead(&pMulticastFabricDesc->gpuInfoList)) != NULL) 556 { 557 // 558 // Enqueue unimport event before the callback to release GPU. 559 // This ordering is important. 560 // 561 if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)) 562 { 563 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 564 NV00F1_CTRL_FABRIC_EVENT unimportEvent; 565 566 _memMulticastFabricInitUnimportEvent(pNode->attachEventId, 567 pMulticastFabricDesc->exportNodeId, 568 &unimportEvent); 569 570 NV_CHECK(LEVEL_WARNING, 571 fabricPostEventsV2(pFabric, &unimportEvent, 1) == NV_OK); 572 } 573 574 freeCallback.pCb = osReleaseGpuOsInfo; 575 freeCallback.pCbData = (void *)pNode->pGpuOsInfo; 576 577 NV_ASSERT_OK(threadStateEnqueueCallbackOnFree(pThreadNode, 578 &freeCallback)); 579 listRemove(&pMulticastFabricDesc->gpuInfoList, pNode); 580 } 581 582 pMulticastFabricDesc->localAttachedGpusMask = 0; 583 pMulticastFabricDesc->numAttachedGpus = 0; 584 } 585 586 NV_STATUS 587 _memMulticastFabricSendInbandTeamSetupRequestV2 588 ( 589 OBJGPU *pGpu, 590 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 591 ) 592 { 593 NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams; 594 nvlink_inband_mc_team_setup_req_v2_msg_t *pMcTeamSetupReqMsg = NULL; 595 nvlink_inband_mc_team_setup_req_v2_t *pMcTeamSetupReq = NULL; 596 MEM_MULTICAST_FABRIC_GPU_INFO *pNode; 597 NvU32 idx = 0; 598 NvU32 payloadSize; 599 NvU32 sendDataSize; 600 NV_STATUS status = NV_OK; 601 NvU16 numKeys = 1; 602 MemMulticastFabricRemoteGpuInfoMapSupermapIter smIter; 603 604 sendDataParams = 605 (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *) 606 portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS)); 607 608 if (sendDataParams == NULL) 609 return NV_ERR_NO_MEMORY; 610 611 pMcTeamSetupReqMsg = 612 (nvlink_inband_mc_team_setup_req_v2_msg_t *)&sendDataParams->buffer[0]; 613 614 pMcTeamSetupReq = 615 (nvlink_inband_mc_team_setup_req_v2_t *)&pMcTeamSetupReqMsg->mcTeamSetupReq; 616 617 // 618 // Submap of remoteGpuInfoMap represent a node. As there is a key/node, 619 // count submap to calculate numKeys. 620 // 621 numKeys += multimapCountSubmaps(&pMulticastFabricDesc->remoteGpuInfoMap); 622 623 payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_setup_req_v2_t) + 624 (sizeof(pMcTeamSetupReq->gpuHandlesAndKeys[0]) * 625 (pMulticastFabricDesc->numMaxGpus + numKeys))); 626 627 sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize); 628 629 if ((NvU32)sendDataSize > sizeof(sendDataParams->buffer)) 630 { 631 NV_ASSERT(0); 632 status = NV_ERR_INSUFFICIENT_RESOURCES; 633 goto done; 634 } 635 636 portMemSet(sendDataParams, 0, sendDataSize); 637 638 pMcTeamSetupReq->mcAllocSize = pMulticastFabricDesc->allocSize; 639 pMcTeamSetupReq->numGpuHandles = pMulticastFabricDesc->numMaxGpus; 640 pMcTeamSetupReq->numKeys = numKeys; 641 642 // Fill local GPUs 643 for (pNode = listHead(&pMulticastFabricDesc->gpuInfoList); 644 pNode != NULL; 645 pNode = listNext(&pMulticastFabricDesc->gpuInfoList, pNode)) 646 pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pNode->gpuProbeHandle; 647 648 // Fill remote GPUs probe handles per submap (i.e per node) 649 smIter = multimapSubmapIterAll(&pMulticastFabricDesc->remoteGpuInfoMap); 650 651 while (multimapSubmapIterNext(&smIter)) 652 { 653 MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = smIter.pValue; 654 MemMulticastFabricRemoteGpuInfoMapIter iter; 655 656 iter = multimapSubmapIterItems(&pMulticastFabricDesc->remoteGpuInfoMap, 657 pSubmap); 658 659 while (multimapItemIterNext(&iter)) 660 { 661 // Item key is the GPU probe handle 662 pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = 663 multimapItemKey(&pMulticastFabricDesc->remoteGpuInfoMap, 664 iter.pValue); 665 } 666 } 667 668 // Fill local key 669 pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pMulticastFabricDesc->inbandReqId; 670 671 // Fill remote keys per submap (i.e per node) 672 smIter = multimapSubmapIterAll(&pMulticastFabricDesc->remoteGpuInfoMap); 673 674 while (multimapSubmapIterNext(&smIter)) 675 { 676 MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = smIter.pValue; 677 MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteNode = NULL; 678 MemMulticastFabricRemoteGpuInfoMapIter iter; 679 680 iter = multimapSubmapIterItems(&pMulticastFabricDesc->remoteGpuInfoMap, 681 pSubmap); 682 683 if (multimapItemIterNext(&iter)) 684 { 685 pRemoteNode = iter.pValue; 686 pMcTeamSetupReq->gpuHandlesAndKeys[idx++] = pRemoteNode->key; 687 } 688 } 689 690 if (idx != (pMcTeamSetupReq->numGpuHandles + numKeys)) 691 { 692 NV_ASSERT(0); 693 status = NV_ERR_INVALID_STATE; 694 goto done; 695 } 696 697 sendDataParams->dataSize = sendDataSize; 698 699 nvlinkInitInbandMsgHdr(&pMcTeamSetupReqMsg->msgHdr, 700 NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ_V2, 701 payloadSize, pMulticastFabricDesc->inbandReqId); 702 703 status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu), 704 sendDataParams); 705 if (status != NV_OK) 706 goto done; 707 708 pMulticastFabricDesc->bInbandReqInProgress = NV_TRUE; 709 710 done: 711 portMemFree(sendDataParams); 712 713 return status; 714 } 715 716 NV_STATUS 717 _memMulticastFabricSendInbandTeamSetupRequestV1 718 ( 719 OBJGPU *pGpu, 720 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 721 ) 722 { 723 NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams; 724 nvlink_inband_mc_team_setup_req_msg_t *pMcTeamSetupReqMsg = NULL; 725 nvlink_inband_mc_team_setup_req_t *pMcTeamSetupReq = NULL; 726 MEM_MULTICAST_FABRIC_GPU_INFO *pNode; 727 NvU32 idx = 0; 728 NvU32 payloadSize; 729 NvU32 sendDataSize; 730 NV_STATUS status = NV_OK; 731 732 sendDataParams = \ 733 (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *) 734 portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS)); 735 736 if (sendDataParams == NULL) 737 return NV_ERR_NO_MEMORY; 738 739 pMcTeamSetupReqMsg = \ 740 (nvlink_inband_mc_team_setup_req_msg_t *)&sendDataParams->buffer[0]; 741 742 pMcTeamSetupReq = \ 743 (nvlink_inband_mc_team_setup_req_t *)&pMcTeamSetupReqMsg->mcTeamSetupReq; 744 745 payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_setup_req_t) + \ 746 (sizeof(pMcTeamSetupReq->gpuHandles[0]) * \ 747 pMulticastFabricDesc->numMaxGpus)); 748 749 sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize); 750 751 NV_ASSERT((NvU32)sendDataSize <= sizeof(sendDataParams->buffer)); 752 753 portMemSet(sendDataParams, 0, sendDataSize); 754 755 pMcTeamSetupReq->mcAllocSize = pMulticastFabricDesc->allocSize; 756 pMcTeamSetupReq->numGpuHandles = pMulticastFabricDesc->numMaxGpus; 757 758 for (pNode = listHead(&pMulticastFabricDesc->gpuInfoList); 759 pNode != NULL; 760 pNode = listNext(&pMulticastFabricDesc->gpuInfoList, pNode)) 761 pMcTeamSetupReq->gpuHandles[idx++] = pNode->gpuProbeHandle; 762 763 NV_ASSERT(idx == pMcTeamSetupReq->numGpuHandles); 764 765 sendDataParams->dataSize = sendDataSize; 766 767 nvlinkInitInbandMsgHdr(&pMcTeamSetupReqMsg->msgHdr, 768 NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ, 769 payloadSize, pMulticastFabricDesc->inbandReqId); 770 771 status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu), 772 sendDataParams); 773 if (status != NV_OK) 774 goto done; 775 776 pMulticastFabricDesc->bInbandReqInProgress = NV_TRUE; 777 778 done: 779 portMemFree(sendDataParams); 780 781 return status; 782 } 783 784 NV_STATUS 785 _memMulticastFabricSendInbandTeamReleaseRequestV1 786 ( 787 OBJGPU *pGpu, 788 NvU64 mcTeamHandle 789 ) 790 { 791 NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *sendDataParams; 792 nvlink_inband_mc_team_release_req_msg_t *pMcTeamReleaseReqMsg = NULL; 793 nvlink_inband_mc_team_release_req_t *pMcTeamReleaseReq = NULL; 794 NvU32 payloadSize; 795 NvU32 sendDataSize; 796 NV_STATUS status; 797 798 sendDataParams = (NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS *) 799 portMemAllocNonPaged(sizeof(NV2080_CTRL_NVLINK_INBAND_SEND_DATA_PARAMS)); 800 801 if (sendDataParams == NULL) 802 return NV_ERR_NO_MEMORY; 803 804 pMcTeamReleaseReqMsg = (nvlink_inband_mc_team_release_req_msg_t *) \ 805 &sendDataParams->buffer[0]; 806 807 pMcTeamReleaseReq = (nvlink_inband_mc_team_release_req_t *) \ 808 &pMcTeamReleaseReqMsg->mcTeamReleaseReq; 809 810 payloadSize = (NvU32)(sizeof(nvlink_inband_mc_team_release_req_t)); 811 812 sendDataSize = (NvU32)(sizeof(nvlink_inband_msg_header_t) + payloadSize); 813 814 portMemSet(sendDataParams, 0, sendDataSize); 815 816 pMcTeamReleaseReq->mcTeamHandle = mcTeamHandle; 817 818 nvlinkInitInbandMsgHdr(&pMcTeamReleaseReqMsg->msgHdr, 819 NVLINK_INBAND_MSG_TYPE_MC_TEAM_RELEASE_REQ, 820 payloadSize, osGetTimestamp()); 821 822 sendDataParams->dataSize = sendDataSize; 823 824 status = knvlinkSendInbandData(pGpu, GPU_GET_KERNEL_NVLINK(pGpu), 825 sendDataParams); 826 827 portMemFree(sendDataParams); 828 829 return status; 830 } 831 832 NV_STATUS 833 _memMulticastFabricSendInbandTeamSetupRequest 834 ( 835 OBJGPU *pGpu, 836 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 837 ) 838 { 839 NvU64 fmCaps; 840 NV_STATUS status = NV_OK; 841 842 status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel, 843 &fmCaps); 844 if (status != NV_OK) 845 return status; 846 847 if (fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V2) 848 { 849 return _memMulticastFabricSendInbandTeamSetupRequestV2(pGpu, 850 pMulticastFabricDesc); 851 } 852 else if (fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V1) 853 { 854 return _memMulticastFabricSendInbandTeamSetupRequestV1(pGpu, 855 pMulticastFabricDesc); 856 } 857 else 858 { 859 return NV_ERR_NOT_SUPPORTED; 860 } 861 } 862 863 NV_STATUS 864 _memMulticastFabricSendInbandTeamReleaseRequest 865 ( 866 OBJGPU *pGpu, 867 NvU64 mcTeamHandle 868 ) 869 { 870 NvU64 fmCaps; 871 NV_STATUS status = NV_OK; 872 873 // If mcTeamHandle is zero, nothing to do... 874 if (mcTeamHandle == 0) 875 return NV_OK; 876 877 status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel, 878 &fmCaps); 879 if (status != NV_OK) 880 return status; 881 882 if (!(fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_RELEASE_V1)) 883 return NV_ERR_NOT_SUPPORTED; 884 885 return _memMulticastFabricSendInbandTeamReleaseRequestV1(pGpu, 886 mcTeamHandle); 887 } 888 889 static NV_STATUS 890 _memMulticastFabricSendInbandRequest 891 ( 892 OBJGPU *pGpu, 893 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 894 MEM_MULTICAST_FABRIC_REQUEST_TYPE requestType 895 ) 896 { 897 NV_STATUS status = NV_OK; 898 NvU32 gpuMask; 899 900 if (pGpu == NULL) 901 return NV_ERR_NOT_SUPPORTED; 902 903 gpuMask = NVBIT(gpuGetInstance(pGpu)); 904 if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask)) 905 { 906 NV_ASSERT(0); 907 return NV_ERR_INVALID_STATE; 908 } 909 910 switch (requestType) 911 { 912 case MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST: 913 status = _memMulticastFabricSendInbandTeamSetupRequest(pGpu, 914 pMulticastFabricDesc); 915 break; 916 case MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST: 917 status = _memMulticastFabricSendInbandTeamReleaseRequest(pGpu, 918 pMulticastFabricDesc->mcTeamHandle); 919 break; 920 default: 921 status = NV_ERR_NOT_SUPPORTED; 922 break; 923 } 924 925 return status; 926 } 927 928 // 929 // This function may be called with RO pMulticastFabricDesc->pLock. Don't modify 930 // pMulticastFabricDesc. 931 // 932 static void 933 _memorymulticastfabricDetachMem 934 ( 935 FABRIC_VASPACE *pFabricVAS, 936 MEMORY_DESCRIPTOR *pFabricMemDesc, 937 NODE *pMemNode 938 ) 939 { 940 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 941 MEMORY_DESCRIPTOR *pPhysMemDesc; 942 MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *pAttachMemInfoNode; 943 944 pAttachMemInfoNode = 945 (MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *)pMemNode->Data; 946 pPhysMemDesc = pAttachMemInfoNode->pPhysMemDesc; 947 948 fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, 949 pMemNode->keyStart, 950 pPhysMemDesc, 951 pAttachMemInfoNode->physMapLength); 952 953 NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient, 954 pAttachMemInfoNode->hDupedPhysMem)); 955 } 956 957 static void 958 _memorymulticastfabricBatchDetachMem 959 ( 960 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 961 ) 962 { 963 MEMORY_DESCRIPTOR *pFabricMemDesc; 964 MEM_MULTICAST_FABRIC_GPU_INFO *pGpuNode; 965 NODE *pMemNode; 966 FABRIC_VASPACE *pFabricVAS; 967 NvU32 gpuMask; 968 OBJGPU *pGpu; 969 970 pFabricMemDesc = pMulticastFabricDesc->pMemDesc; 971 NV_ASSERT_OR_RETURN_VOID(pFabricMemDesc != NULL); 972 973 for (pGpuNode = listHead(&pMulticastFabricDesc->gpuInfoList); 974 pGpuNode != NULL; 975 pGpuNode = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuNode)) 976 { 977 pGpu = pGpuNode->pGpu; 978 gpuMask = NVBIT(gpuGetInstance(pGpu)); 979 980 NV_ASSERT(rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask)); 981 982 pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE); 983 if (pFabricVAS == NULL) 984 { 985 NV_ASSERT(0); 986 continue; 987 } 988 989 btreeEnumStart(0, &pMemNode, pGpuNode->pAttachMemInfoTree); 990 while (pMemNode != NULL) 991 { 992 _memorymulticastfabricDetachMem(pFabricVAS, pFabricMemDesc, 993 pMemNode); 994 995 btreeUnlink(pMemNode, &pGpuNode->pAttachMemInfoTree); 996 portMemFree(pMemNode->Data); 997 998 btreeEnumStart(0, &pMemNode, pGpuNode->pAttachMemInfoTree); 999 } 1000 1001 // Everything is detached during object cleanup, free MCFLA now. 1002 if (pGpuNode->bMcflaAlloc) 1003 { 1004 fabricvaspaceBatchFree(pFabricVAS, &pFabricMemDesc->_pteArray[0], 1005 1, 1); 1006 pGpuNode->bMcflaAlloc = NV_FALSE; 1007 } 1008 } 1009 } 1010 1011 static void 1012 _memMulticastFabricDescriptorFree 1013 ( 1014 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc 1015 ) 1016 { 1017 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 1018 NvU32 allocFlags; 1019 NvU32 gpuMask; 1020 NvBool bGpuLockTaken = NV_FALSE; 1021 1022 if (pMulticastFabricDesc == NULL) 1023 return; 1024 1025 // 1026 // Take pMulticastFabricModuleLock to synchronize with 1027 // memorymulticastfabricTeamSetupResponseCallback() and 1028 // _memMulticastFabricDescriptorAllocUsingExpPacket(). 1029 // We don't want to delete pMulticastFabricDesc under them. 1030 // 1031 portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock); 1032 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 1033 1034 pMulticastFabricDesc->refCount--; 1035 1036 if (pMulticastFabricDesc->refCount != 0) 1037 { 1038 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1039 portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock); 1040 1041 return; 1042 } 1043 1044 allocFlags = pMulticastFabricDesc->allocFlags; 1045 1046 // 1047 // Empty caches so new calls to 1048 // memorymulticastfabricTeamSetupResponseCallback() and 1049 // _memMulticastFabricDescriptorAllocUsingExpPacket() fail. 1050 // 1051 if (!_memMulticastFabricIsPrime(allocFlags)) 1052 { 1053 NV_ASSERT(pMulticastFabricDesc->cacheKey != 0); 1054 fabricImportCacheDelete(pFabric, &pMulticastFabricDesc->expUuid, 1055 pMulticastFabricDesc->cacheKey); 1056 } 1057 1058 fabricMulticastSetupCacheDelete(pFabric, 1059 pMulticastFabricDesc->inbandReqId); 1060 1061 // Now I am the only one holding the pMulticastFabricDesc, drop the locks. 1062 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1063 portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock); 1064 1065 // 1066 // TODO: Make this a per-GPU lock in future. For now both dup() and free() 1067 // forces all GPU lock. 1068 // 1069 1070 if (!rmGpuLockIsOwner()) 1071 { 1072 gpuMask = GPUS_LOCK_ALL; 1073 1074 if (rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE, 1075 RM_LOCK_MODULES_MEM, &gpuMask) == NV_OK) 1076 { 1077 bGpuLockTaken = NV_TRUE; 1078 } 1079 else 1080 { 1081 NV_ASSERT(0); 1082 } 1083 } 1084 1085 if (pMulticastFabricDesc->pMemDesc != NULL) 1086 { 1087 MEM_MULTICAST_FABRIC_GPU_INFO *pNode = 1088 listHead(&pMulticastFabricDesc->gpuInfoList); 1089 1090 NV_ASSERT(pMulticastFabricDesc->bMemdescInstalled); 1091 NV_ASSERT(pNode != NULL); 1092 1093 _memorymulticastfabricBatchDetachMem(pMulticastFabricDesc); 1094 1095 if (_memMulticastFabricIsPrime(allocFlags)) 1096 { 1097 _memMulticastFabricSendInbandRequest(pNode->pGpu, 1098 pMulticastFabricDesc, 1099 MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST); 1100 } 1101 } 1102 1103 if (pMulticastFabricDesc->bInbandReqInProgress) 1104 { 1105 OS_WAIT_QUEUE *pWq; 1106 THREAD_STATE_NODE *pThreadNode = NULL; 1107 THREAD_STATE_FREE_CALLBACK freeCallback; 1108 1109 NV_ASSERT_OK(osAllocWaitQueue(&pWq)); 1110 1111 if (pWq != NULL) 1112 { 1113 NV_ASSERT_OK(fabricMulticastCleanupCacheInsert(pFabric, 1114 pMulticastFabricDesc->inbandReqId, 1115 pWq)); 1116 1117 NV_ASSERT_OK(threadStateGetCurrent(&pThreadNode, NULL)); 1118 1119 freeCallback.pCb = fabricMulticastWaitOnTeamCleanupCallback; 1120 freeCallback.pCbData = (void *)pMulticastFabricDesc->inbandReqId; 1121 1122 NV_ASSERT_OK(threadStateEnqueueCallbackOnFree(pThreadNode, 1123 &freeCallback)); 1124 } 1125 } 1126 1127 _memMulticastFabricGpuInfoRemove(pMulticastFabricDesc); 1128 1129 if (bGpuLockTaken) 1130 { 1131 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 1132 bGpuLockTaken = NV_FALSE; 1133 } 1134 1135 _memMulticastFabricDescriptorCleanup(pMulticastFabricDesc); 1136 } 1137 1138 static 1139 MEM_MULTICAST_FABRIC_DESCRIPTOR* 1140 _memMulticastFabricDescriptorAllocUsingExpPacket 1141 ( 1142 MemoryMulticastFabric *pMemoryMulticastFabric, 1143 NV00FD_ALLOCATION_PARAMETERS *pAllocParams 1144 ) 1145 { 1146 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 1147 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 1148 NvU64 cacheKey; 1149 NvUuid expUuid; 1150 NV_EXPORT_MEM_PACKET *pExportPacket = &pAllocParams->expPacket; 1151 NV_STATUS status = NV_OK; 1152 1153 ct_assert(NV_MEM_EXPORT_UUID_LEN == NV_UUID_LEN); 1154 portMemCopy(expUuid.uuid, NV_UUID_LEN, pExportPacket->uuid, 1155 NV_MEM_EXPORT_UUID_LEN); 1156 1157 // 1158 // To reuse import cache with the UCFLA class (0xf9), we create 1159 // unique import cache keys for MCFLA using NV00F9_IMPORT_ID_MAX, which 1160 // are never used by UCFLA. 1161 // 1162 cacheKey = 1163 (NV00F9_IMPORT_ID_MAX << NV00F9_IMPORT_ID_SHIFT) | pAllocParams->index; 1164 1165 // 1166 // Take pMulticastFabricModuleLock to synchronize multiple constructors 1167 // to share the cached pMulticastFabricDesc. Without this lock, two or 1168 // more constructors may see the import cache empty at the same time. 1169 // 1170 // We want the following sequence happen atomically: 1171 // 1172 // pMulticastFabricDesc = fabricImportCacheGet(..); 1173 // if (pMulticastFabricDesc == NULL) 1174 // { 1175 // pMulticastFabricDesc = alloc(); 1176 // fabricImportCacheInsert(pMulticastFabricDesc); 1177 // ... 1178 // } 1179 // 1180 // Also, it is important to synchronize the constructor with the destructor. 1181 // The constructor looks up and refcounts pMulticastFabricDesc 1182 // non-atomically. Thus, pMulticastFabricDesc may be destroyed before 1183 // it could be refcounted. 1184 // 1185 // pMulticastFabricDesc = fabricImportCacheGet(..); 1186 // if (pMulticastFabricDesc != NULL) 1187 // { 1188 // pMulticastFabricDesc->lock(); 1189 // pMulticastFabricDesc->refCount++; 1190 // ... 1191 // } 1192 // 1193 1194 portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock); 1195 1196 pMulticastFabricDesc = fabricImportCacheGet(pFabric, &expUuid, cacheKey); 1197 if (pMulticastFabricDesc != NULL) 1198 { 1199 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 1200 1201 NV_ASSERT(!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)); 1202 1203 pMulticastFabricDesc->refCount++; 1204 1205 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1206 1207 goto done; 1208 } 1209 1210 pMulticastFabricDesc = 1211 _memMulticastFabricDescriptorAlloc(pMemoryMulticastFabric, 1212 pAllocParams); 1213 1214 if (pMulticastFabricDesc == NULL) 1215 { 1216 status = NV_ERR_NO_MEMORY; 1217 goto done; 1218 } 1219 1220 // 1221 // For the imported object, set numMaxGpus to NV_U32_MAX for two reasons. 1222 // a. It doesn't track how many GPUs the team should have. That's tracked 1223 // by the exporter. 1224 // b. This object should never send MCFLA team setup request to the FM. 1225 // Setting numMaxGpus to NV_U32_MAX, makes sure that implicitly. 1226 // 1227 pMulticastFabricDesc->numMaxGpus = NV_U32_MAX; 1228 1229 // 1230 // allocSize is set to zero. GFM will provide that in the team setup 1231 // response. 1232 // 1233 pMulticastFabricDesc->allocSize = 0; 1234 1235 // 1236 // For now only pageSize support is 512MB. This needs to be revisited 1237 // in case we support more pagesizes. 1238 // 1239 pMulticastFabricDesc->pageSize = NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M; 1240 pMulticastFabricDesc->alignment = NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M; 1241 1242 pMulticastFabricDesc->exportNodeId = memoryExportGetNodeId(pExportPacket); 1243 pMulticastFabricDesc->expUuid = expUuid; 1244 pMulticastFabricDesc->cacheKey = cacheKey; 1245 pMulticastFabricDesc->index = pAllocParams->index; 1246 1247 // insert into cache once ready... 1248 status = fabricImportCacheInsert(pFabric, &expUuid, cacheKey, 1249 pMulticastFabricDesc); 1250 1251 done: 1252 portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock); 1253 1254 if (status != NV_OK) 1255 { 1256 _memMulticastFabricDescriptorFree(pMulticastFabricDesc); 1257 pMulticastFabricDesc = NULL; 1258 } 1259 1260 return pMulticastFabricDesc; 1261 } 1262 1263 NV_STATUS 1264 _memMulticastFabricConstruct 1265 ( 1266 MemoryMulticastFabric *pMemoryMulticastFabric, 1267 CALL_CONTEXT *pCallContext, 1268 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 1269 ) 1270 { 1271 Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory); 1272 NV00FD_ALLOCATION_PARAMETERS *pAllocParams = pParams->pAllocParams; 1273 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 1274 NV_STATUS status = NV_OK; 1275 1276 if (!_memMulticastFabricIsPrime(pAllocParams->allocFlags)) 1277 { 1278 NV_EXPORT_MEM_PACKET *pExportPacket = &pAllocParams->expPacket; 1279 1280 // If this a single-node UUID? 1281 if (memoryExportGetNodeId(pExportPacket) == NV_FABRIC_INVALID_NODE_ID) 1282 return NV_ERR_NOT_SUPPORTED; 1283 1284 pMulticastFabricDesc = 1285 _memMulticastFabricDescriptorAllocUsingExpPacket( 1286 pMemoryMulticastFabric, 1287 pAllocParams); 1288 } 1289 else 1290 { 1291 pMulticastFabricDesc = 1292 _memMulticastFabricDescriptorAlloc(pMemoryMulticastFabric, 1293 pAllocParams); 1294 } 1295 1296 if (pMulticastFabricDesc == NULL) 1297 return NV_ERR_NO_MEMORY; 1298 1299 // 1300 // At this point the pMulticastFabricDesc can be fresh or in-use, but will 1301 // be refcounted for safe access. If in-use, it can get modified before 1302 // pMulticastFabricDesc->pLock() is acquired. 1303 // 1304 1305 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 1306 1307 status = _memMulticastFabricDescriptorEnqueueWait(pParams->hClient, 1308 pMulticastFabricDesc, 1309 pAllocParams->pOsEvent, 1310 pMemory); 1311 if (status != NV_OK) 1312 goto fail; 1313 1314 pMemoryMulticastFabric->pMulticastFabricDesc = pMulticastFabricDesc; 1315 1316 pMemoryMulticastFabric->expNodeId = pMulticastFabricDesc->exportNodeId; 1317 pMemoryMulticastFabric->bImported = 1318 !_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags); 1319 1320 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1321 1322 return NV_OK; 1323 1324 fail: 1325 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1326 1327 _memMulticastFabricDescriptorFree(pMulticastFabricDesc); 1328 1329 return status; 1330 } 1331 1332 NV_STATUS 1333 _memMulticastFabricCreateMemDesc 1334 ( 1335 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 1336 NvU64 mcAddressBase, 1337 MEMORY_DESCRIPTOR **ppMemDesc 1338 ) 1339 { 1340 NV_STATUS status; 1341 MEMORY_DESCRIPTOR *pTempMemDesc = NULL; 1342 1343 status = memdescCreate(&pTempMemDesc, NULL, 1344 pMulticastFabricDesc->allocSize, 1345 0, NV_TRUE, ADDR_FABRIC_MC, 1346 NV_MEMORY_UNCACHED, 1347 MEMDESC_FLAGS_NONE); 1348 if (status != NV_OK) 1349 { 1350 NV_PRINTF(LEVEL_ERROR, 1351 "Failed to allocate memory descriptor for multicast object\n"); 1352 return status; 1353 } 1354 1355 memdescSetPte(pTempMemDesc, AT_GPU, 0, mcAddressBase); 1356 1357 memdescSetPageSize(pTempMemDesc, AT_GPU, 1358 pMulticastFabricDesc->pageSize); 1359 1360 pTempMemDesc->_pteKind = NV_MMU_PTE_KIND_SMSKED_MESSAGE; 1361 1362 memdescSetFlag(pTempMemDesc, MEMDESC_FLAGS_SET_KIND, NV_TRUE); 1363 1364 memdescSetGpuCacheAttrib(pTempMemDesc, NV_MEMORY_UNCACHED); 1365 1366 *ppMemDesc = pTempMemDesc; 1367 1368 return NV_OK; 1369 } 1370 1371 void 1372 _memMulticastFabricInstallMemDesc 1373 ( 1374 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 1375 MEMORY_DESCRIPTOR *pMemDesc, 1376 NvU64 mcTeamHandle, 1377 NV_STATUS status 1378 ) 1379 { 1380 NV_ASSERT(pMulticastFabricDesc->pMemDesc == NULL); 1381 1382 pMulticastFabricDesc->pMemDesc = pMemDesc; 1383 pMulticastFabricDesc->bMemdescInstalled = NV_TRUE; 1384 pMulticastFabricDesc->mcTeamHandle = mcTeamHandle; 1385 pMulticastFabricDesc->mcTeamStatus = status; 1386 1387 _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc); 1388 } 1389 1390 static NV_STATUS 1391 _memorymulticastFabricAllocVas 1392 ( 1393 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 1394 MEMORY_DESCRIPTOR *pFabricMemDesc 1395 ) 1396 { 1397 NV_STATUS status = NV_OK; 1398 FABRIC_VASPACE *pFabricVAS; 1399 MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo; 1400 VAS_ALLOC_FLAGS flags = { 0 }; 1401 NvU64 gpuProbeHandle; 1402 1403 for (pGpuInfo = listHead(&pMulticastFabricDesc->gpuInfoList); 1404 pGpuInfo != NULL; 1405 pGpuInfo = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuInfo)) 1406 { 1407 OBJGPU *pGpu = pGpuInfo->pGpu; 1408 NvU32 gpuMask = NVBIT(gpuGetInstance(pGpu)); 1409 1410 if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask)) 1411 { 1412 NV_ASSERT(0); 1413 status = NV_ERR_INVALID_STATE; 1414 goto cleanup; 1415 } 1416 1417 pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE); 1418 if (pFabricVAS == NULL) 1419 { 1420 status = NV_ERR_INVALID_STATE; 1421 goto cleanup; 1422 } 1423 1424 // 1425 // The fabric handle might not be available or have changed, if fabric 1426 // state was ever invalidated while MCFLA allocation was in progress. 1427 // 1428 status = gpuFabricProbeGetGpuFabricHandle(pGpu->pGpuFabricProbeInfoKernel, 1429 &gpuProbeHandle); 1430 if ((status != NV_OK) || (pGpuInfo->gpuProbeHandle != gpuProbeHandle)) 1431 { 1432 NV_PRINTF(LEVEL_ERROR, "Attached GPU's probe handle is stale\n"); 1433 status = NV_ERR_INVALID_DEVICE; 1434 goto cleanup; 1435 } 1436 1437 status = fabricvaspaceAllocMulticast(pFabricVAS, 1438 memdescGetPageSize(pFabricMemDesc, AT_GPU), 1439 pMulticastFabricDesc->alignment, 1440 flags, pFabricMemDesc->_pteArray[0], 1441 pMulticastFabricDesc->allocSize); 1442 if (status != NV_OK) 1443 { 1444 NV_PRINTF(LEVEL_ERROR, 1445 "Fabric VA space alloc failed for GPU %d\n", 1446 pGpuInfo->pGpu->gpuInstance); 1447 goto cleanup; 1448 } 1449 1450 pGpuInfo->bMcflaAlloc = NV_TRUE; 1451 } 1452 1453 return NV_OK; 1454 1455 cleanup: 1456 for (pGpuInfo = listHead(&pMulticastFabricDesc->gpuInfoList); 1457 pGpuInfo != NULL; 1458 pGpuInfo = listNext(&pMulticastFabricDesc->gpuInfoList, pGpuInfo)) 1459 { 1460 if (pGpuInfo->bMcflaAlloc) 1461 { 1462 pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE); 1463 1464 fabricvaspaceBatchFree(pFabricVAS, &pFabricMemDesc->_pteArray[0], 1465 1, 1); 1466 1467 pGpuInfo->bMcflaAlloc = NV_FALSE; 1468 } 1469 } 1470 1471 return status; 1472 } 1473 1474 static void 1475 _memMulticastFabricAttachGpuPostProcessor 1476 ( 1477 OBJGPU *pGpu, 1478 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc, 1479 NV_STATUS mcTeamStatus, 1480 NvU64 mcTeamHandle, 1481 NvU64 mcAddressBase, 1482 NvU64 mcAddressSize 1483 ) 1484 { 1485 NV_STATUS status = mcTeamStatus; 1486 MEMORY_DESCRIPTOR *pMemDesc = NULL; 1487 1488 // 1489 // FM is never expected to return NV_ERR_NOT_READY 1490 // as part of the inband response. 1491 // 1492 NV_ASSERT(mcTeamStatus != NV_ERR_NOT_READY); 1493 1494 // 1495 // There is a failure case on multinode systems, where a buggy client 1496 // may cause an error the prime object (e.g. attaching a GPU even after all 1497 // the required GPUs were attached) after MCFLA team request is sent to the 1498 // GFM. This should never happen under normal case, but in case it happens, 1499 // fail loudly. 1500 // 1501 // Initial value is not NV_ERR_NOT_READY, means a failure was observed. 1502 if (pMulticastFabricDesc->mcTeamStatus != NV_ERR_NOT_READY) 1503 { 1504 status = pMulticastFabricDesc->mcTeamStatus; 1505 NV_ASSERT((status != NV_OK)); 1506 goto installMemDesc; 1507 } 1508 1509 if (mcTeamStatus != NV_OK) 1510 goto installMemDesc; 1511 1512 if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)) 1513 pMulticastFabricDesc->allocSize = mcAddressSize; 1514 1515 if (mcAddressSize < pMulticastFabricDesc->allocSize) 1516 { 1517 NV_PRINTF(LEVEL_ERROR, 1518 "Insufficient mcAddressSize returned from Fabric Manager\n"); 1519 status = NV_ERR_INSUFFICIENT_RESOURCES; 1520 goto installMemDesc; 1521 } 1522 1523 if (!NV_IS_ALIGNED64(mcAddressBase, 1524 NV_MEMORY_MULTICAST_FABRIC_PAGE_SIZE_512M)) 1525 { 1526 NV_PRINTF(LEVEL_ERROR, 1527 "Insufficient mcAddressSize returned from Fabric Manager\n"); 1528 status = NV_ERR_INVALID_ADDRESS; 1529 goto installMemDesc; 1530 } 1531 1532 status = _memMulticastFabricCreateMemDesc(pMulticastFabricDesc, 1533 mcAddressBase, &pMemDesc); 1534 if (status != NV_OK) 1535 { 1536 NV_PRINTF(LEVEL_ERROR, "Failed to allocate fabric memdesc\n"); 1537 goto installMemDesc; 1538 } 1539 1540 status = _memorymulticastFabricAllocVas(pMulticastFabricDesc, pMemDesc); 1541 if (status != NV_OK) 1542 { 1543 NV_PRINTF(LEVEL_ERROR, "Failed to allocate fabric VAS\n"); 1544 memdescDestroy(pMemDesc); 1545 pMemDesc = NULL; 1546 goto installMemDesc; 1547 } 1548 1549 installMemDesc: 1550 _memMulticastFabricInstallMemDesc(pMulticastFabricDesc, pMemDesc, 1551 mcTeamHandle, status); 1552 1553 if ((status != NV_OK) && (mcTeamStatus == NV_OK)) 1554 _memMulticastFabricSendInbandRequest(pGpu, pMulticastFabricDesc, 1555 MEM_MULTICAST_FABRIC_TEAM_RELEASE_REQUEST); 1556 } 1557 1558 NV_STATUS 1559 memorymulticastfabricTeamSetupResponseCallback 1560 ( 1561 NvU32 gpuInstance, 1562 NvU64 *pNotifyGfidMask, 1563 NV2080_CTRL_NVLINK_INBAND_RECEIVED_DATA_PARAMS *pInbandRcvParams 1564 ) 1565 { 1566 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 1567 nvlink_inband_mc_team_setup_rsp_msg_t *pMcTeamSetupRspMsg; 1568 nvlink_inband_mc_team_setup_rsp_t *pMcTeamSetupRsp; 1569 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 1570 NvU64 requestId; 1571 NV_STATUS mcTeamStatus; 1572 NvU64 mcTeamHandle = 0; 1573 NvU64 mcAddressBase = 0; 1574 NvU64 mcAddressSize = 0; 1575 OBJGPU *pGpu; 1576 1577 NV_ASSERT(pInbandRcvParams != NULL); 1578 NV_ASSERT(rmGpuLockIsOwner()); 1579 1580 if ((pGpu = gpumgrGetGpu(gpuInstance)) == NULL) 1581 { 1582 NV_ASSERT_FAILED("Invalid GPU instance"); 1583 return NV_ERR_INVALID_ARGUMENT; 1584 } 1585 1586 pMcTeamSetupRspMsg = 1587 (nvlink_inband_mc_team_setup_rsp_msg_t *)&pInbandRcvParams->data[0]; 1588 1589 pMcTeamSetupRsp = 1590 (nvlink_inband_mc_team_setup_rsp_t *)&pMcTeamSetupRspMsg->mcTeamSetupRsp; 1591 1592 requestId = pMcTeamSetupRspMsg->msgHdr.requestId; 1593 1594 mcTeamStatus = pMcTeamSetupRspMsg->msgHdr.status; 1595 1596 if (mcTeamStatus == NV_OK) 1597 { 1598 mcTeamHandle = pMcTeamSetupRsp->mcTeamHandle; 1599 mcAddressBase = pMcTeamSetupRsp->mcAddressBase; 1600 mcAddressSize = pMcTeamSetupRsp->mcAddressSize; 1601 1602 #if defined(DEBUG) || defined(DEVELOP) 1603 { 1604 // Make sure that the reserved fields are initialized to 0 1605 NvU8 *pRsvd = &pMcTeamSetupRsp->reserved[0]; 1606 1607 NV_ASSERT((pRsvd[0] == 0) && portMemCmp(pRsvd, pRsvd + 1, 1608 (sizeof(pMcTeamSetupRsp->reserved) - 1)) == 0); 1609 } 1610 #endif 1611 } 1612 1613 // 1614 // Acquire pMulticastFabricModuleLock here, to make sure 1615 // pMulticastFabricDesc is not removed underneath us. 1616 // The thread doesn't hold refcount on pMulticastFabricDesc. 1617 // 1618 portSyncRwLockAcquireWrite(pFabric->pMulticastFabricModuleLock); 1619 1620 pMulticastFabricDesc = fabricMulticastSetupCacheGet(pFabric, requestId); 1621 1622 if (pMulticastFabricDesc != NULL) 1623 { 1624 fabricMulticastSetupCacheDelete(pFabric, requestId); 1625 1626 // 1627 // We have now safely acquired pMulticastFabricDesc->lock, which 1628 // should block the destructor from removing pMulticastFabricDesc 1629 // under us even if pMulticastFabricModuleLock is dropped. 1630 // 1631 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 1632 1633 // 1634 // Drop pMulticastFabricModuleLock here as out of order, now we 1635 // shouldn't need it ever. 1636 // 1637 portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock); 1638 1639 pMulticastFabricDesc->bInbandReqInProgress = NV_FALSE; 1640 1641 _memMulticastFabricAttachGpuPostProcessor(pGpu, 1642 pMulticastFabricDesc, 1643 mcTeamStatus, 1644 mcTeamHandle, 1645 mcAddressBase, 1646 mcAddressSize); 1647 1648 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1649 } 1650 else 1651 { 1652 // 1653 // As pMulticastFabricDesc is not found, drop the lock right 1654 // away. GPU locks are already taken to submit the team release 1655 // request. 1656 // 1657 portSyncRwLockReleaseWrite(pFabric->pMulticastFabricModuleLock); 1658 1659 if (mcTeamStatus == NV_OK) 1660 (void)_memMulticastFabricSendInbandTeamReleaseRequest(pGpu, 1661 mcTeamHandle); 1662 1663 // 1664 // Check if there is any thread waiting for team release and 1665 // wake it up. 1666 // 1667 // The multicast fabric descriptor could have undergone the 1668 // destruct sequence while an inband team setup request was in 1669 // progress with FM. 1670 // 1671 // In such a scenario the last thread to free the multicast 1672 // descriptor is put to sleep until the team setup response 1673 // is received and a subsequent team release request is sent. 1674 // 1675 fabricMulticastCleanupCacheInvokeCallback(pFabric, requestId, 1676 fabricWakeUpThreadCallback); 1677 } 1678 1679 return NV_OK; 1680 } 1681 1682 NV_STATUS 1683 memorymulticastfabricConstruct_IMPL 1684 ( 1685 MemoryMulticastFabric *pMemoryMulticastFabric, 1686 CALL_CONTEXT *pCallContext, 1687 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 1688 ) 1689 { 1690 NV00FD_ALLOCATION_PARAMETERS *pAllocParams = pParams->pAllocParams; 1691 1692 if (RS_IS_COPY_CTOR(pParams)) 1693 { 1694 return memorymulticastfabricCopyConstruct(pMemoryMulticastFabric, 1695 pCallContext, pParams); 1696 } 1697 1698 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 1699 _memMulticastFabricValidateAllocParams(pAllocParams)); 1700 1701 return _memMulticastFabricConstruct(pMemoryMulticastFabric, 1702 pCallContext, pParams); 1703 } 1704 1705 // 1706 // Note this function is not always called with the GPU lock. Be careful 1707 // while accessing GPU state. 1708 // 1709 static NV_STATUS 1710 _memorymulticastfabricCtrlAttachGpu 1711 ( 1712 OBJGPU *pGpu, 1713 MemoryMulticastFabric *pMemoryMulticastFabric, 1714 NV00FD_CTRL_ATTACH_GPU_PARAMS *pParams 1715 ) 1716 { 1717 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 1718 pMemoryMulticastFabric->pMulticastFabricDesc; 1719 NV_STATUS status; 1720 FABRIC_VASPACE *pFabricVAS; 1721 MEM_MULTICAST_FABRIC_GPU_INFO *pNode; 1722 MEM_MULTICAST_FABRIC_GPU_INFO *pHead; 1723 MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteHead; 1724 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 1725 1726 if (pParams->flags != 0) 1727 { 1728 NV_PRINTF(LEVEL_ERROR, "flags passed for attach mem must be zero\n"); 1729 return NV_ERR_INVALID_ARGUMENT; 1730 } 1731 1732 // Check if the Multicast FLA object has any additional slots for GPUs 1733 if (pMulticastFabricDesc->numAttachedGpus == 1734 pMulticastFabricDesc->numMaxGpus) 1735 { 1736 NV_PRINTF(LEVEL_ERROR, "Max no. of GPUs have already attached!\n"); 1737 return NV_ERR_INVALID_OPERATION; 1738 } 1739 1740 if (pMulticastFabricDesc->bMemdescInstalled) 1741 { 1742 NV_PRINTF(LEVEL_ERROR, "The object is already ready.\n"); 1743 return NV_ERR_STATE_IN_USE; 1744 } 1745 1746 if (RMCFG_FEATURE_PLATFORM_WINDOWS || 1747 gpuIsCCFeatureEnabled(pGpu) || 1748 IS_VIRTUAL(pGpu)) 1749 { 1750 NV_PRINTF(LEVEL_ERROR, 1751 "Multicast attach not supported on Windows/CC/vGPU modes\n"); 1752 return NV_ERR_NOT_SUPPORTED; 1753 } 1754 1755 status = _memMulticastFabricGpuInfoAdd(pMemoryMulticastFabric, 1756 pCallContext->pControlParams); 1757 if (status != NV_OK) 1758 { 1759 NV_PRINTF(LEVEL_ERROR, "Failed to populate GPU info\n"); 1760 return status; 1761 } 1762 1763 pNode = listTail(&pMulticastFabricDesc->gpuInfoList); 1764 1765 status = gpuFabricProbeGetGpuFabricHandle(pGpu->pGpuFabricProbeInfoKernel, 1766 &pNode->gpuProbeHandle); 1767 if (status != NV_OK) 1768 { 1769 NV_PRINTF(LEVEL_ERROR, 1770 "Attaching GPU does not have a valid probe handle\n"); 1771 goto fail; 1772 } 1773 1774 status = gpuFabricProbeGetFabricCliqueId(pGpu->pGpuFabricProbeInfoKernel, 1775 &pNode->cliqueId); 1776 if (status != NV_OK) 1777 { 1778 NV_PRINTF(LEVEL_ERROR, 1779 "Attaching GPU does not have a valid clique ID\n"); 1780 goto fail; 1781 } 1782 1783 pHead = listHead(&pMulticastFabricDesc->gpuInfoList); 1784 pRemoteHead = multimapFirstItem(&pMulticastFabricDesc->remoteGpuInfoMap); 1785 1786 if ((pHead != NULL) && (pHead->cliqueId != pNode->cliqueId)) 1787 { 1788 NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n"); 1789 status = NV_ERR_INVALID_DEVICE; 1790 goto fail; 1791 } 1792 1793 if ((pRemoteHead != NULL) && (pRemoteHead->cliqueId != pNode->cliqueId)) 1794 { 1795 NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n"); 1796 status = NV_ERR_INVALID_DEVICE; 1797 goto fail; 1798 } 1799 1800 pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE); 1801 if (pFabricVAS == NULL) 1802 { 1803 NV_PRINTF(LEVEL_ERROR, 1804 "Fabric vaspace object not available for GPU %x\n", 1805 pGpu->gpuInstance); 1806 status = NV_ERR_NOT_SUPPORTED; 1807 goto fail; 1808 } 1809 1810 if ((pMulticastFabricDesc->numAttachedGpus + 1) 1811 == pMulticastFabricDesc->numMaxGpus) 1812 { 1813 NV_ASSERT(_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)); 1814 1815 status = _memMulticastFabricSendInbandRequest(pGpu, 1816 pMulticastFabricDesc, 1817 MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST); 1818 if (status != NV_OK) 1819 { 1820 NV_PRINTF(LEVEL_ERROR, 1821 "Inband request Multicast Team Setup failed!\n"); 1822 goto fail; 1823 } 1824 } 1825 // Invoke remote GPU attach event.. 1826 else if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)) 1827 { 1828 NvU64 fmCaps; 1829 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 1830 NV00F1_CTRL_FABRIC_EVENT event; 1831 1832 status = gpuFabricProbeGetfmCaps(pGpu->pGpuFabricProbeInfoKernel, 1833 &fmCaps); 1834 if (status != NV_OK) 1835 { 1836 NV_PRINTF(LEVEL_ERROR, "Failed to query IMEX FM caps\n"); 1837 goto fail; 1838 } 1839 1840 if (!(fmCaps & NVLINK_INBAND_FM_CAPS_MC_TEAM_SETUP_V2)) 1841 { 1842 NV_PRINTF(LEVEL_ERROR, "Remote attach is supported from V2+\n"); 1843 status = NV_ERR_NOT_SUPPORTED; 1844 goto fail; 1845 } 1846 1847 _memMulticastFabricInitAttachEvent(pNode->gpuProbeHandle, 1848 pMulticastFabricDesc->inbandReqId, 1849 pNode->cliqueId, 1850 pMulticastFabricDesc->exportNodeId, 1851 pMulticastFabricDesc->index, 1852 &pMulticastFabricDesc->expUuid, 1853 &event); 1854 1855 pNode->attachEventId = event.id; 1856 1857 status = fabricPostEventsV2(pFabric, &event, 1); 1858 if (status != NV_OK) 1859 { 1860 NV_PRINTF(LEVEL_ERROR, 1861 "Failed to notify IMEX daemon of import event\n"); 1862 goto fail; 1863 } 1864 } 1865 1866 // 1867 // DO NOT FAIL after the call to fabricPostEventsV2 or 1868 // _memMulticastFabricSendInbandRequest(). 1869 // 1870 // These functions are used to communicate with the external 1871 // entities like FM/IMEX daemons. It is recommended that we 1872 // do not fail the control call after these to avoid 1873 // complicated cleanups. 1874 // 1875 1876 pMulticastFabricDesc->numAttachedGpus++; 1877 pMulticastFabricDesc->localAttachedGpusMask |= NVBIT32(pGpu->gpuInstance); 1878 1879 return NV_OK; 1880 1881 fail: 1882 // Remove GPU OS info added in the prologue. 1883 listRemove(&pMulticastFabricDesc->gpuInfoList, pNode); 1884 1885 return status; 1886 } 1887 1888 NV_STATUS 1889 memorymulticastfabricCtrlAttachGpu_IMPL 1890 ( 1891 MemoryMulticastFabric *pMemoryMulticastFabric, 1892 NV00FD_CTRL_ATTACH_GPU_PARAMS *pParams 1893 ) 1894 { 1895 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 1896 pMemoryMulticastFabric->pMulticastFabricDesc; 1897 NV_STATUS status; 1898 NvBool bLastAttach; 1899 NvBool bLastAttachRecheck; 1900 NvBool bGpuLockTaken = NV_FALSE; 1901 Subdevice *pSubdevice; 1902 OBJGPU *pGpu; 1903 NvU32 gpuMask; 1904 1905 status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric), 1906 pParams->hSubdevice, &pSubdevice); 1907 if (status == NV_ERR_OBJECT_NOT_FOUND) 1908 status = NV_ERR_INVALID_DEVICE; 1909 1910 if (status != NV_OK) 1911 return status; 1912 1913 pGpu = GPU_RES_GET_GPU(pSubdevice); 1914 gpuMask = NVBIT(gpuGetInstance(pGpu)); 1915 1916 retry: 1917 // 1918 // Find if I am the last attach. If yes, take a GPU lock to submit 1919 // inband request. Otherwise, skip taking GPU lock. 1920 // 1921 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 1922 1923 bLastAttach = ((pMulticastFabricDesc->numMaxGpus - 1924 pMulticastFabricDesc->numAttachedGpus) == 1); 1925 1926 // Drop the lock to avoid lock inversion with the GPU lock... 1927 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 1928 1929 if (bLastAttach && (gpuMask != 0)) 1930 { 1931 status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, 1932 GPUS_LOCK_FLAGS_NONE, 1933 RM_LOCK_MODULES_MEM, &gpuMask); 1934 if (status != NV_OK) 1935 { 1936 NV_ASSERT(0); 1937 return status; 1938 } 1939 1940 bGpuLockTaken = NV_TRUE; 1941 } 1942 1943 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 1944 1945 // 1946 // Recheck to avoid TOCTOU, make sure bLastAttach state didn't change.. 1947 // 1948 // This is a deterministic check as once the GPU is attached to 1949 // this object it can't be detached unless the object is destroyed. 1950 // 1951 bLastAttachRecheck = ((pMulticastFabricDesc->numMaxGpus - 1952 pMulticastFabricDesc->numAttachedGpus) == 1); 1953 1954 if (bLastAttachRecheck != bLastAttach) 1955 { 1956 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1957 1958 if (bGpuLockTaken) 1959 { 1960 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 1961 bGpuLockTaken = NV_FALSE; 1962 } 1963 1964 goto retry; 1965 } 1966 1967 status = _memorymulticastfabricCtrlAttachGpu(pGpu, pMemoryMulticastFabric, 1968 pParams); 1969 1970 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 1971 1972 if (bGpuLockTaken) 1973 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 1974 1975 return status; 1976 } 1977 1978 #ifdef NV00FD_CTRL_CMD_SET_FAILURE 1979 static NV_STATUS 1980 _memorymulticastfabricCtrlSetFailure 1981 ( 1982 MemoryMulticastFabric *pMemoryMulticastFabric, 1983 NV00FD_CTRL_SET_FAILURE_PARAMS *pParams 1984 ) 1985 { 1986 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 1987 NvHandle hClient = pCallContext->pClient->hClient; 1988 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 1989 pMemoryMulticastFabric->pMulticastFabricDesc; 1990 1991 if (!rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) && 1992 !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel)) 1993 return NV_ERR_INSUFFICIENT_PERMISSIONS; 1994 1995 if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)) 1996 { 1997 NV_PRINTF(LEVEL_ERROR, "Only supported on prime MCLFA object\n"); 1998 return NV_ERR_NOT_SUPPORTED; 1999 } 2000 2001 if ((pParams->status == NV_OK) || (pParams->status == NV_ERR_NOT_READY)) 2002 return NV_ERR_INVALID_ARGUMENT; 2003 2004 pMulticastFabricDesc->mcTeamStatus = pParams->status; 2005 2006 // Notify the callers about the failure. 2007 _memMulticastFabricDescriptorFlushClients(pMulticastFabricDesc); 2008 2009 return NV_OK; 2010 } 2011 2012 NV_STATUS memorymulticastfabricCtrlSetFailure_IMPL 2013 ( 2014 MemoryMulticastFabric *pMemoryMulticastFabric, 2015 NV00FD_CTRL_SET_FAILURE_PARAMS *pParams 2016 ) 2017 { 2018 NV_STATUS status; 2019 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2020 pMemoryMulticastFabric->pMulticastFabricDesc; 2021 2022 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 2023 2024 status = _memorymulticastfabricCtrlSetFailure(pMemoryMulticastFabric, 2025 pParams); 2026 2027 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2028 2029 return status; 2030 } 2031 #endif 2032 2033 #ifdef NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU 2034 // 2035 // Note this function is not always called with the GPU lock. Be careful 2036 // while accessing GPU state. 2037 // 2038 static NV_STATUS 2039 _memorymulticastfabricCtrlAttachRemoteGpu 2040 ( 2041 OBJGPU *pGpu, 2042 MemoryMulticastFabric *pMemoryMulticastFabric, 2043 NV00FD_CTRL_ATTACH_REMOTE_GPU_PARAMS *pParams 2044 ) 2045 { 2046 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2047 pMemoryMulticastFabric->pMulticastFabricDesc; 2048 MEM_MULTICAST_FABRIC_GPU_INFO *pHead = 2049 listHead(&pMulticastFabricDesc->gpuInfoList); 2050 MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pRemoteHead = 2051 multimapFirstItem(&pMulticastFabricDesc->remoteGpuInfoMap); 2052 MEM_MULTICAST_FABRIC_REMOTE_GPU_INFO *pNode; 2053 MEM_MULTICAST_FABRIC_GPU_INFO *pIter; 2054 MemMulticastFabricRemoteGpuInfoMapSubmap *pSubmap = NULL; 2055 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 2056 NvHandle hClient = pCallContext->pClient->hClient; 2057 NV00FD_CTRL_SET_FAILURE_PARAMS params; 2058 NV_STATUS status; 2059 2060 if (!rmclientIsCapableByHandle(hClient, NV_RM_CAP_SYS_FABRIC_IMEX_MGMT) && 2061 !rmclientIsAdminByHandle(hClient, pCallContext->secInfo.privLevel)) 2062 { 2063 // Don't set failure as non-priv client can hit this failure too. 2064 return NV_ERR_INSUFFICIENT_PERMISSIONS; 2065 } 2066 2067 if (!_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags)) 2068 { 2069 NV_PRINTF(LEVEL_ERROR, "Only supported on prime MCLFA object\n"); 2070 status = NV_ERR_NOT_SUPPORTED; 2071 goto fail; 2072 } 2073 2074 // Check if the Multicast FLA object has any additional slots for GPUs 2075 if (pMulticastFabricDesc->numAttachedGpus == 2076 pMulticastFabricDesc->numMaxGpus) 2077 { 2078 NV_PRINTF(LEVEL_ERROR, "Max no. of GPUs have already attached!\n"); 2079 status = NV_ERR_INVALID_OPERATION; 2080 goto fail; 2081 } 2082 2083 if ((pHead != NULL) && (pHead->cliqueId != pParams->cliqueId)) 2084 { 2085 NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n"); 2086 status = NV_ERR_INVALID_DEVICE; 2087 goto fail; 2088 } 2089 2090 if ((pRemoteHead != NULL) && (pRemoteHead->cliqueId != pParams->cliqueId)) 2091 { 2092 NV_PRINTF(LEVEL_ERROR, "Clique ID mismatch\n"); 2093 status = NV_ERR_INVALID_DEVICE; 2094 goto fail; 2095 } 2096 2097 if (pParams->nodeId == NV_FABRIC_INVALID_NODE_ID) 2098 { 2099 NV_PRINTF(LEVEL_ERROR, "Invalid node ID\n"); 2100 status = NV_ERR_INVALID_ARGUMENT; 2101 goto fail; 2102 } 2103 2104 for (pIter = listHead(&pMulticastFabricDesc->gpuInfoList); 2105 pIter != NULL; 2106 pIter = listNext(&pMulticastFabricDesc->gpuInfoList, pIter)) 2107 { 2108 if (pIter->gpuProbeHandle == pParams->gpuFabricProbeHandle) 2109 { 2110 NV_PRINTF(LEVEL_ERROR, "GPU is already attached\n"); 2111 status = NV_ERR_INSERT_DUPLICATE_NAME; 2112 goto fail; 2113 } 2114 } 2115 2116 if (multimapFindSubmap(&pMulticastFabricDesc->remoteGpuInfoMap, 2117 pParams->nodeId) == NULL) 2118 { 2119 pSubmap = multimapInsertSubmap(&pMulticastFabricDesc->remoteGpuInfoMap, 2120 pParams->nodeId); 2121 if (pSubmap == NULL) 2122 { 2123 status = NV_ERR_NO_MEMORY; 2124 goto fail; 2125 } 2126 } 2127 2128 pNode = multimapInsertItemNew(&pMulticastFabricDesc->remoteGpuInfoMap, 2129 pParams->nodeId, 2130 pParams->gpuFabricProbeHandle); 2131 if (pNode == NULL) 2132 { 2133 // This could also fail due to NO_MEMORY error.. 2134 NV_PRINTF(LEVEL_ERROR, "Failed to track remote GPU info\n"); 2135 status = NV_ERR_INSERT_DUPLICATE_NAME; 2136 goto cleanup_submap; 2137 } 2138 2139 portMemSet(pNode, 0, sizeof(*pNode)); 2140 2141 pNode->key = pParams->key; 2142 pNode->cliqueId = pParams->cliqueId; 2143 2144 if ((pMulticastFabricDesc->numAttachedGpus + 1) 2145 == pMulticastFabricDesc->numMaxGpus) 2146 { 2147 status = _memMulticastFabricSendInbandRequest(pGpu, 2148 pMulticastFabricDesc, 2149 MEM_MULTICAST_FABRIC_TEAM_SETUP_REQUEST); 2150 if (status != NV_OK) 2151 { 2152 NV_PRINTF(LEVEL_ERROR, 2153 "Inband request Multicast Team Setup failed!\n"); 2154 goto unlink_node; 2155 } 2156 } 2157 2158 // 2159 // DO NOT FAIL after the call to _memMulticastFabricSendInbandRequest(). 2160 // 2161 // This function is used to communicate with the external 2162 // entities like FM daemon. It is recommended that we do not 2163 // fail the control call after this to avoid complicated state 2164 // cleanups. 2165 // 2166 2167 pMulticastFabricDesc->numAttachedGpus++; 2168 2169 return NV_OK; 2170 2171 unlink_node: 2172 multimapRemoveItemByKey(&pMulticastFabricDesc->remoteGpuInfoMap, 2173 pParams->nodeId, pParams->gpuFabricProbeHandle); 2174 2175 cleanup_submap: 2176 if (pSubmap != NULL) 2177 multimapRemoveSubmap(&pMulticastFabricDesc->remoteGpuInfoMap, pSubmap); 2178 2179 fail: 2180 // 2181 // NV_ERR_NOT_READY means the object is not ready, which is the initial 2182 // state. Thus, in case any API returns NV_ERR_NOT_READY, overwrite it. 2183 // 2184 if (status == NV_ERR_NOT_READY) 2185 status = NV_ERR_GENERIC; 2186 2187 params.status = status; 2188 2189 NV_ASSERT_OK(_memorymulticastfabricCtrlSetFailure(pMemoryMulticastFabric, 2190 ¶ms)); 2191 2192 return status; 2193 } 2194 2195 NV_STATUS 2196 memorymulticastfabricCtrlAttachRemoteGpu_IMPL 2197 ( 2198 MemoryMulticastFabric *pMemoryMulticastFabric, 2199 NV00FD_CTRL_ATTACH_REMOTE_GPU_PARAMS *pParams 2200 ) 2201 { 2202 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2203 pMemoryMulticastFabric->pMulticastFabricDesc; 2204 NV_STATUS status; 2205 OBJGPU *pGpu; 2206 MEM_MULTICAST_FABRIC_GPU_INFO *pNode; 2207 NvU32 gpuMask; 2208 NvBool bLastAttach; 2209 NvBool bLastAttachRecheck; 2210 NvBool bGpuLockTaken = NV_FALSE; 2211 2212 retry: 2213 // 2214 // Find if I am the last attach. If yes, take a GPU lock to submit 2215 // inband request. Otherwise, skip taking GPU lock. 2216 // 2217 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2218 2219 bLastAttach = ((pMulticastFabricDesc->numMaxGpus - 2220 pMulticastFabricDesc->numAttachedGpus) == 1); 2221 2222 pNode = listHead(&pMulticastFabricDesc->gpuInfoList); 2223 pGpu = (pNode != NULL) ? pNode->pGpu : NULL; 2224 gpuMask = (pGpu != NULL) ? NVBIT(gpuGetInstance(pGpu)) : 0; 2225 2226 // Drop the lock to avoid lock inversion with the GPU lock... 2227 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2228 2229 if (bLastAttach && (gpuMask != 0)) 2230 { 2231 status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, 2232 GPUS_LOCK_FLAGS_NONE, 2233 RM_LOCK_MODULES_MEM, &gpuMask); 2234 if (status != NV_OK) 2235 { 2236 NV_ASSERT(0); 2237 return status; 2238 } 2239 2240 bGpuLockTaken = NV_TRUE; 2241 } 2242 2243 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 2244 2245 // 2246 // Recheck to avoid TOCTOU, make sure bLastAttach state didn't change.. 2247 // 2248 // This is a deterministic check as once the GPU is attached to 2249 // this object it can't be detached unless the object is destroyed. 2250 // 2251 bLastAttachRecheck = ((pMulticastFabricDesc->numMaxGpus - 2252 pMulticastFabricDesc->numAttachedGpus) == 1); 2253 2254 if (bLastAttachRecheck != bLastAttach) 2255 { 2256 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2257 2258 if (bGpuLockTaken) 2259 { 2260 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 2261 bGpuLockTaken = NV_FALSE; 2262 } 2263 2264 goto retry; 2265 } 2266 2267 status = _memorymulticastfabricCtrlAttachRemoteGpu(pGpu, 2268 pMemoryMulticastFabric, 2269 pParams); 2270 2271 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2272 2273 if (bGpuLockTaken) 2274 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 2275 2276 return status; 2277 } 2278 #endif 2279 2280 static MEM_MULTICAST_FABRIC_GPU_INFO* 2281 _memorymulticastfabricGetAttchedGpuInfo 2282 ( 2283 MemoryMulticastFabric *pMemoryMulticastFabric, 2284 Subdevice *pSubdevice 2285 ) 2286 { 2287 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2288 pMemoryMulticastFabric->pMulticastFabricDesc; 2289 MEM_MULTICAST_FABRIC_GPU_INFO *pNodeItr; 2290 2291 for (pNodeItr = listHead(&pMulticastFabricDesc->gpuInfoList); 2292 pNodeItr != NULL; 2293 pNodeItr = listNext(&pMulticastFabricDesc->gpuInfoList, pNodeItr)) 2294 { 2295 if (pNodeItr->pGpu == GPU_RES_GET_GPU(pSubdevice)) 2296 return pNodeItr; 2297 } 2298 2299 return NULL; 2300 } 2301 2302 // 2303 // This function is called with RO pMulticastFabricDesc->pLock. Don't modify 2304 // pMulticastFabricDesc. 2305 // 2306 static NV_STATUS 2307 _memorymulticastfabricCtrlDetachMem 2308 ( 2309 Subdevice *pSubdevice, 2310 MemoryMulticastFabric *pMemoryMulticastFabric, 2311 NV00FD_CTRL_DETACH_MEM_PARAMS *pParams 2312 ) 2313 { 2314 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2315 pMemoryMulticastFabric->pMulticastFabricDesc; 2316 MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo; 2317 NODE *pNode; 2318 MEMORY_DESCRIPTOR *pFabricMemDesc; 2319 FABRIC_VASPACE *pFabricVAS; 2320 NV_STATUS status; 2321 OBJGPU *pGpu; 2322 NvU32 gpuMask; 2323 2324 if (pParams->flags != 0) 2325 return NV_ERR_INVALID_ARGUMENT; 2326 2327 pGpuInfo = _memorymulticastfabricGetAttchedGpuInfo(pMemoryMulticastFabric, 2328 pSubdevice); 2329 if (pGpuInfo == NULL) 2330 return NV_ERR_INVALID_DEVICE; 2331 2332 pGpu = pGpuInfo->pGpu; 2333 gpuMask = NVBIT(gpuGetInstance(pGpu)); 2334 2335 if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask)) 2336 return NV_ERR_INVALID_LOCK_STATE; 2337 2338 status = btreeSearch(pParams->offset, &pNode, pGpuInfo->pAttachMemInfoTree); 2339 if (status != NV_OK) 2340 return status; 2341 2342 pFabricMemDesc = pMulticastFabricDesc->pMemDesc; 2343 NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_STATE); 2344 2345 pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE); 2346 NV_ASSERT_OR_RETURN(pFabricVAS != NULL, NV_ERR_INVALID_STATE); 2347 2348 _memorymulticastfabricDetachMem(pFabricVAS, pFabricMemDesc, pNode); 2349 2350 btreeUnlink(pNode, &pGpuInfo->pAttachMemInfoTree); 2351 portMemFree(pNode->Data); 2352 2353 return NV_OK; 2354 } 2355 2356 NV_STATUS 2357 memorymulticastfabricCtrlDetachMem_IMPL 2358 ( 2359 MemoryMulticastFabric *pMemoryMulticastFabric, 2360 NV00FD_CTRL_DETACH_MEM_PARAMS *pParams 2361 ) 2362 { 2363 NV_STATUS status; 2364 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2365 pMemoryMulticastFabric->pMulticastFabricDesc; 2366 Subdevice *pSubdevice; 2367 NvU32 gpuMask; 2368 2369 status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric), 2370 pParams->hSubdevice, &pSubdevice); 2371 if (status == NV_ERR_OBJECT_NOT_FOUND) 2372 status = NV_ERR_INVALID_DEVICE; 2373 2374 if (status != NV_OK) 2375 return status; 2376 2377 // 2378 // TODO: Make this a per-GPU lock in future. For now both dup() and free() 2379 // forces all GPU lock. 2380 // 2381 gpuMask = GPUS_LOCK_ALL; 2382 2383 // 2384 // Take per-GPU lock as we ensure source and destination devices are the 2385 // same. 2386 // 2387 status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE, 2388 RM_LOCK_MODULES_MEM, &gpuMask); 2389 if (status != NV_OK) 2390 { 2391 NV_ASSERT(0); 2392 return status; 2393 } 2394 2395 // 2396 // pMulticastFabricDesc->pLock read locking is sufficient as rest of the 2397 // state in being modified is per-GPU, which is protected by the GPU lock. 2398 // 2399 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2400 2401 status = _memorymulticastfabricCtrlDetachMem(pSubdevice, 2402 pMemoryMulticastFabric, 2403 pParams); 2404 2405 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2406 2407 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 2408 2409 return status; 2410 } 2411 2412 static NV_STATUS 2413 _memorymulticastfabricValidatePhysMem 2414 ( 2415 MemoryMulticastFabric *pMemoryMulticastFabric, 2416 NvHandle hPhysMem, 2417 OBJGPU *pAttachedGpu, 2418 MEMORY_DESCRIPTOR **ppPhysMemDesc 2419 ) 2420 { 2421 MEMORY_DESCRIPTOR *pPhysMemDesc; 2422 NvU64 physPageSize; 2423 NV_STATUS status; 2424 Memory *pMemory; 2425 2426 status = memGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric), 2427 hPhysMem, &pMemory); 2428 if (status != NV_OK) 2429 { 2430 NV_PRINTF(LEVEL_ERROR, "Invalid object handle passed\n"); 2431 return status; 2432 } 2433 2434 pPhysMemDesc = pMemory->pMemDesc; 2435 2436 if ((pAttachedGpu != pPhysMemDesc->pGpu) || 2437 !memmgrIsMemDescSupportedByFla_HAL(pAttachedGpu, 2438 GPU_GET_MEMORY_MANAGER(pAttachedGpu), 2439 pPhysMemDesc)) 2440 { 2441 NV_PRINTF(LEVEL_ERROR, "Invalid physmem handle passed\n"); 2442 2443 return NV_ERR_INVALID_ARGUMENT; 2444 } 2445 2446 physPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU); 2447 if ((physPageSize != RM_PAGE_SIZE_HUGE) && 2448 (physPageSize != RM_PAGE_SIZE_512M)) 2449 { 2450 NV_PRINTF(LEVEL_ERROR, "Physmem page size should be 2MB\n"); 2451 2452 return NV_ERR_INVALID_ARGUMENT; 2453 } 2454 2455 *ppPhysMemDesc = pPhysMemDesc; 2456 2457 return NV_OK; 2458 } 2459 2460 // 2461 // This function is called with RO pMulticastFabricDesc->pLock. Don't modify 2462 // pMulticastFabricDesc. 2463 // 2464 static NV_STATUS 2465 _memorymulticastfabricCtrlAttachMem 2466 ( 2467 Subdevice *pSubdevice, 2468 MemoryMulticastFabric *pMemoryMulticastFabric, 2469 NV00FD_CTRL_ATTACH_MEM_PARAMS *pParams 2470 ) 2471 { 2472 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2473 pMemoryMulticastFabric->pMulticastFabricDesc; 2474 MEM_MULTICAST_FABRIC_GPU_INFO *pGpuInfo; 2475 NV_STATUS status; 2476 MEMORY_DESCRIPTOR *pPhysMemDesc; 2477 MEMORY_DESCRIPTOR *pFabricMemDesc; 2478 NvHandle hDupedPhysMem = 0; 2479 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2480 FABRIC_VASPACE *pFabricVAS; 2481 MEM_MULTICAST_FABRIC_ATTACH_MEM_INFO_NODE *pNode; 2482 OBJGPU *pGpu; 2483 NvU32 gpuMask; 2484 2485 if (pParams->flags != 0) 2486 return NV_ERR_INVALID_ARGUMENT; 2487 2488 pGpuInfo = _memorymulticastfabricGetAttchedGpuInfo(pMemoryMulticastFabric, 2489 pSubdevice); 2490 if (pGpuInfo == NULL) 2491 return NV_ERR_INVALID_DEVICE; 2492 2493 pGpu = pGpuInfo->pGpu; 2494 gpuMask = NVBIT(gpuGetInstance(pGpu)); 2495 2496 if (!rmGpuGroupLockIsOwner(0, GPU_LOCK_GRP_MASK, &gpuMask)) 2497 return NV_ERR_INVALID_LOCK_STATE; 2498 2499 status = _memorymulticastfabricValidatePhysMem(pMemoryMulticastFabric, 2500 pParams->hMemory, 2501 pGpuInfo->pGpu, 2502 &pPhysMemDesc); 2503 if (status != NV_OK) 2504 { 2505 NV_PRINTF(LEVEL_ERROR, "Failed to validate physmem handle\n"); 2506 return status; 2507 } 2508 2509 pFabricVAS = dynamicCast(pGpuInfo->pGpu->pFabricVAS, FABRIC_VASPACE); 2510 NV_ASSERT_OR_RETURN(pFabricVAS != NULL, NV_ERR_INVALID_STATE); 2511 2512 pFabricMemDesc = pMulticastFabricDesc->pMemDesc; 2513 NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_STATE); 2514 2515 NV_ASSERT_OR_RETURN(pGpuInfo->bMcflaAlloc, NV_ERR_INVALID_STATE); 2516 2517 status = pRmApi->DupObject(pRmApi, pFabricVAS->hClient, 2518 pFabricVAS->hDevice, &hDupedPhysMem, 2519 RES_GET_CLIENT_HANDLE(pMemoryMulticastFabric), 2520 pParams->hMemory, 0); 2521 if (status != NV_OK) 2522 { 2523 NV_PRINTF(LEVEL_ERROR, "Failed to dup physmem handle\n"); 2524 return status; 2525 } 2526 2527 status = fabricvaspaceMapPhysMemdesc(pFabricVAS, 2528 pFabricMemDesc, 2529 pParams->offset, 2530 pPhysMemDesc, 2531 pParams->mapOffset, 2532 pParams->mapLength, 2533 0); 2534 if (status != NV_OK) 2535 { 2536 NV_PRINTF(LEVEL_ERROR, "Failed to map FLA\n"); 2537 goto freeDupedMem; 2538 } 2539 2540 pNode = portMemAllocNonPaged(sizeof(*pNode)); 2541 if (pNode == NULL) 2542 { 2543 status = NV_ERR_NO_MEMORY; 2544 goto unmapVas; 2545 } 2546 2547 portMemSet(pNode, 0, sizeof(*pNode)); 2548 2549 pNode->node.keyStart = pParams->offset; 2550 pNode->node.keyEnd = pParams->offset; 2551 pNode->physMapLength = pParams->mapLength; 2552 pNode->pPhysMemDesc = pPhysMemDesc; 2553 pNode->hDupedPhysMem = hDupedPhysMem; 2554 pNode->node.Data = pNode; 2555 2556 status = btreeInsert(&pNode->node, &pGpuInfo->pAttachMemInfoTree); 2557 if (status != NV_OK) 2558 { 2559 NV_PRINTF(LEVEL_ERROR, "Failed to track attach mem info\n"); 2560 goto freeNode; 2561 } 2562 2563 return NV_OK; 2564 2565 freeNode: 2566 portMemFree(pNode); 2567 2568 unmapVas: 2569 fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, pParams->offset, 2570 pPhysMemDesc, pParams->mapLength); 2571 2572 freeDupedMem: 2573 NV_ASSERT_OK(pRmApi->Free(pRmApi, pFabricVAS->hClient, hDupedPhysMem)); 2574 2575 return status; 2576 } 2577 2578 NV_STATUS 2579 memorymulticastfabricCtrlAttachMem_IMPL 2580 ( 2581 MemoryMulticastFabric *pMemoryMulticastFabric, 2582 NV00FD_CTRL_ATTACH_MEM_PARAMS *pParams 2583 ) 2584 { 2585 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2586 pMemoryMulticastFabric->pMulticastFabricDesc; 2587 NV_STATUS status; 2588 Subdevice *pSubdevice; 2589 NvU32 gpuMask; 2590 2591 status = subdeviceGetByHandle(RES_GET_CLIENT(pMemoryMulticastFabric), 2592 pParams->hSubdevice, &pSubdevice); 2593 if (status == NV_ERR_OBJECT_NOT_FOUND) 2594 status = NV_ERR_INVALID_DEVICE; 2595 2596 if (status != NV_OK) 2597 return status; 2598 2599 // 2600 // TODO: Make this a per-GPU lock in future. For now both dup() and free() 2601 // forces all GPU lock. 2602 // 2603 gpuMask = GPUS_LOCK_ALL; 2604 2605 // 2606 // Take per-GPU lock as we ensure source and destination devices are the 2607 // same. 2608 // 2609 status = rmGpuGroupLockAcquire(0, GPU_LOCK_GRP_MASK, GPUS_LOCK_FLAGS_NONE, 2610 RM_LOCK_MODULES_MEM, &gpuMask); 2611 if (status != NV_OK) 2612 { 2613 NV_ASSERT(0); 2614 return status; 2615 } 2616 2617 // 2618 // pMulticastFabricDesc->pLock read locking is sufficient as rest of the 2619 // state in being modified is per-GPU, which is protected by the GPU lock. 2620 // 2621 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2622 2623 status = _memorymulticastfabricCtrlAttachMem(pSubdevice, 2624 pMemoryMulticastFabric, 2625 pParams); 2626 2627 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2628 2629 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE); 2630 2631 return status; 2632 } 2633 2634 void 2635 memorymulticastfabricDestruct_IMPL 2636 ( 2637 MemoryMulticastFabric *pMemoryMulticastFabric 2638 ) 2639 { 2640 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2641 pMemoryMulticastFabric->pMulticastFabricDesc; 2642 Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory); 2643 2644 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 2645 2646 memDestructCommon(pMemory); 2647 2648 _memMulticastFabricDescriptorDequeueWait(pMulticastFabricDesc, pMemory); 2649 2650 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2651 2652 _memMulticastFabricDescriptorFree(pMulticastFabricDesc); 2653 } 2654 2655 NvBool 2656 memorymulticastfabricCanCopy_IMPL 2657 ( 2658 MemoryMulticastFabric *pMemoryMulticastFabric 2659 ) 2660 { 2661 return NV_TRUE; 2662 } 2663 2664 NV_STATUS 2665 memorymulticastfabricCopyConstruct_IMPL 2666 ( 2667 MemoryMulticastFabric *pMemoryMulticastFabric, 2668 CALL_CONTEXT *pCallContext, 2669 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 2670 ) 2671 { 2672 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 2673 2674 MemoryMulticastFabric *pSourceMemoryMulticastFabric = 2675 dynamicCast(pParams->pSrcRef->pResource, MemoryMulticastFabric); 2676 2677 pMulticastFabricDesc = pSourceMemoryMulticastFabric->pMulticastFabricDesc; 2678 2679 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 2680 2681 pMulticastFabricDesc->refCount++; 2682 2683 pMemoryMulticastFabric->pMulticastFabricDesc = pMulticastFabricDesc; 2684 2685 pMemoryMulticastFabric->expNodeId = pMulticastFabricDesc->exportNodeId; 2686 pMemoryMulticastFabric->bImported = 2687 !_memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags); 2688 2689 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2690 2691 return NV_OK; 2692 } 2693 2694 static NV_STATUS 2695 _memorymulticastfabricCtrlGetInfo 2696 ( 2697 MemoryMulticastFabric *pMemoryMulticastFabric, 2698 NV00FD_CTRL_GET_INFO_PARAMS *pParams 2699 ) 2700 { 2701 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 2702 2703 pMulticastFabricDesc = pMemoryMulticastFabric->pMulticastFabricDesc; 2704 2705 pParams->alignment = pMulticastFabricDesc->alignment; 2706 pParams->allocSize = pMulticastFabricDesc->allocSize; 2707 pParams->pageSize = pMulticastFabricDesc->pageSize; 2708 pParams->numMaxGpus = pMulticastFabricDesc->numMaxGpus; 2709 pParams->numAttachedGpus = pMulticastFabricDesc->numAttachedGpus; 2710 2711 return NV_OK; 2712 } 2713 2714 NV_STATUS 2715 memorymulticastfabricCtrlGetInfo_IMPL 2716 ( 2717 MemoryMulticastFabric *pMemoryMulticastFabric, 2718 NV00FD_CTRL_GET_INFO_PARAMS *pParams 2719 ) 2720 { 2721 NV_STATUS status = NV_OK; 2722 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc; 2723 2724 pMulticastFabricDesc = pMemoryMulticastFabric->pMulticastFabricDesc; 2725 2726 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2727 2728 status = _memorymulticastfabricCtrlGetInfo(pMemoryMulticastFabric, 2729 pParams); 2730 2731 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2732 2733 return status; 2734 } 2735 2736 NV_STATUS 2737 memorymulticastfabricIsReady_IMPL 2738 ( 2739 MemoryMulticastFabric *pMemoryMulticastFabric, 2740 NvBool bCopyConstructorContext 2741 ) 2742 { 2743 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2744 pMemoryMulticastFabric->pMulticastFabricDesc; 2745 Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory); 2746 NV_STATUS mcTeamStatus; 2747 2748 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2749 2750 mcTeamStatus = pMulticastFabricDesc->mcTeamStatus; 2751 2752 if (bCopyConstructorContext && (mcTeamStatus == NV_ERR_NOT_READY)) 2753 { 2754 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2755 return NV_OK; 2756 } 2757 2758 if (pMemory->pMemDesc != pMulticastFabricDesc->pMemDesc) 2759 { 2760 // This function only initializes pMemory so it should never fail. 2761 NV_ASSERT_OK(memConstructCommon(pMemory, 2762 NV_MEMORY_MULTICAST_FABRIC, 2763 0, pMulticastFabricDesc->pMemDesc, 2764 0, NULL, 0, 0, 0, 0, 2765 NVOS32_MEM_TAG_NONE, NULL)); 2766 } 2767 2768 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2769 2770 return mcTeamStatus; 2771 } 2772 2773 static NV_STATUS 2774 _memorymulticastfabricCtrlRegisterEvent 2775 ( 2776 MemoryMulticastFabric *pMemoryMulticastFabric, 2777 NV00FD_CTRL_REGISTER_EVENT_PARAMS *pParams 2778 ) 2779 { 2780 Memory *pMemory = staticCast(pMemoryMulticastFabric, Memory); 2781 NvHandle hClient = RES_GET_CLIENT_HANDLE(pMemoryMulticastFabric); 2782 2783 return _memMulticastFabricDescriptorEnqueueWait(hClient, 2784 pMemoryMulticastFabric->pMulticastFabricDesc, 2785 pParams->pOsEvent, pMemory); 2786 } 2787 2788 2789 NV_STATUS 2790 memorymulticastfabricCtrlRegisterEvent_IMPL 2791 ( 2792 MemoryMulticastFabric *pMemoryMulticastFabric, 2793 NV00FD_CTRL_REGISTER_EVENT_PARAMS *pParams 2794 ) 2795 { 2796 NV_STATUS status; 2797 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2798 pMemoryMulticastFabric->pMulticastFabricDesc; 2799 2800 portSyncRwLockAcquireWrite(pMulticastFabricDesc->pLock); 2801 2802 status = _memorymulticastfabricCtrlRegisterEvent(pMemoryMulticastFabric, 2803 pParams); 2804 2805 portSyncRwLockReleaseWrite(pMulticastFabricDesc->pLock); 2806 2807 return status; 2808 } 2809 2810 NV_STATUS 2811 memorymulticastfabricControl_IMPL 2812 ( 2813 MemoryMulticastFabric *pMemoryMulticastFabric, 2814 CALL_CONTEXT *pCallContext, 2815 RS_RES_CONTROL_PARAMS_INTERNAL *pParams 2816 ) 2817 { 2818 NV_STATUS status = NV_OK; 2819 2820 if ( 2821 #ifdef NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU 2822 (pParams->cmd != NV00FD_CTRL_CMD_ATTACH_REMOTE_GPU) && 2823 #endif 2824 #ifdef NV00FD_CTRL_CMD_SET_FAILURE 2825 (pParams->cmd != NV00FD_CTRL_CMD_SET_FAILURE) && 2826 #endif 2827 (pParams->cmd != NV00FD_CTRL_CMD_ATTACH_GPU)) 2828 { 2829 status = memorymulticastfabricIsReady(pMemoryMulticastFabric, 2830 NV_FALSE); 2831 } 2832 2833 // 2834 // If clients try to register when the multicast object 2835 // is ready, then there is nothing left to do as the memory 2836 // descriptor is already installed. 2837 // 2838 // If the status is NV_ERR_NOT_READY then we are yet to 2839 // receive the inband response and we register the event. 2840 // 2841 if (pParams->cmd == NV00FD_CTRL_CMD_REGISTER_EVENT) 2842 { 2843 if (status == NV_OK) 2844 return NV_WARN_NOTHING_TO_DO; 2845 2846 if (status != NV_ERR_NOT_READY) 2847 return status; 2848 } 2849 else 2850 { 2851 // Clients may busy-loop on this error status, don't log error. 2852 if (status == NV_ERR_NOT_READY) 2853 return status; 2854 2855 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status); 2856 } 2857 2858 // 2859 // Note: GPU lock(s) is required for some control calls. Thus, it is 2860 // incorrect to take the leaf lock here. resControl_IMPL() attempts to 2861 // acquire the GPU locks before it calls the control call body. 2862 // 2863 return resControl_IMPL(staticCast(pMemoryMulticastFabric, RsResource), 2864 pCallContext, pParams); 2865 } 2866 2867 NvBool 2868 memorymulticastfabricIsExportAllowed_IMPL 2869 ( 2870 MemoryMulticastFabric *pMemoryMulticastFabric 2871 ) 2872 { 2873 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2874 pMemoryMulticastFabric->pMulticastFabricDesc; 2875 NvBool bAllowed; 2876 2877 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2878 2879 bAllowed = _memMulticastFabricIsPrime(pMulticastFabricDesc->allocFlags); 2880 2881 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2882 2883 return bAllowed; 2884 } 2885 2886 NvBool 2887 memorymulticastfabricIsGpuMapAllowed_IMPL 2888 ( 2889 MemoryMulticastFabric *pMemoryMulticastFabric, 2890 OBJGPU *pGpu 2891 ) 2892 { 2893 NvU32 localAttachedGpusMask; 2894 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2895 pMemoryMulticastFabric->pMulticastFabricDesc; 2896 2897 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2898 2899 localAttachedGpusMask = pMulticastFabricDesc->localAttachedGpusMask; 2900 2901 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2902 2903 return ((localAttachedGpusMask & NVBIT32(pGpu->gpuInstance)) != 0U); 2904 } 2905 2906 NV_STATUS 2907 memorymulticastfabricGetMapAddrSpace_IMPL 2908 ( 2909 MemoryMulticastFabric *pMemoryMulticastFabric, 2910 CALL_CONTEXT *pCallContext, 2911 NvU32 mapFlags, 2912 NV_ADDRESS_SPACE *pAddrSpace 2913 ) 2914 { 2915 *pAddrSpace = ADDR_FABRIC_MC; 2916 return NV_OK; 2917 } 2918 2919 void 2920 memorymulticastfabricRemoveFromCache_IMPL 2921 ( 2922 MemoryMulticastFabric *pMemoryMulticastFabric 2923 ) 2924 { 2925 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 2926 MEM_MULTICAST_FABRIC_DESCRIPTOR *pMulticastFabricDesc = 2927 pMemoryMulticastFabric->pMulticastFabricDesc; 2928 2929 if (!pMemoryMulticastFabric->bImported) 2930 return; 2931 2932 portSyncRwLockAcquireRead(pMulticastFabricDesc->pLock); 2933 2934 fabricImportCacheDelete(pFabric, &pMulticastFabricDesc->expUuid, 2935 pMulticastFabricDesc->cacheKey); 2936 2937 portSyncRwLockReleaseRead(pMulticastFabricDesc->pLock); 2938 } 2939