1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @file 26 * @brief This file contains the functions managing the NVLink fabric 27 */ 28 29 #include "os/os.h" 30 #include "compute/fabric.h" 31 32 static NV_STATUS 33 _fabricCacheInsert 34 ( 35 FabricCache *pCache, 36 NvU64 key1, 37 NvU64 key2, 38 NvU64 key3, 39 void *pData 40 ) 41 { 42 FabricCacheSubmap *pInsertedSubmap = NULL; 43 FabricCacheEntry *pInsertedEntry = NULL; 44 FabricCacheEntry *pEntry; 45 FabricCacheMapEntry *pMapEntry; 46 47 pEntry = multimapFindItem(pCache, key1, key2); 48 if (pEntry != NULL) 49 goto insert; 50 51 if (multimapFindSubmap(pCache, key1) == NULL) 52 { 53 pInsertedSubmap = multimapInsertSubmap(pCache, key1); 54 if (pInsertedSubmap == NULL) 55 goto fail; 56 } 57 58 pInsertedEntry = multimapInsertItemNew(pCache, key1, key2); 59 if (pInsertedEntry == NULL) 60 goto fail; 61 62 mapInit(&pInsertedEntry->map, portMemAllocatorGetGlobalNonPaged()); 63 pEntry = pInsertedEntry; 64 65 insert: 66 pMapEntry = mapInsertNew(&pEntry->map, key3); 67 if (pMapEntry == NULL) 68 goto fail; 69 70 pMapEntry->pData = pData; 71 72 return NV_OK; 73 74 fail: 75 if (pInsertedEntry != NULL) 76 { 77 mapDestroy(&pInsertedEntry->map); 78 multimapRemoveItem(pCache, pInsertedEntry); 79 } 80 81 if (pInsertedSubmap != NULL) 82 multimapRemoveSubmap(pCache, pInsertedSubmap); 83 84 return NV_ERR_INVALID_STATE; 85 } 86 87 static void 88 _fabricCacheDelete 89 ( 90 FabricCache *pCache, 91 NvU64 key1, 92 NvU64 key2, 93 NvU64 key3 94 ) 95 { 96 FabricCacheSubmap *pSubmap; 97 FabricCacheEntry *pEntry; 98 99 pEntry = multimapFindItem(pCache, key1, key2); 100 if (pEntry == NULL) 101 return; 102 103 mapRemoveByKey(&pEntry->map, key3); 104 if (mapCount(&pEntry->map) > 0) 105 return; 106 107 mapDestroy(&pEntry->map); 108 multimapRemoveItem(pCache, pEntry); 109 110 pSubmap = multimapFindSubmap(pCache, key1); 111 NV_ASSERT_OR_RETURN_VOID(pSubmap != NULL); 112 113 if (multimapCountSubmapItems(pCache, pSubmap) > 0) 114 return; 115 116 multimapRemoveSubmap(pCache, pSubmap); 117 } 118 119 static FabricCacheMapEntry* 120 _fabricCacheFind 121 ( 122 FabricCache *pCache, 123 NvU64 key1, 124 NvU64 key2, 125 NvU64 key3 126 ) 127 { 128 FabricCacheEntry *pEntry; 129 FabricCacheMapEntry *pMapEntry; 130 131 pEntry = multimapFindItem(pCache, key1, key2); 132 if (pEntry == NULL) 133 return NULL; 134 135 pMapEntry = mapFind(&pEntry->map, key3); 136 if (pMapEntry == NULL) 137 return NULL; 138 139 return pMapEntry; 140 } 141 142 void 143 fabricMulticastFabricOpsMutexAcquire_IMPL 144 ( 145 Fabric *pFabric 146 ) 147 { 148 portSyncMutexAcquire(pFabric->pMulticastFabricOpsMutex); 149 } 150 151 void 152 fabricMulticastFabricOpsMutexRelease_IMPL 153 ( 154 Fabric *pFabric 155 ) 156 { 157 portSyncMutexRelease(pFabric->pMulticastFabricOpsMutex); 158 } 159 160 NV_STATUS 161 fabricMulticastSetupCacheInsertUnderLock_IMPL 162 ( 163 Fabric *pFabric, 164 NvU64 requestId, 165 void *pData 166 ) 167 { 168 return _fabricCacheInsert(&pFabric->fabricMulticastCache, 169 0, requestId, 0, pData); 170 } 171 172 void 173 fabricMulticastSetupCacheDeleteUnderLock_IMPL 174 ( 175 Fabric *pFabric, 176 NvU64 requestId 177 ) 178 { 179 _fabricCacheDelete(&pFabric->fabricMulticastCache, 180 0, requestId, 0); 181 } 182 183 void* 184 fabricMulticastSetupCacheGetUnderLock_IMPL 185 ( 186 Fabric *pFabric, 187 NvU64 requestId 188 ) 189 { 190 FabricCacheMapEntry *pMapEntry; 191 192 pMapEntry = _fabricCacheFind(&pFabric->fabricMulticastCache, 193 0, requestId, 0); 194 195 return (pMapEntry == NULL ? NULL : pMapEntry->pData); 196 } 197 198 NV_STATUS 199 fabricMulticastCleanupCacheInsertUnderLock_IMPL 200 ( 201 Fabric *pFabric, 202 NvU64 requestId, 203 void *pData 204 ) 205 { 206 return _fabricCacheInsert(&pFabric->fabricMulticastCache, 207 1, requestId, 0, pData); 208 } 209 210 void 211 fabricMulticastCleanupCacheDeleteUnderLock_IMPL 212 ( 213 Fabric *pFabric, 214 NvU64 requestId 215 ) 216 { 217 _fabricCacheDelete(&pFabric->fabricMulticastCache, 218 1, requestId, 0); 219 } 220 221 void* 222 fabricMulticastCleanupCacheGetUnderLock_IMPL 223 ( 224 Fabric *pFabric, 225 NvU64 requestId 226 ) 227 { 228 FabricCacheMapEntry *pMapEntry; 229 230 pMapEntry = _fabricCacheFind(&pFabric->fabricMulticastCache, 231 1, requestId, 0); 232 233 return (pMapEntry == NULL ? NULL : pMapEntry->pData); 234 } 235 236 void 237 fabricSetFmSessionFlags_IMPL 238 ( 239 Fabric *pFabric, 240 NvU32 flags 241 ) 242 { 243 pFabric->flags = flags; 244 } 245 246 NvU32 247 fabricGetFmSessionFlags_IMPL 248 ( 249 Fabric *pFabric 250 ) 251 { 252 return pFabric->flags; 253 } 254 255 NV_STATUS 256 fabricConstruct_IMPL 257 ( 258 Fabric *pFabric 259 ) 260 { 261 NV_STATUS status = NV_OK; 262 263 pFabric->pMulticastFabricOpsMutex = portSyncMutexCreate(portMemAllocatorGetGlobalNonPaged()); 264 if (pFabric->pMulticastFabricOpsMutex == NULL) 265 { 266 status = NV_ERR_NO_MEMORY; 267 goto fail; 268 } 269 270 multimapInit(&pFabric->fabricMulticastCache, portMemAllocatorGetGlobalNonPaged()); 271 272 return NV_OK; 273 274 //TODO: Remove the WAR to suppress unused label warning 275 goto fail; 276 fail: 277 fabricDestruct_IMPL(pFabric); 278 return status; 279 } 280 281 void 282 fabricDestruct_IMPL 283 ( 284 Fabric *pFabric 285 ) 286 { 287 NV_ASSERT(multimapCountItems(&pFabric->fabricMulticastCache) == 0); 288 289 multimapDestroy(&pFabric->fabricMulticastCache); 290 291 if (pFabric->pMulticastFabricOpsMutex != NULL) 292 portSyncMutexDestroy(pFabric->pMulticastFabricOpsMutex); 293 294 } 295 296 NV_STATUS 297 fabricInitInbandMsgHdr 298 ( 299 nvlink_inband_msg_header_t *pMsgHdr, 300 NvU32 type, 301 NvU32 len 302 ) 303 { 304 NV_STATUS status; 305 306 portMemSet(pMsgHdr, 0, sizeof(*pMsgHdr)); 307 308 status = osGetRandomBytes((NvU8 *)&pMsgHdr->requestId, 309 sizeof(pMsgHdr->requestId)); 310 if (status != NV_OK) 311 { 312 return status; 313 } 314 315 pMsgHdr->magicId = NVLINK_INBAND_MSG_MAGIC_ID_FM; 316 pMsgHdr->type = type; 317 pMsgHdr->length = len; 318 319 return NV_OK; 320 } 321 322 void 323 fabricMulticastWaitOnTeamCleanupCallback 324 ( 325 void *pCbData 326 ) 327 { 328 NvU64 inbandReqId = (NvU64)pCbData; 329 Fabric *pFabric = SYS_GET_FABRIC(SYS_GET_INSTANCE()); 330 OS_WAIT_QUEUE *pWq; 331 332 fabricMulticastFabricOpsMutexAcquire(pFabric); 333 334 pWq = (OS_WAIT_QUEUE *)fabricMulticastCleanupCacheGetUnderLock_IMPL(pFabric, 335 inbandReqId); 336 fabricMulticastFabricOpsMutexRelease(pFabric); 337 338 if (pWq == NULL) 339 return; 340 341 osWaitInterruptible(pWq); 342 343 fabricMulticastFabricOpsMutexAcquire(pFabric); 344 345 fabricMulticastCleanupCacheDeleteUnderLock_IMPL(pFabric, 346 inbandReqId); 347 348 fabricMulticastFabricOpsMutexRelease(pFabric); 349 350 osFreeWaitQueue(pWq); 351 } 352