1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 25 #include "nvrm_registry.h" 26 #include "rmapi/rmapi.h" 27 #include "entry_points.h" 28 #include "resserv/rs_server.h" 29 #include "rmapi/rs_utils.h" 30 #include "gpu/gpu_resource.h" 31 #include "gpu/device/device.h" 32 #include "core/locks.h" 33 #include "gpu/gpu.h" 34 #include "diagnostics/tracer.h" 35 #include "tls/tls.h" 36 #include "core/thread_state.h" 37 #include "gpu_mgr/gpu_mgr.h" 38 #include "resource_desc.h" 39 40 typedef struct 41 { 42 PORT_RWLOCK * pLock; 43 NvU64 threadId; 44 NvU64 timestamp; 45 LOCK_TRACE_INFO traceInfo; 46 NvU64 tlsEntryId; 47 volatile NvU32 contentionCount; 48 NvU32 lowPriorityAging; 49 } RMAPI_LOCK; 50 51 RsServer g_resServ; 52 static RM_API g_RmApiList[RMAPI_TYPE_MAX]; 53 static NvBool g_bResServInit = NV_FALSE; 54 static RMAPI_LOCK g_RmApiLock; 55 56 static void _rmapiInitInterface(RM_API *pRmApi, API_SECURITY_INFO *pDefaultSecurityInfo, NvBool bTlsInternal, 57 NvBool bApiLockInternal, NvBool bGpuLockInternal); 58 static NV_STATUS _rmapiLockAlloc(void); 59 static void _rmapiLockFree(void); 60 61 // from rmapi_stubs.c 62 void rmapiInitStubInterface(RM_API *pRmApi); 63 64 NV_STATUS 65 rmapiInitialize 66 ( 67 void 68 ) 69 { 70 NV_STATUS status = NV_OK; 71 API_SECURITY_INFO secInfo = {0}; 72 73 NV_ASSERT(!g_bResServInit); 74 75 status = _rmapiLockAlloc(); 76 77 if (status != NV_OK) 78 { 79 NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate rmapi locks\n"); 80 goto failed; 81 } 82 83 status = rmapiControlCacheInit(); 84 if (status != NV_OK) 85 { 86 NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize rmapi cache\n"); 87 goto failed_free_lock; 88 } 89 90 RsResInfoInitialize(); 91 status = serverConstruct(&g_resServ, RS_PRIV_LEVEL_HOST, 0); 92 93 if (status != NV_OK) 94 { 95 NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize resource server\n"); 96 goto failed_free_cache; 97 } 98 99 serverSetClientHandleBase(&g_resServ, RS_CLIENT_HANDLE_BASE); 100 101 listInit(&g_clientListBehindGpusLock, g_resServ.pAllocator); 102 listInit(&g_userInfoList, g_resServ.pAllocator); 103 multimapInit(&g_osInfoList, g_resServ.pAllocator); 104 105 secInfo.privLevel = RS_PRIV_LEVEL_KERNEL; 106 secInfo.paramLocation = PARAM_LOCATION_KERNEL; 107 108 _rmapiInitInterface(&g_RmApiList[RMAPI_EXTERNAL], NULL, NV_FALSE /* bTlsInternal */, NV_FALSE /* bApiLockInternal */, NV_FALSE /* bGpuLockInternal */); 109 _rmapiInitInterface(&g_RmApiList[RMAPI_EXTERNAL_KERNEL], &secInfo, NV_FALSE /* bTlsInternal */, NV_FALSE /* bApiLockInternal */, NV_FALSE /* bGpuLockInternal */); 110 _rmapiInitInterface(&g_RmApiList[RMAPI_MODS_LOCK_BYPASS], &secInfo, NV_FALSE /* bTlsInternal */, NV_TRUE /* bApiLockInternal */, NV_TRUE /* bGpuLockInternal */); 111 _rmapiInitInterface(&g_RmApiList[RMAPI_API_LOCK_INTERNAL], &secInfo, NV_TRUE /* bTlsInternal */, NV_TRUE /* bApiLockInternal */, NV_FALSE /* bGpuLockInternal */); 112 _rmapiInitInterface(&g_RmApiList[RMAPI_GPU_LOCK_INTERNAL], &secInfo, NV_TRUE /* bTlsInternal */, NV_TRUE /* bApiLockInternal */, NV_TRUE /* bGpuLockInternal */); 113 114 rmapiInitStubInterface(&g_RmApiList[RMAPI_STUBS]); 115 116 g_bResServInit = NV_TRUE; 117 118 return NV_OK; 119 120 failed_free_cache: 121 rmapiControlCacheFree(); 122 failed_free_lock: 123 _rmapiLockFree(); 124 failed: 125 return status; 126 } 127 128 void 129 rmapiShutdown 130 ( 131 void 132 ) 133 { 134 if (!g_bResServInit) 135 return; 136 137 serverFreeDomain(&g_resServ, 0); 138 serverDestruct(&g_resServ); 139 _rmapiLockFree(); 140 141 rmapiControlCacheFree(); 142 143 g_bResServInit = NV_FALSE; 144 } 145 146 static void 147 _rmapiInitInterface 148 ( 149 RM_API *pRmApi, 150 API_SECURITY_INFO *pDefaultSecInfo, 151 NvBool bTlsInternal, 152 NvBool bApiLockInternal, 153 NvBool bGpuLockInternal 154 ) 155 { 156 // 157 // Initialize to all stubs first, so any APIs not explicitly set here 158 // will return NV_ERR_NOT_SUPPORTED if called 159 // 160 rmapiInitStubInterface(pRmApi); 161 162 // 163 // Init members 164 // 165 if (pDefaultSecInfo) 166 pRmApi->defaultSecInfo = *pDefaultSecInfo; 167 168 pRmApi->bHasDefaultSecInfo = !!pDefaultSecInfo; 169 pRmApi->bTlsInternal = bTlsInternal; 170 pRmApi->bApiLockInternal = bApiLockInternal; 171 pRmApi->bRmSemaInternal = bApiLockInternal; 172 pRmApi->bGpuLockInternal = bGpuLockInternal; 173 pRmApi->pPrivateContext = NULL; 174 175 // 176 // Init function pointers 177 // 178 pRmApi->Alloc = rmapiAlloc; 179 pRmApi->AllocWithHandle = rmapiAllocWithHandle; 180 pRmApi->AllocWithSecInfo = pRmApi->bTlsInternal ? rmapiAllocWithSecInfo : rmapiAllocWithSecInfoTls; 181 182 pRmApi->DisableClients = rmapiDisableClients; 183 pRmApi->DisableClientsWithSecInfo = pRmApi->bTlsInternal ? rmapiDisableClientsWithSecInfo : rmapiDisableClientsWithSecInfoTls; 184 185 pRmApi->Free = rmapiFree; 186 pRmApi->FreeWithSecInfo = pRmApi->bTlsInternal ? rmapiFreeWithSecInfo : rmapiFreeWithSecInfoTls; 187 188 pRmApi->Control = rmapiControl; 189 pRmApi->ControlWithSecInfo = pRmApi->bTlsInternal ? rmapiControlWithSecInfo : rmapiControlWithSecInfoTls; 190 191 pRmApi->DupObject = rmapiDupObject; 192 pRmApi->DupObjectWithSecInfo = pRmApi->bTlsInternal ? rmapiDupObjectWithSecInfo : rmapiDupObjectWithSecInfoTls; 193 194 pRmApi->Share = rmapiShare; 195 pRmApi->ShareWithSecInfo = pRmApi->bTlsInternal ? rmapiShareWithSecInfo : rmapiShareWithSecInfoTls; 196 197 pRmApi->MapToCpu = rmapiMapToCpu; 198 pRmApi->MapToCpuWithSecInfo = pRmApi->bTlsInternal ? rmapiMapToCpuWithSecInfo : rmapiMapToCpuWithSecInfoTls; 199 pRmApi->MapToCpuWithSecInfoV2 = pRmApi->bTlsInternal ? rmapiMapToCpuWithSecInfoV2 : rmapiMapToCpuWithSecInfoTlsV2; 200 201 pRmApi->UnmapFromCpu = rmapiUnmapFromCpu; 202 pRmApi->UnmapFromCpuWithSecInfo = pRmApi->bTlsInternal ? rmapiUnmapFromCpuWithSecInfo : rmapiUnmapFromCpuWithSecInfoTls; 203 204 pRmApi->Map = rmapiMap; 205 pRmApi->MapWithSecInfo = pRmApi->bTlsInternal ? rmapiMapWithSecInfo : rmapiMapWithSecInfoTls; 206 207 pRmApi->Unmap = rmapiUnmap; 208 pRmApi->UnmapWithSecInfo = pRmApi->bTlsInternal ? rmapiUnmapWithSecInfo : rmapiUnmapWithSecInfoTls; 209 } 210 211 RM_API * 212 rmapiGetInterface 213 ( 214 RMAPI_TYPE rmapiType 215 ) 216 { 217 return &g_RmApiList[rmapiType]; 218 } 219 220 static void 221 _rmapiUnrefGpuAccessNeeded 222 ( 223 NvU32 gpuMask 224 ) 225 { 226 NvU32 gpuInstance = 0; 227 OBJGPU *pGpu = NULL; 228 229 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 230 { 231 osUnrefGpuAccessNeeded(pGpu->pOsGpuInfo); 232 } 233 } 234 235 static NV_STATUS 236 _rmapiRefGpuAccessNeeded 237 ( 238 NvU32 *pGpuMask 239 ) 240 { 241 NV_STATUS status = NV_OK; 242 NvU32 mask = 0; 243 NvU32 gpuInstance = 0; 244 OBJGPU *pGpu = NULL; 245 246 status = gpumgrGetGpuAttachInfo(NULL, &mask); 247 if (status != NV_OK) 248 { 249 return status; 250 } 251 252 while ((pGpu = gpumgrGetNextGpu(mask, &gpuInstance)) != NULL) 253 { 254 status = osRefGpuAccessNeeded(pGpu->pOsGpuInfo); 255 if (status != NV_OK) 256 { 257 goto unref; 258 } 259 260 /* 261 *_rmapiRefGpuAccessNeeded records the gpuMask 262 * during ref up and this is used to unref exact same 263 * GPUs in _rmapiUnrefGpuAccessNeeded. This is done 264 * to protect against obtaining incorrect pGpu if the mask 265 * changes due to a RM_API called between ref/unref 266 * sequence. 267 */ 268 *pGpuMask |= (1 << pGpu->gpuInstance); 269 } 270 271 unref: 272 if (status != NV_OK) 273 { 274 _rmapiUnrefGpuAccessNeeded(*pGpuMask); 275 } 276 return status; 277 } 278 279 NV_STATUS 280 rmapiPrologue 281 ( 282 RM_API *pRmApi, 283 RM_API_CONTEXT *pContext 284 ) 285 { 286 NV_STATUS status = NV_OK; 287 NvBool bApiLockTaken = NV_FALSE; 288 NvU32 mask; 289 290 NV_ASSERT_OR_RETURN(pRmApi != NULL, NV_ERR_INVALID_ARGUMENT); 291 NV_ASSERT_OR_RETURN(pContext != NULL, NV_ERR_INVALID_ARGUMENT); 292 293 /* 294 * Check for external clients. This condition is checked here 295 * in order to avoid a check at all caller sites of 296 * rmapiPrologue. Effectively rmapiprologue is a no-op for 297 * internal clients. 298 */ 299 if (!pRmApi->bTlsInternal) 300 { 301 mask = osGetDynamicPowerSupportMask(); 302 if (!mask) 303 return status; 304 /* 305 * NOTE1: Callers of rmapiPro{Epi}logue function call may call 306 * it with or without API lock taken. Hence, we check here 307 * whether API lock has been taken. We take API lock if 308 * it not taken already. 309 * We obtain the pGPU by using the gpuMask in 310 * _rmapiRef{Unref}GpuAccessNeeded. This needs API lock to be 311 * safe against init/teardown of GPUs while we ref/unref 312 * the GPUs. We release the lock after we have finished 313 * with ref/unref, if we had taken it. 314 */ 315 if (!rmapiLockIsOwner()) 316 { 317 status = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_CLIENT); 318 if (status != NV_OK) 319 { 320 return status; 321 } 322 bApiLockTaken = NV_TRUE; 323 } 324 status = _rmapiRefGpuAccessNeeded(&pContext->gpuMask); 325 if (bApiLockTaken == NV_TRUE) 326 { 327 rmapiLockRelease(); 328 } 329 } 330 return status; 331 } 332 333 void 334 rmapiEpilogue 335 ( 336 RM_API *pRmApi, 337 RM_API_CONTEXT *pContext 338 ) 339 { 340 NV_STATUS status = NV_OK; 341 NvBool bApiLockTaken = NV_FALSE; 342 NvU32 mask; 343 344 NV_ASSERT_OR_RETURN_VOID(pRmApi != NULL); 345 NV_ASSERT_OR_RETURN_VOID(pContext != NULL); 346 347 /* 348 * Check for external clients. This condition is checked here 349 * in order to avoid a check at all caller sites of 350 * rmapiEpilogue. Effectively rmapiEpilogue is a no-op for 351 * internal clients. 352 */ 353 if (!pRmApi->bTlsInternal) 354 { 355 mask = osGetDynamicPowerSupportMask(); 356 if (!mask) 357 return; 358 359 /* Please see NOTE1 */ 360 if (!rmapiLockIsOwner()) 361 { 362 status = rmapiLockAcquire(RMAPI_LOCK_FLAGS_READ, RM_LOCK_MODULES_CLIENT); 363 if (status != NV_OK) 364 { 365 return; 366 } 367 bApiLockTaken = NV_TRUE; 368 } 369 370 _rmapiUnrefGpuAccessNeeded(pContext->gpuMask); 371 372 if (bApiLockTaken == NV_TRUE) 373 { 374 rmapiLockRelease(); 375 } 376 } 377 } 378 379 void 380 rmapiInitLockInfo 381 ( 382 RM_API *pRmApi, 383 NvHandle hClient, 384 RS_LOCK_INFO *pLockInfo 385 ) 386 { 387 NV_ASSERT_OR_RETURN_VOID(pLockInfo != NULL); 388 pLockInfo->flags = 0; 389 pLockInfo->state = 0; 390 391 if (hClient != 0) 392 { 393 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 394 if ((pCallContext != NULL) && (pCallContext->pLockInfo != NULL)) 395 { 396 pLockInfo->state = pCallContext->pLockInfo->state; 397 398 if ((pCallContext->pLockInfo->pClient != NULL) && 399 (pCallContext->pLockInfo->pClient->hClient == hClient)) 400 { 401 pLockInfo->pClient = pCallContext->pLockInfo->pClient; 402 } 403 else 404 { 405 pLockInfo->state &= ~RM_LOCK_STATES_CLIENT_LOCK_ACQUIRED; 406 } 407 } 408 } 409 410 if (!pRmApi->bRmSemaInternal) 411 pLockInfo->flags |= RM_LOCK_FLAGS_RM_SEMA; 412 413 if (pRmApi->bApiLockInternal) 414 { 415 pLockInfo->state |= RM_LOCK_STATES_API_LOCK_ACQUIRED; 416 417 // RS-TODO: Assert that API rwlock is taken if no client is locked 418 if (pLockInfo->pClient == NULL) 419 pLockInfo->flags |= RM_LOCK_FLAGS_NO_CLIENT_LOCK; 420 } 421 422 if (pRmApi->bGpuLockInternal) 423 pLockInfo->state |= RM_LOCK_STATES_ALLOW_RECURSIVE_LOCKS; 424 } 425 426 static NV_STATUS 427 _rmapiLockAlloc(void) 428 { 429 // Turn on by default for Linux to get some soak time 430 // bug 2539044, bug 2536036: Enable by default. 431 g_resServ.bUnlockedParamCopy = NV_TRUE; 432 433 NvU32 val = 0; 434 435 if ((osReadRegistryDword(NULL, 436 NV_REG_STR_RM_LOCKING_LOW_PRIORITY_AGING, 437 &val) == NV_OK)) 438 { 439 g_RmApiLock.lowPriorityAging = val; 440 } 441 442 if ((osReadRegistryDword(NULL, 443 NV_REG_STR_RM_PARAM_COPY_NO_LOCK, 444 &val) == NV_OK)) 445 { 446 g_resServ.bUnlockedParamCopy = (val != 0); 447 } 448 449 portMemSet(&g_RmApiLock, 0, sizeof(g_RmApiLock)); 450 g_RmApiLock.threadId = ~((NvU64)(0)); 451 g_RmApiLock.pLock = portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged()); 452 if (g_RmApiLock.pLock == NULL) 453 return NV_ERR_INSUFFICIENT_RESOURCES; 454 455 g_RmApiLock.tlsEntryId = tlsEntryAlloc(); 456 457 return NV_OK; 458 } 459 460 static void 461 _rmapiLockFree(void) 462 { 463 portSyncRwLockDestroy(g_RmApiLock.pLock); 464 } 465 466 NV_STATUS 467 rmapiLockAcquire(NvU32 flags, NvU32 module) 468 { 469 NV_STATUS rmStatus = NV_OK; 470 NvU64 threadId = portThreadGetCurrentThreadId(); 471 472 NvU64 myPriority = 0; 473 474 LOCK_ASSERT_AND_RETURN(!rmapiLockIsOwner()); 475 476 // 477 // If a read-only lock was requested, check to see if the module is allowed 478 // to take read-only locks 479 // 480 if ((flags & RMAPI_LOCK_FLAGS_READ) && (module != RM_LOCK_MODULES_NONE)) 481 { 482 OBJSYS *pSys = SYS_GET_INSTANCE(); 483 if ((pSys->apiLockModuleMask & RM_LOCK_MODULE_GRP(module)) == 0) 484 { 485 flags &= ~RMAPI_LOCK_FLAGS_READ; 486 } 487 } 488 489 // 490 // For conditional acquires and DISPATCH_LEVEL we want to exit 491 // immediately without waiting. 492 // 493 // If RM Locking V3 Lite is not enabled, *always* acquire the API 494 // lock in WRITE mode to ensure compatibility with Locking model V2 495 // behavior (providing exclusive access to the resource). 496 // 497 flags = osApiLockAcquireConfigureFlags(flags); 498 if (flags & API_LOCK_FLAGS_COND_ACQUIRE) 499 { 500 if ((flags & RMAPI_LOCK_FLAGS_READ)) 501 { 502 if (!portSyncRwLockAcquireReadConditional(g_RmApiLock.pLock)) 503 rmStatus = NV_ERR_TIMEOUT_RETRY; 504 } 505 else 506 { 507 // Conditional acquires don't care about contention or priority 508 if (portSyncRwLockAcquireWriteConditional(g_RmApiLock.pLock)) 509 { 510 g_RmApiLock.threadId = threadId; 511 } 512 else 513 { 514 rmStatus = NV_ERR_TIMEOUT_RETRY; 515 } 516 } 517 } 518 else 519 { 520 if ((flags & RMAPI_LOCK_FLAGS_READ)) 521 { 522 portSyncRwLockAcquireRead(g_RmApiLock.pLock); 523 } 524 else 525 { 526 527 if (flags & RMAPI_LOCK_FLAGS_LOW_PRIORITY) 528 { 529 NvS32 age = g_RmApiLock.lowPriorityAging; 530 portSyncRwLockAcquireWrite(g_RmApiLock.pLock); 531 while ((g_RmApiLock.contentionCount > 0) && (age--)) 532 { 533 portSyncRwLockReleaseWrite(g_RmApiLock.pLock); 534 osDelay(10); 535 portSyncRwLockAcquireWrite(g_RmApiLock.pLock); 536 } 537 } 538 else 539 { 540 portAtomicIncrementU32(&g_RmApiLock.contentionCount); 541 portSyncRwLockAcquireWrite(g_RmApiLock.pLock); 542 portAtomicDecrementU32(&g_RmApiLock.contentionCount); 543 } 544 g_RmApiLock.threadId = threadId; 545 } 546 } 547 548 549 if (rmStatus == NV_OK) 550 { 551 NvU64 timestamp; 552 osGetCurrentTick(×tamp); 553 554 if (g_RmApiLock.threadId == threadId) 555 g_RmApiLock.timestamp = timestamp; 556 557 // save off owning thread 558 RMTRACE_RMLOCK(_API_LOCK_ACQUIRE); 559 560 // add api lock trace record 561 INSERT_LOCK_TRACE(&g_RmApiLock.traceInfo, 562 NV_RETURN_ADDRESS(), 563 lockTraceAcquire, 564 flags, module, 565 threadId, 566 !portSyncExSafeToSleep(), 567 myPriority, 568 timestamp); 569 570 // 571 // If enabled, reset the timeout now that we are running and off 572 // the Sleep Queue. 573 // 574 if (threadStateGetSetupFlags() & 575 THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED) 576 { 577 threadStateResetTimeout(NULL); 578 } 579 } 580 581 NvP64 *pAcquireAddress = tlsEntryAcquire(g_RmApiLock.tlsEntryId); 582 if (pAcquireAddress != NULL) 583 { 584 *pAcquireAddress = (NvP64)(NvUPtr)NV_RETURN_ADDRESS(); 585 } 586 587 return rmStatus; 588 } 589 590 void 591 rmapiLockRelease(void) 592 { 593 NvU64 threadId = portThreadGetCurrentThreadId(); 594 NvU64 timestamp; 595 596 osGetCurrentTick(×tamp); 597 598 RMTRACE_RMLOCK(_API_LOCK_RELEASE); 599 600 // add api lock trace record 601 INSERT_LOCK_TRACE(&g_RmApiLock.traceInfo, 602 NV_RETURN_ADDRESS(), 603 lockTraceRelease, 604 0, 0, 605 threadId, 606 !portSyncExSafeToSleep(), 607 0, 608 timestamp); 609 610 if (g_RmApiLock.threadId == threadId) 611 { 612 // 613 // If the threadId in the global is same as current thread id, then 614 // we know that it was acquired in WRITE mode. 615 // 616 g_RmApiLock.threadId = ~0ull; 617 g_RmApiLock.timestamp = timestamp; 618 portSyncRwLockReleaseWrite(g_RmApiLock.pLock); 619 620 } 621 else 622 { 623 portSyncRwLockReleaseRead(g_RmApiLock.pLock); 624 } 625 626 tlsEntryRelease(g_RmApiLock.tlsEntryId); 627 } 628 629 NvBool 630 rmapiLockIsOwner(void) 631 { 632 return tlsEntryGet(g_RmApiLock.tlsEntryId) != NvP64_NULL; 633 } 634 635 // 636 // Mark for deletion the client resources from the data base, given a GPU mask 637 // 638 void 639 rmapiSetDelPendingClientResourcesFromGpuMask 640 ( 641 NvU32 gpuMask 642 ) 643 { 644 RS_ITERATOR it; 645 RmClient **ppClient; 646 RmClient *pClient; 647 RsClient *pRsClient; 648 Device *pDevice; 649 NvBool bDevicesInMask = NV_FALSE; 650 OBJGPU *pGpu; 651 652 for (ppClient = serverutilGetFirstClientUnderLock(); 653 ppClient; 654 ppClient = serverutilGetNextClientUnderLock(ppClient)) 655 { 656 pClient = *ppClient; 657 pRsClient = staticCast(pClient, RsClient); 658 659 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 660 661 // Check that one of the devices is in the GPU mask 662 bDevicesInMask = NV_FALSE; 663 while (clientRefIterNext(it.pClient, &it)) 664 { 665 pDevice = dynamicCast(it.pResourceRef->pResource, Device); 666 667 if (!pDevice) 668 { 669 continue; 670 } 671 672 pGpu = GPU_RES_GET_GPU(pDevice); 673 if ((gpuMask & NVBIT(gpuGetInstance(pGpu))) != 0) 674 { 675 bDevicesInMask = NV_TRUE; 676 break; 677 } 678 } 679 680 if (bDevicesInMask == NV_FALSE) 681 { 682 continue; 683 } 684 685 pClient->Flags |= RMAPI_CLIENT_FLAG_DELETE_PENDING; 686 } 687 } 688 689 void 690 rmapiDelPendingDevices 691 ( 692 NvU32 gpuMask 693 ) 694 { 695 RmClient **ppClient; 696 RmClient *pClient; 697 RsClient *pRsClient; 698 RS_ITERATOR it; 699 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 700 701 ppClient = serverutilGetFirstClientUnderLock(); 702 while (ppClient) 703 { 704 pClient = *ppClient; 705 pRsClient = staticCast(pClient, RsClient); 706 707 if (((pClient->Flags & RMAPI_CLIENT_FLAG_DELETE_PENDING) != 0) && 708 ((pClient->Flags & RMAPI_CLIENT_FLAG_RM_INTERNAL_CLIENT) == 0)) 709 { 710 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 711 while(clientRefIterNext(pRsClient, &it)) 712 { 713 RsResourceRef *pDeviceRef = it.pResourceRef; 714 Device *pDevice = dynamicCast(pDeviceRef->pResource, Device); 715 716 if ((gpuMask & NVBIT(gpuGetInstance(GPU_RES_GET_GPU(pDevice)))) != 0) 717 { 718 pRmApi->Free(pRmApi, pRsClient->hClient, pDeviceRef->hResource); 719 720 // Client's resource map has been modified, re-snap iterator 721 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 722 } 723 } 724 725 } 726 727 ppClient = serverutilGetNextClientUnderLock(ppClient); 728 } 729 } 730 731 void 732 rmapiReportLeakedDevices 733 ( 734 NvU32 gpuMask 735 ) 736 { 737 RmClient **ppClient; 738 RmClient *pClient; 739 RsClient *pRsClient; 740 RS_ITERATOR it; 741 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 742 743 ppClient = serverutilGetFirstClientUnderLock(); 744 while (ppClient) 745 { 746 pClient = *ppClient; 747 pRsClient = staticCast(pClient, RsClient); 748 749 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 750 while(clientRefIterNext(pRsClient, &it)) 751 { 752 RsResourceRef *pDeviceRef = it.pResourceRef; 753 Device *pDevice = dynamicCast(pDeviceRef->pResource, Device); 754 755 if ((gpuMask & NVBIT(gpuGetInstance(GPU_RES_GET_GPU(pDevice)))) != 0) 756 { 757 NV_PRINTF(LEVEL_ERROR, 758 "Device object leak: (0x%x, 0x%x). Please file a bug against RM-core.\n", 759 pRsClient->hClient, pDeviceRef->hResource); 760 NV_ASSERT(0); 761 762 // Delete leaked resource from database 763 pRmApi->Free(pRmApi, pRsClient->hClient, pDeviceRef->hResource); 764 765 // Client's resource map has been modified, re-snap iterator 766 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 767 } 768 } 769 770 ppClient = serverutilGetNextClientUnderLock(ppClient); 771 } 772 } 773 774 // 775 // Delete the marked client resources 776 // 777 void 778 rmapiDelPendingClients 779 ( 780 void 781 ) 782 { 783 RmClient **ppClient; 784 RmClient *pClient; 785 RsClient *pRsClient; 786 RS_ITERATOR it; 787 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 788 789 ppClient = serverutilGetFirstClientUnderLock(); 790 while (ppClient) 791 { 792 pClient = *ppClient; 793 pRsClient = staticCast(pClient, RsClient); 794 ppClient = serverutilGetNextClientUnderLock(ppClient); 795 if ((pClient->Flags & RMAPI_CLIENT_FLAG_DELETE_PENDING) != 0) 796 { 797 // Only free clients that have no devices left 798 it = clientRefIter(pRsClient, NULL, classId(Device), RS_ITERATE_CHILDREN, NV_TRUE); 799 if (!clientRefIterNext(pRsClient, &it)) 800 pRmApi->Free(pRmApi, pRsClient->hClient, pRsClient->hClient); 801 } 802 } 803 } 804 805 extern OsInfoMap g_osInfoList; 806 807 NV_STATUS 808 rmapiGetClientHandlesFromOSInfo 809 ( 810 void *pOSInfo, 811 NvHandle **ppClientHandleList, 812 NvU32 *pClientHandleListSize 813 ) 814 { 815 OBJSYS *pSys = SYS_GET_INSTANCE(); 816 817 NvHandle *pClientHandleList; 818 NvU32 clientHandleListSize = 0; 819 NvU32 k; 820 821 RmClient **ppClient; 822 RmClient **ppFirstClient; 823 RmClient *pClient; 824 RsClient *pRsClient; 825 826 NvBool clientHandleLookup = pSys->getProperty(pSys, PDB_PROP_SYS_CLIENT_HANDLE_LOOKUP); 827 828 if (!clientHandleLookup) 829 { 830 ppFirstClient = NULL; 831 for (ppClient = serverutilGetFirstClientUnderLock(); 832 ppClient; 833 ppClient = serverutilGetNextClientUnderLock(ppClient)) 834 { 835 pClient = *ppClient; 836 if (pClient->pOSInfo != pOSInfo) 837 { 838 continue; 839 } 840 clientHandleListSize++; 841 842 if (NULL == ppFirstClient) 843 ppFirstClient = ppClient; 844 } 845 846 if (clientHandleListSize == 0) 847 { 848 *pClientHandleListSize = 0; 849 *ppClientHandleList = NULL; 850 return NV_ERR_INVALID_ARGUMENT; 851 } 852 853 pClientHandleList = portMemAllocNonPaged(clientHandleListSize * sizeof(NvU32)); 854 if (pClientHandleList == NULL) 855 { 856 return NV_ERR_NO_MEMORY; 857 } 858 859 *pClientHandleListSize = clientHandleListSize; 860 *ppClientHandleList = pClientHandleList; 861 862 k = 0; 863 for (ppClient = ppFirstClient; 864 ppClient; 865 ppClient = serverutilGetNextClientUnderLock(ppClient)) 866 { 867 pClient = *ppClient; 868 pRsClient = staticCast(pClient, RsClient); 869 if (pClient->pOSInfo != pOSInfo) 870 { 871 continue; 872 } 873 pClientHandleList[k++] = pRsClient->hClient; 874 875 if (clientHandleListSize <= k) 876 break; 877 } 878 } 879 else 880 { 881 OsInfoMapSubmap *pSubmap = NULL; 882 OsInfoMapIter it; 883 NvU64 key1 = (NvUPtr)pOSInfo; 884 pSubmap = multimapFindSubmap(&g_osInfoList, key1); 885 if (pSubmap != NULL) 886 { 887 clientHandleListSize = multimapCountSubmapItems(&g_osInfoList, pSubmap); 888 NV_PRINTF(LEVEL_INFO, "*** Found %d clients for %llx\n", clientHandleListSize, key1); 889 } 890 if (clientHandleListSize == 0) 891 { 892 *pClientHandleListSize = 0; 893 *ppClientHandleList = NULL; 894 return NV_ERR_INVALID_ARGUMENT; 895 } 896 pClientHandleList = portMemAllocNonPaged(clientHandleListSize * sizeof(NvU32)); 897 if (pClientHandleList == NULL) 898 { 899 return NV_ERR_NO_MEMORY; 900 } 901 *pClientHandleListSize = clientHandleListSize; 902 *ppClientHandleList = pClientHandleList; 903 k = 0; 904 it = multimapSubmapIterItems(&g_osInfoList, pSubmap); 905 while(multimapItemIterNext(&it)) 906 { 907 pClient = *it.pValue; 908 pRsClient = staticCast(pClient, RsClient); 909 910 NV_CHECK_OR_ELSE_STR(LEVEL_ERROR, pClient->pOSInfo == pOSInfo, "*** OS info mismatch", continue); 911 912 pClientHandleList[k++] = pRsClient->hClient; 913 NV_PRINTF(LEVEL_INFO, "*** Found: %x\n", pRsClient->hClient); 914 if (clientHandleListSize <= k) 915 break; 916 } 917 } 918 919 return NV_OK; 920 } 921 922