1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2018-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "mem_mgr/mem.h" 25 26 #include "mem_mgr/fla_mem.h" 27 28 #include "gpu/gpu.h" 29 #include "gpu/mem_mgr/mem_mgr.h" 30 #include "gpu/disp/disp_objs.h" 31 #include "gpu/mem_mgr/mem_desc.h" 32 #include "os/os.h" 33 #include "core/locks.h" 34 #include "gpu/device/device.h" 35 #include "gpu/subdevice/subdevice.h" 36 #include "vgpu/rpc.h" 37 38 #include "class/cl0041.h" // NV04_MEMORY 39 #include "class/cl003e.h" // NV01_MEMORY_SYSTEM 40 #include "class/cl0071.h" // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR 41 42 NV_STATUS 43 memConstruct_IMPL 44 ( 45 Memory *pMemory, 46 CALL_CONTEXT *pCallContext, 47 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 48 ) 49 { 50 RsResourceRef *pResourceRef = pCallContext->pResourceRef; 51 RsResourceRef *pParentRef = pResourceRef->pParentRef; 52 53 // 54 // Common initialization used for both normal construction & copy 55 // constructor 56 // 57 58 // NULL if parent isn't a device 59 pMemory->pDevice = dynamicCast(pParentRef->pResource, Device); 60 61 // NULL if parent isn't a subdevice 62 pMemory->pSubDevice = dynamicCast(pParentRef->pResource, Subdevice); 63 64 // If parent subdevice, grandparent must be a device 65 if (pMemory->pSubDevice) 66 { 67 RsResourceRef *pGrandParentRef = pParentRef->pParentRef; 68 69 pMemory->pDevice = dynamicCast(pGrandParentRef->pResource, Device); 70 71 if (pMemory->pDevice == NULL) 72 return NV_ERR_INVALID_OBJECT_HANDLE; 73 } 74 75 // If child of device, we have a pGpu 76 if (pMemory->pDevice) 77 { 78 // NOTE: pGpu and pDevice be NULL for NoDeviceMemory 79 pMemory->pGpu = CliGetGpuFromContext(pResourceRef, &pMemory->bBcResource); 80 81 NV_ASSERT_OR_RETURN(pMemory->pGpu != NULL, NV_ERR_INVALID_ARGUMENT); 82 83 // Set thread BC state 84 gpuSetThreadBcState(pMemory->pGpu, pMemory->bBcResource); 85 } 86 87 if (RS_IS_COPY_CTOR(pParams)) 88 { 89 // 90 // Copy constructor path (NvRmDupObject) 91 // 92 return memCopyConstruct_IMPL(pMemory, pCallContext, pParams); 93 } 94 else 95 { 96 // 97 // Default constructor path (NvRmAlloc) 98 // 99 } 100 101 return NV_OK; 102 } 103 104 NV_STATUS 105 memGetMapAddrSpace_IMPL 106 ( 107 Memory *pMemory, 108 CALL_CONTEXT *pCallContext, 109 NvU32 mapFlags, 110 NV_ADDRESS_SPACE *pAddrSpace 111 ) 112 { 113 NV_ADDRESS_SPACE addrSpace; 114 OBJGPU *pGpu = pMemory->pGpu; 115 NvBool bBcResource = pMemory->bBcResource; 116 MEMORY_DESCRIPTOR *pMemDesc = NULL; 117 118 if (pGpu == NULL) 119 return NV_ERR_INVALID_OBJECT; 120 121 gpuSetThreadBcState(pGpu, bBcResource); 122 123 pMemDesc = memdescGetMemDescFromGpu(pMemory->pMemDesc, pGpu); 124 125 NV_ASSERT_OK_OR_RETURN(rmapiGetEffectiveAddrSpace(pGpu, pMemDesc, mapFlags, &addrSpace)); 126 127 if (addrSpace == ADDR_SYSMEM) 128 { 129 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_BAR0_REFLECT)) 130 { 131 addrSpace = ADDR_REGMEM; 132 } 133 else if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_BAR1_REFLECT)) 134 { 135 addrSpace = ADDR_FBMEM; 136 } 137 } 138 139 if (pAddrSpace) 140 *pAddrSpace = addrSpace; 141 142 return NV_OK; 143 } 144 145 void 146 memDestruct_IMPL 147 ( 148 Memory *pMemory 149 ) 150 { 151 OBJGPU *pGpu = pMemory->pGpu; 152 NvHandle hClient = RES_GET_CLIENT_HANDLE(pMemory); 153 NvHandle hParent = RES_GET_PARENT_HANDLE(pMemory); 154 NvHandle hMemory = RES_GET_HANDLE(pMemory); 155 NV_STATUS status = NV_OK; 156 157 // 158 // The default destructor is used when memConstructCommon() is called by 159 // the subclass but not memDestructCommon(). 160 // 161 if (pMemory->bConstructed && pMemory->pMemDesc != NULL) 162 { 163 // Remove the system memory reference from the client 164 memDestructCommon(pMemory); 165 memdescFree(pMemory->pMemDesc); 166 memdescDestroy(pMemory->pMemDesc); 167 } 168 169 // if the allocation is RPC-ed, free using RPC 170 if (pMemory->bRpcAlloc && (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu))) 171 { 172 NV_RM_RPC_FREE(pGpu, hClient, hParent, hMemory, status); 173 NV_ASSERT(status == NV_OK); 174 } 175 } 176 177 NV_STATUS 178 memCreateMemDesc_IMPL 179 ( 180 OBJGPU *pGpu, 181 MEMORY_DESCRIPTOR **ppMemDesc, 182 NV_ADDRESS_SPACE addrSpace, 183 NvU64 FBOffset, 184 NvU64 length, 185 NvU32 attr, 186 NvU32 attr2 187 ) 188 { 189 NV_STATUS status = NV_OK; 190 NvU32 CpuCacheAttrib, gpuCacheAttrib; 191 MEMORY_DESCRIPTOR *pMemDesc = NULL; 192 193 *ppMemDesc = NULL; 194 195 if (addrSpace == ADDR_SYSMEM) 196 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr), NV_ERR_INVALID_ARGUMENT); 197 198 // setup the CpuCacheAttrib as well.. (if the caller doesn't specify anything it will be 0=UNCACHED) 199 switch (DRF_VAL(OS32, _ATTR, _COHERENCY, attr)) 200 { 201 case NVOS32_ATTR_COHERENCY_UNCACHED: 202 CpuCacheAttrib = NV_MEMORY_UNCACHED; 203 break; 204 case NVOS32_ATTR_COHERENCY_WRITE_COMBINE: 205 CpuCacheAttrib = NV_MEMORY_WRITECOMBINED; 206 break; 207 case NVOS32_ATTR_COHERENCY_CACHED: 208 case NVOS32_ATTR_COHERENCY_WRITE_THROUGH: 209 case NVOS32_ATTR_COHERENCY_WRITE_PROTECT: 210 case NVOS32_ATTR_COHERENCY_WRITE_BACK: 211 CpuCacheAttrib = NV_MEMORY_CACHED; 212 break; 213 default: 214 NV_ASSERT(0); 215 CpuCacheAttrib = NV_MEMORY_UNCACHED; 216 break; 217 } 218 219 gpuCacheAttrib = FLD_TEST_DRF(OS32, _ATTR2, _GPU_CACHEABLE, _YES, attr2) ? NV_MEMORY_CACHED : NV_MEMORY_UNCACHED; 220 221 // Create and fill in a memory descriptor 222 status = memdescCreate(&pMemDesc, pGpu, length, 0, NV_TRUE, addrSpace, 223 CpuCacheAttrib, 224 MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE_FB_BC_ONLY(pGpu, addrSpace)); 225 if (status == NV_OK) 226 { 227 if (memdescHasSubDeviceMemDescs(pMemDesc)) 228 { 229 MEMORY_DESCRIPTOR *pMemDescNext = pMemDesc->_pNext; 230 while (pMemDescNext) 231 { 232 memdescDescribe(pMemDescNext, addrSpace, FBOffset, length); 233 memdescSetGpuCacheAttrib(pMemDescNext, gpuCacheAttrib); 234 pMemDescNext = pMemDescNext->_pNext; 235 } 236 } 237 else 238 { 239 memdescDescribe(pMemDesc, addrSpace, FBOffset, length); 240 memdescSetGpuCacheAttrib(pMemDesc, gpuCacheAttrib); 241 } 242 243 *ppMemDesc = pMemDesc; 244 } 245 246 return status; 247 } 248 249 NV_STATUS 250 memCreateKernelMapping_IMPL 251 ( 252 Memory *pMemory, 253 NvU32 Protect, 254 NvBool bClear 255 ) 256 { 257 NV_STATUS status; 258 259 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE)); 260 261 if (pMemory->KernelVAddr == NvP64_NULL) 262 { 263 if (memdescGetAddressSpace(pMemory->pMemDesc) != ADDR_SYSMEM) 264 { 265 return NV_ERR_NOT_SUPPORTED; 266 } 267 268 status = memdescMap(pMemory->pMemDesc, 0, pMemory->Length, NV_TRUE, 269 Protect, &pMemory->KernelVAddr, &pMemory->KernelMapPriv); 270 271 if (status != NV_OK) 272 { 273 pMemory->KernelVAddr = NvP64_NULL; 274 pMemory->KernelMapPriv = NvP64_NULL; 275 return status; 276 } 277 278 memdescSetKernelMapping(pMemory->pMemDesc, pMemory->KernelVAddr); 279 memdescSetKernelMappingPriv(pMemory->pMemDesc, pMemory->KernelMapPriv); 280 281 if (bClear) 282 { 283 portMemSet(NvP64_VALUE(pMemory->KernelVAddr), 0, pMemory->Length); 284 } 285 } 286 287 return NV_OK; 288 } 289 290 RM_ATTR_PAGE_SIZE 291 dmaNvos32ToPageSizeAttr 292 ( 293 NvU32 attr, 294 NvU32 attr2 295 ) 296 { 297 switch (DRF_VAL(OS32, _ATTR, _PAGE_SIZE, attr)) 298 { 299 case NVOS32_ATTR_PAGE_SIZE_DEFAULT: 300 return RM_ATTR_PAGE_SIZE_DEFAULT; 301 case NVOS32_ATTR_PAGE_SIZE_4KB: 302 return RM_ATTR_PAGE_SIZE_4KB; 303 case NVOS32_ATTR_PAGE_SIZE_BIG: 304 return RM_ATTR_PAGE_SIZE_BIG; 305 case NVOS32_ATTR_PAGE_SIZE_HUGE: 306 switch (DRF_VAL(OS32, _ATTR2, _PAGE_SIZE_HUGE, attr2)) 307 { 308 case NVOS32_ATTR2_PAGE_SIZE_HUGE_DEFAULT: 309 case NVOS32_ATTR2_PAGE_SIZE_HUGE_2MB: 310 return RM_ATTR_PAGE_SIZE_HUGE; 311 case NVOS32_ATTR2_PAGE_SIZE_HUGE_512MB: 312 return RM_ATTR_PAGE_SIZE_512MB; 313 } 314 break; 315 } 316 317 NV_ASSERT_FAILED("Invalid attr and attr2 page size arguments"); 318 return RM_ATTR_PAGE_SIZE_DEFAULT; 319 } 320 321 NV_STATUS 322 memConstructCommon_IMPL 323 ( 324 Memory *pMemory, 325 NvU32 categoryClassId, 326 NvU32 flags, 327 MEMORY_DESCRIPTOR *pMemDesc, 328 NvU32 heapOwner, 329 Heap *pHeap, 330 NvU32 attr, 331 NvU32 attr2, 332 NvU32 Pitch, 333 NvU32 type, 334 NvU32 tag, 335 HWRESOURCE_INFO *pHwResource 336 ) 337 { 338 OBJGPU *pGpu = NULL; 339 NV_STATUS status = NV_OK; 340 341 if (pMemDesc == NULL) 342 return NV_ERR_INVALID_ARGUMENT; 343 344 // initialize the memory description 345 pMemory->categoryClassId = categoryClassId; 346 pMemory->pMemDesc = pMemDesc; 347 pMemory->Length = pMemDesc->Size; 348 pMemory->RefCount = 1; 349 pMemory->HeapOwner = heapOwner; 350 pMemory->pHeap = pHeap; 351 pMemory->Attr = attr; 352 pMemory->Attr2 = attr2; 353 pMemory->Pitch = Pitch; 354 pMemory->Type = type; 355 pMemory->Flags = flags; 356 pMemory->tag = tag; 357 pMemory->isMemDescOwner = NV_TRUE; 358 pMemory->bRpcAlloc = NV_FALSE; 359 360 // We are finished if this instance is device-less 361 if (pMemory->pDevice == NULL) 362 { 363 goto done; 364 } 365 366 if (pMemDesc->pGpu == NULL) 367 { 368 return NV_ERR_INVALID_STATE; 369 } 370 371 // Memory has hw resources associated with it that need to be tracked. 372 if (pHwResource != NULL) 373 { 374 pMemory->pHwResource = portMemAllocNonPaged(sizeof(HWRESOURCE_INFO)); 375 if (pMemory->pHwResource != NULL) 376 { 377 *pMemory->pHwResource = *pHwResource; // struct copy 378 pMemory->pHwResource->refCount = 1; 379 } 380 else 381 { 382 NV_PRINTF(LEVEL_ERROR, 383 "Unable to allocate HWRESOURCE_INFO tracking structure\n"); 384 status = NV_ERR_NO_MEMORY; 385 goto done; 386 } 387 } 388 389 NV_ASSERT(status == NV_OK); 390 391 // 392 // Apply attr and flags to the memory descriptor. Ideally all should 393 // be handled before we get here. 394 // 395 396 // Check whether encryption should be enabled 397 if (flags & NVOS32_ALLOC_FLAGS_TURBO_CIPHER_ENCRYPTED) 398 { 399 pGpu = pMemDesc->pGpu; 400 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 401 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_ENCRYPTED, NV_TRUE); 402 SLI_LOOP_END 403 } 404 405 if (FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2)) 406 { 407 pGpu = pMemDesc->pGpu; 408 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 409 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_USER_READ_ONLY, NV_TRUE); 410 SLI_LOOP_END 411 } 412 413 if (FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2)) 414 { 415 pGpu = pMemDesc->pGpu; 416 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 417 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_DEVICE_READ_ONLY, NV_TRUE); 418 SLI_LOOP_END 419 } 420 421 // setup GpuP2PCacheAttrib 422 switch (DRF_VAL(OS32, _ATTR2, _P2P_GPU_CACHEABLE, attr2)) 423 { 424 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_YES: 425 pGpu = pMemDesc->pGpu; 426 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 427 memdescSetGpuP2PCacheAttrib(memdescGetMemDescFromGpu(pMemDesc, pGpu), NV_MEMORY_CACHED); 428 SLI_LOOP_END 429 break; 430 default: 431 NV_ASSERT(0); 432 /*FALLSTHRU*/ 433 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_NO: 434 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_DEFAULT: 435 pGpu = pMemDesc->pGpu; 436 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 437 memdescSetGpuP2PCacheAttrib(memdescGetMemDescFromGpu(pMemDesc, pGpu), NV_MEMORY_UNCACHED); 438 SLI_LOOP_END 439 break; 440 } 441 442 // 443 // Page size may be specified at allocation. This if for fermi family 444 // chips and is a nop for previous generations. At this point the hal call 445 // to set the page size should never fail as the memory was just allocated. 446 // 447 if (pMemDesc->pGpu) 448 { 449 pGpu = pMemDesc->pGpu; 450 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 451 452 RM_ATTR_PAGE_SIZE pageSizeAttr = dmaNvos32ToPageSizeAttr(attr, attr2); 453 status = memmgrSetMemDescPageSize_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), memdescGetMemDescFromGpu(pMemDesc, pGpu), 454 AT_GPU, pageSizeAttr); 455 if (status != NV_OK) 456 { 457 SLI_LOOP_BREAK; 458 } 459 SLI_LOOP_END 460 461 if (status != NV_OK) 462 { 463 goto done; 464 } 465 } 466 467 pMemory->Node.keyStart = RES_GET_HANDLE(pMemory); 468 pMemory->Node.keyEnd = RES_GET_HANDLE(pMemory); 469 pMemory->Node.Data = pMemory; 470 471 status = btreeInsert(&pMemory->Node, &pMemory->pDevice->DevMemoryTable); 472 if (status != NV_OK) 473 goto done; 474 475 // Initialize the circular list item for tracking dup/sharing of pMemDesc 476 pMemory->dupListItem.pNext = pMemory->dupListItem.pPrev = pMemory; 477 478 done: 479 if (status != NV_OK) 480 { 481 if (pMemory->pHwResource != NULL) 482 { 483 portMemFree(pMemory->pHwResource); 484 } 485 } 486 else 487 { 488 pMemory->bConstructed = NV_TRUE; 489 } 490 491 return status; 492 } 493 494 static NvBool 495 _memCheckHostVgpuDeviceExists 496 ( 497 OBJGPU *pGpu 498 ) 499 { 500 NV_STATUS status; 501 502 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL; 503 504 NV_ASSERT_OK_OR_ELSE(status, vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice), return NV_FALSE); 505 506 return (pKernelHostVgpuDevice != NULL); 507 } 508 509 static void 510 _memDestructCommonWithDevice 511 ( 512 Memory *pMemory 513 ) 514 { 515 NvHandle hMemory = RES_GET_HANDLE(pMemory); 516 OBJGPU *pGpu = pMemory->pGpu; 517 Device *pDevice = pMemory->pDevice; 518 RsResourceRef *pDeviceRef = RES_GET_REF(pDevice); 519 NvHandle hDevice = RES_GET_HANDLE(pDevice); 520 Subdevice *pSubDeviceInfo; 521 DispCommon *pDispCommon; 522 RsClient *pRsClient = RES_GET_CLIENT(pMemory); 523 NV_STATUS status; 524 RS_ITERATOR subDevIt; 525 FB_ALLOC_INFO *pFbAllocInfo = NULL; 526 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 527 528 gpuSetThreadBcState(pGpu, pMemory->bBcResource); 529 530 subDevIt = clientRefIter(pRsClient, pDeviceRef, classId(Subdevice), RS_ITERATE_CHILDREN, NV_TRUE); 531 while (clientRefIterNext(pRsClient, &subDevIt)) 532 { 533 pSubDeviceInfo = dynamicCast(subDevIt.pResourceRef->pResource, Subdevice); 534 535 if (hMemory == pSubDeviceInfo->hNotifierMemory) 536 { 537 pSubDeviceInfo->hNotifierMemory = NV01_NULL_OBJECT; 538 pSubDeviceInfo->pNotifierMemory = NULL; 539 } 540 } 541 542 dispcmnGetByDevice(pRsClient, hDevice, &pDispCommon); 543 544 if (pDispCommon != NULL) 545 { 546 DisplayApi *pDisplayApi = staticCast(pDispCommon, DisplayApi); 547 if (pDisplayApi->hNotifierMemory == hMemory) 548 { 549 pDisplayApi->hNotifierMemory = NV01_NULL_OBJECT; 550 pDisplayApi->pNotifierMemory = NULL; 551 } 552 } 553 554 // 555 // Release any FB HW resources 556 // 557 if (pMemory->pHwResource) 558 { 559 if (--pMemory->pHwResource->refCount == 0) 560 { 561 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 562 NvBool bHostVgpuDeviceExists = _memCheckHostVgpuDeviceExists(pGpu); 563 564 if ((pMemory->categoryClassId == NV01_MEMORY_SYSTEM && memmgrComprSupported(pMemoryManager, ADDR_SYSMEM)) || 565 (bHostVgpuDeviceExists && (pMemory->pHwResource->isGuestAllocated))) 566 { 567 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 568 if (pFbAllocInfo == NULL) 569 { 570 NV_ASSERT(0); 571 status = NV_ERR_NO_MEMORY; 572 goto done; 573 } 574 575 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 576 if (pFbAllocPageFormat == NULL) { 577 NV_ASSERT(0); 578 status = NV_ERR_NO_MEMORY; 579 goto done; 580 } 581 582 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 583 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 584 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 585 586 pFbAllocInfo->pageFormat->type = pMemory->Type; 587 pFbAllocInfo->pageFormat->attr = pMemory->Attr; 588 pFbAllocInfo->pageFormat->attr2 = pMemory->Attr2; 589 pFbAllocInfo->hwResId = memdescGetHwResId(pMemory->pMemDesc); 590 pFbAllocInfo->size = pMemory->Length; 591 pFbAllocInfo->format = memdescGetPteKind(pMemory->pMemDesc); 592 593 // 594 // Note that while freeing duped memory under a device, the 595 // device may not be the memory owning device. Hence, always use 596 // memory owning device (pMemDesc->pGpu) to free HW resources. 597 // 598 status = memmgrFreeHwResources(pMemory->pMemDesc->pGpu, pMemoryManager, pFbAllocInfo); 599 NV_ASSERT(status == NV_OK); 600 } 601 portMemFree(pMemory->pHwResource); 602 } 603 } 604 605 NV_ASSERT_OK_OR_GOTO(status, btreeUnlink(&pMemory->Node, &pDevice->DevMemoryTable), done); 606 607 pMemory->pMemDesc->DupCount--; 608 609 // Choose the new owner 610 if (pMemory->isMemDescOwner) 611 { 612 (pMemory->dupListItem.pNext)->isMemDescOwner = NV_TRUE; 613 } 614 // Remove from circular list tracking dup/sharing of pMemDesc 615 pMemory->dupListItem.pPrev->dupListItem.pNext = pMemory->dupListItem.pNext; 616 pMemory->dupListItem.pNext->dupListItem.pPrev = pMemory->dupListItem.pPrev; 617 pMemory->dupListItem.pNext = pMemory->dupListItem.pPrev = NULL; 618 619 pMemory->bConstructed = NV_FALSE; 620 621 done: 622 portMemFree(pFbAllocPageFormat); 623 portMemFree(pFbAllocInfo); 624 625 // The unmap call(s) above may have changed the broadcast state so restore it here 626 gpuSetThreadBcState(pGpu, pMemory->bBcResource); 627 } 628 629 void 630 memDestructCommon_IMPL 631 ( 632 Memory *pMemory 633 ) 634 { 635 OBJGPU *pGpu = pMemory->pGpu; 636 RsResourceRef *pResourceRef = RES_GET_REF(pMemory); 637 RsResourceRef *pParentRef = pResourceRef->pParentRef; 638 RsClient *pClient = RES_GET_CLIENT(pMemory); 639 NvHandle hClient = pClient->hClient; 640 NvHandle hParent = pParentRef->hResource; 641 NvHandle hMemory = RES_GET_HANDLE(pMemory); 642 643 if (!pMemory->bConstructed) 644 return; 645 646 NV_ASSERT_OK(memdescDeregisterFromGSP(pGpu, hClient, hParent, hMemory)); 647 648 // Do device specific teardown if we have a device 649 if (pMemory->pDevice != NULL) 650 { 651 _memDestructCommonWithDevice(pMemory); 652 } 653 else 654 { 655 pMemory->bConstructed = NV_FALSE; 656 } 657 658 if (pMemory->KernelVAddr != NvP64_NULL) 659 { 660 memdescUnmap(pMemory->pMemDesc, NV_TRUE, osGetCurrentProcess(), 661 pMemory->KernelVAddr, pMemory->KernelMapPriv); 662 pMemory->KernelVAddr = NvP64_NULL; 663 pMemory->KernelMapPriv = NvP64_NULL; 664 } 665 } 666 667 NV_STATUS 668 memGetByHandleAndDevice_IMPL 669 ( 670 RsClient *pClient, 671 NvHandle hMemory, 672 NvHandle hDevice, 673 Memory **ppMemory 674 ) 675 { 676 NV_STATUS status; 677 678 status = memGetByHandle(pClient, hMemory, ppMemory); 679 if (status != NV_OK) 680 return status; 681 682 if (hDevice != RES_GET_HANDLE((*ppMemory)->pDevice)) 683 { 684 *ppMemory = NULL; 685 return NV_ERR_OBJECT_NOT_FOUND; 686 } 687 688 return NV_OK; 689 } 690 691 NV_STATUS 692 memGetByHandle_IMPL 693 ( 694 RsClient *pClient, 695 NvHandle hMemory, 696 Memory **ppMemory 697 ) 698 { 699 RsResourceRef *pResourceRef; 700 NV_STATUS status; 701 702 *ppMemory = NULL; 703 704 status = clientGetResourceRef(pClient, hMemory, &pResourceRef); 705 if (status != NV_OK) 706 return status; 707 708 *ppMemory = dynamicCast(pResourceRef->pResource, Memory); 709 710 if (*ppMemory == NULL) 711 return NV_ERR_INVALID_OBJECT_HANDLE; 712 713 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(*ppMemory, NV_FALSE)); 714 715 return NV_OK; 716 } 717 718 NV_STATUS 719 memGetByHandleAndGroupedGpu_IMPL 720 ( 721 RsClient *pClient, 722 NvHandle hMemory, 723 OBJGPU *pGpu, 724 Memory **ppMemory 725 ) 726 { 727 Device *pDevice; 728 NV_STATUS status; 729 730 // Get device handle 731 status = deviceGetByInstance(pClient, gpuGetDeviceInstance(pGpu), &pDevice); 732 if (status != NV_OK) 733 return NV_ERR_INVALID_OBJECT_HANDLE; 734 735 return memGetByHandleAndDevice(pClient, hMemory, RES_GET_HANDLE(pDevice), ppMemory); 736 } 737 738 NV_STATUS 739 memIsReady_IMPL 740 ( 741 Memory *pMemory, 742 NvBool bCopyConstructorContext 743 ) 744 { 745 if (pMemory->pMemDesc == NULL) 746 return NV_ERR_INVALID_OBJECT; 747 748 return NV_OK; 749 } 750 751 NV_STATUS 752 memControl_IMPL 753 ( 754 Memory *pMemory, 755 CALL_CONTEXT *pCallContext, 756 RS_RES_CONTROL_PARAMS_INTERNAL *pParams 757 ) 758 { 759 RmCtrlParams *pRmCtrlParams = pParams->pLegacyParams; 760 761 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE)); 762 763 if (!pMemory->pGpu) 764 return NV_ERR_INVALID_OBJECT_PARENT; 765 766 if (REF_VAL(NVXXXX_CTRL_CMD_CLASS, pParams->cmd) == NV04_MEMORY) 767 { 768 if (pMemory->categoryClassId == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR) 769 return NV_ERR_NOT_SUPPORTED; 770 } 771 772 pRmCtrlParams->pGpu = pMemory->pGpu; 773 774 gpuSetThreadBcState(pMemory->pGpu, pMemory->bBcResource); 775 776 return resControl_IMPL(staticCast(pMemory, RsResource), pCallContext, pParams); 777 } 778 779 NV_STATUS 780 memCopyConstruct_IMPL 781 ( 782 Memory *pMemory, 783 CALL_CONTEXT *pCallContext, 784 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 785 ) 786 { 787 RsClient *pDstClient = pCallContext->pClient; 788 RsClient *pSrcClient = pParams->pSrcClient; 789 RsResourceRef *pDstRef = pCallContext->pResourceRef; 790 RsResourceRef *pSrcRef = pParams->pSrcRef; 791 Memory *pMemorySrc = dynamicCast(pSrcRef->pResource, Memory); 792 Memory *pMemoryDst = pMemory; 793 OBJGPU *pSrcGpu = NULL; 794 OBJGPU *pDstGpu = NULL; 795 NV_STATUS status = NV_OK; 796 NvBool bReleaseGpuLock = NV_FALSE; 797 Device *pSrcDevice = NULL; 798 Device *pDstDevice = NULL; 799 Subdevice *pSrcSubDevice = NULL; 800 Subdevice *pDstSubDevice = NULL; 801 RsResourceRef *pSrcParentRef = pSrcRef->pParentRef; 802 RsResourceRef *pDstParentRef = pDstRef->pParentRef; 803 804 NV_ASSERT_OR_RETURN(pSrcParentRef != NULL, NV_ERR_INVALID_OBJECT_PARENT); 805 NV_ASSERT_OR_RETURN(pDstParentRef != NULL, NV_ERR_INVALID_OBJECT_PARENT); 806 NV_ASSERT_OR_RETURN(pMemorySrc != NULL, NV_ERR_INVALID_OBJECT_HANDLE); 807 808 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemorySrc, NV_TRUE)); 809 810 // 811 // Must return early when parent is Client. 812 // This copy constructor is very device-specific so it is up 813 // to the device-less Memory subclasses to define their own dup behavior. 814 // 815 if (RES_GET_CLIENT_HANDLE(pMemoryDst) == RES_GET_PARENT_HANDLE(pMemoryDst)) 816 { 817 NV_ASSERT_OR_RETURN(RES_GET_CLIENT_HANDLE(pMemorySrc) == 818 RES_GET_PARENT_HANDLE(pMemorySrc), 819 NV_ERR_INVALID_OBJECT_PARENT); 820 return NV_OK; 821 } 822 823 pSrcGpu = pMemorySrc->pGpu; 824 pDstGpu = pMemoryDst->pGpu; 825 pSrcDevice = pMemorySrc->pDevice; 826 pDstDevice = pMemoryDst->pDevice; 827 pSrcSubDevice = pMemorySrc->pSubDevice; 828 pDstSubDevice = pMemoryDst->pSubDevice; 829 830 // Only children of device are supported 831 NV_ASSERT_OR_RETURN(pSrcDevice != NULL, NV_ERR_INVALID_OBJECT_PARENT); 832 NV_ASSERT_OR_RETURN(pDstDevice != NULL, NV_ERR_INVALID_OBJECT_PARENT); 833 834 if (!!pSrcSubDevice != !!pDstSubDevice) 835 { 836 NV_PRINTF(LEVEL_ERROR, "Parent type mismatch between Src and Dst objects" 837 "Both should be either device or subDevice\n"); 838 return NV_ERR_INVALID_OBJECT_PARENT; 839 } 840 841 // RS-TODO: This should use pMemorySrc->bBcResource when adding full support for subdevice duping 842 gpuSetThreadBcState(pSrcGpu, NV_TRUE); 843 844 if (!rmGpuLockIsOwner() && 845 !(rmDeviceGpuLockIsOwner(pSrcGpu->gpuInstance) && 846 rmDeviceGpuLockIsOwner(pDstGpu->gpuInstance))) 847 { 848 // LOCK: acquire GPUs lock 849 if ((status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_MEM)) != NV_OK) 850 { 851 NV_PRINTF(LEVEL_ERROR, 852 "Failed to acquire GPU locks, error 0x%x\n", status); 853 return status; 854 } 855 856 bReleaseGpuLock = NV_TRUE; 857 } 858 859 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, 860 memCheckCopyPermissions(pMemorySrc, pDstGpu, pDstClient->hClient), done); 861 862 // Initialize Memory 863 pMemoryDst->categoryClassId = pMemorySrc->categoryClassId; 864 pMemoryDst->Length = pMemorySrc->Length; 865 pMemoryDst->HeapOwner = pMemorySrc->HeapOwner; 866 pMemoryDst->pHeap = pMemorySrc->pHeap; 867 pMemoryDst->pMemDesc = pMemorySrc->pMemDesc; 868 pMemoryDst->KernelVAddr = NvP64_NULL; 869 pMemoryDst->KernelMapPriv = NvP64_NULL; 870 pMemoryDst->Attr = pMemorySrc->Attr; 871 pMemoryDst->Attr2 = pMemorySrc->Attr2; 872 pMemoryDst->Pitch = pMemorySrc->Pitch; 873 pMemoryDst->Type = pMemorySrc->Type; 874 pMemoryDst->Flags = pMemorySrc->Flags; 875 pMemoryDst->tag = pMemorySrc->tag; 876 pMemoryDst->pHwResource = pMemorySrc->pHwResource; 877 pMemoryDst->isMemDescOwner = NV_FALSE; 878 pMemoryDst->bRpcAlloc = pMemorySrc->bRpcAlloc; 879 880 // Link in the new device memory mapping 881 pMemoryDst->Node.keyStart = RES_GET_HANDLE(pMemoryDst); 882 pMemoryDst->Node.keyEnd = RES_GET_HANDLE(pMemoryDst); 883 pMemoryDst->Node.Data = pMemoryDst; 884 885 status = btreeInsert(&pMemoryDst->Node, &pDstDevice->DevMemoryTable); 886 if (status != NV_OK) 887 goto done; 888 889 { 890 OBJGPU *pGpu = pDstGpu; // Need pGpu for SLI loop 891 892 gpuSetThreadBcState(pDstGpu, NV_TRUE); 893 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 894 if (memdescGetPageSize64(memdescGetMemDescFromGpu(pMemoryDst->pMemDesc, pGpu), AT_GPU) == 0) 895 { 896 status = memmgrSetMemDescPageSize_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), 897 memdescGetMemDescFromGpu(pMemoryDst->pMemDesc, pGpu), 898 AT_GPU, RM_ATTR_PAGE_SIZE_DEFAULT); 899 NV_ASSERT(status == NV_OK); 900 } 901 SLI_LOOP_END 902 } 903 904 // 905 // ref-count increments for shared structs after all places where we 906 // could return early. 907 // 908 if (pMemoryDst->pHwResource != NULL) 909 pMemoryDst->pHwResource->refCount++; 910 911 memdescAddRef(pMemoryDst->pMemDesc); 912 pMemoryDst->pMemDesc->DupCount++; 913 if (pMemoryDst->pMemDesc->Allocated) 914 pMemoryDst->pMemDesc->Allocated++; 915 916 // Insert pMemoryDst after pMemorySrc in circular list to track dup/sharing of pMemDesc 917 pMemoryDst->dupListItem.pNext = pMemorySrc->dupListItem.pNext; 918 pMemoryDst->dupListItem.pPrev = pMemorySrc; 919 pMemorySrc->dupListItem.pNext = pMemoryDst; 920 pMemoryDst->dupListItem.pNext->dupListItem.pPrev = pMemoryDst; 921 922 done: 923 924 // If the original allocation was RPCed, also send the Dup. 925 if (pMemory->bRpcAlloc && (IS_VIRTUAL(pSrcGpu) || IS_GSP_CLIENT(pSrcGpu))) 926 { 927 NV_RM_RPC_DUP_OBJECT(pSrcGpu, pDstClient->hClient, pDstParentRef->hResource, pDstRef->hResource, 928 pSrcClient->hClient, pSrcRef->hResource, 0, 929 NV_FALSE, // do not automatically issue RPC_FREE on object free 930 NULL, 931 status); 932 NV_ASSERT(status == NV_OK); 933 } 934 935 // UNLOCK: release GPUs lock 936 if (bReleaseGpuLock) 937 { 938 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 939 } 940 941 pMemory->bConstructed = (status == NV_OK); 942 return status; 943 } 944 945 NV_STATUS 946 memGetMemInterMapParams_IMPL 947 ( 948 Memory *pMemory, 949 RMRES_MEM_INTER_MAP_PARAMS *pParams 950 ) 951 { 952 OBJGPU *pGpu = pParams->pGpu; 953 RsResourceRef *pMemoryRef = pParams->pMemoryRef; 954 955 FlaMemory *pFlaMemory; 956 957 MEMORY_DESCRIPTOR *pSrcMemDesc = pMemory->pMemDesc; 958 Device *pDevice; 959 Subdevice *pSubdevice; 960 NvBool bcState = gpumgrGetBcEnabledStatus(pGpu); 961 962 // Don't expect to use default, but safe thing to do is set src=dest 963 NvHandle hMemoryDevice = 0; 964 OBJGPU *pSrcGpu = pGpu; 965 966 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE)); 967 968 if (pMemoryRef->pParentRef != NULL) 969 { 970 pDevice = dynamicCast(pMemoryRef->pParentRef->pResource, Device); 971 if (pDevice != NULL) 972 { 973 pSrcGpu = GPU_RES_GET_GPU(pDevice); 974 hMemoryDevice = RES_GET_HANDLE(pDevice); 975 GPU_RES_SET_THREAD_BC_STATE(pDevice); 976 } 977 else 978 { 979 pSubdevice = dynamicCast(pMemoryRef->pParentRef->pResource, Subdevice); 980 if (pSubdevice != NULL) 981 { 982 pSrcGpu = GPU_RES_GET_GPU(pSubdevice); 983 hMemoryDevice = RES_GET_HANDLE(pSubdevice->pDevice); 984 GPU_RES_SET_THREAD_BC_STATE(pSubdevice); 985 } 986 } 987 } 988 989 pParams->pSrcGpu = pSrcGpu; 990 pParams->hMemoryDevice = hMemoryDevice; 991 992 // 993 // Restore pGpu's bcState in case it was overwritten above (i.e., 994 // the case that hMemoryDevice and hBroadcastDevice are the same 995 // device, but a unicast mapping was desired). 996 // 997 gpumgrSetBcEnabledStatus(pGpu, bcState); 998 999 // 1000 // Mapping Guest allocated memory in PF is not supported 1001 // 1002 if (pSrcMemDesc->pGpu != pGpu && gpuIsSriovEnabled(pGpu) && 1003 !(memdescGetFlag(pSrcMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))) 1004 { 1005 // 1006 // Memory allocated by pSrcMemDesc->pGpu needs to be 1007 // remapped for pGpu as requested by client. 1008 // 1009 pParams->bDmaMapNeeded = NV_TRUE; 1010 } 1011 1012 pParams->pSrcMemDesc = pSrcMemDesc; 1013 1014 pFlaMemory = dynamicCast(pMemoryRef->pResource, FlaMemory); 1015 if (pFlaMemory != NULL) 1016 { 1017 pParams->pSrcGpu = gpumgrGetGpu(pFlaMemory->peerGpuInst); 1018 pParams->bFlaMapping = NV_TRUE; 1019 1020 NV_PRINTF(LEVEL_INFO, "FLA memory imported as (%s) with exportGpu:%x \n", 1021 (pParams->pSrcGpu != pGpu ? " P2P " : " LOOPBACK "), 1022 pFlaMemory->peerDeviceInst); 1023 } 1024 1025 return NV_OK; 1026 } 1027 1028 NV_STATUS 1029 memGetMemoryMappingDescriptor_IMPL 1030 ( 1031 Memory *pMemory, 1032 MEMORY_DESCRIPTOR **ppMemDesc 1033 ) 1034 { 1035 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE)); 1036 if (pMemory->pGpu != NULL) 1037 { 1038 *ppMemDesc = memdescGetMemDescFromGpu(pMemory->pMemDesc, pMemory->pGpu); 1039 } 1040 else 1041 { 1042 *ppMemDesc = pMemory->pMemDesc; 1043 } 1044 return NV_OK; 1045 } 1046 1047 NV_STATUS 1048 memIsDuplicate_IMPL 1049 ( 1050 Memory *pMemory, 1051 NvHandle hMemory, 1052 NvBool *pDuplicate 1053 ) 1054 { 1055 RsClient *pClient = RES_GET_CLIENT(pMemory); 1056 Memory *pMemory1; 1057 1058 NV_CHECK_OK_OR_RETURN(LEVEL_SILENT, 1059 memIsReady(pMemory, NV_FALSE)); 1060 1061 NV_CHECK_OK_OR_RETURN(LEVEL_SILENT, 1062 memGetByHandle(pClient, hMemory, &pMemory1)); 1063 1064 // 1065 // Do not dereference pMemdesc here. We only take RMAPI RO lock and 1066 // client lock in this context. 1067 // 1068 1069 *pDuplicate = (pMemory->pMemDesc == pMemory1->pMemDesc); 1070 1071 return NV_OK; 1072 } 1073