1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "gpu/mem_mgr/mem_mgr.h" 25 #include "gpu/mem_mgr/heap.h" 26 #include "gpu/mem_sys/kern_mem_sys.h" 27 #include "gpu/mem_mgr/mem_utils.h" 28 #include "gpu/mem_mgr/ce_utils.h" 29 #include "mem_mgr/video_mem.h" 30 #include "gpu/mem_mgr/fbsr.h" 31 #include "gpu/mmu/kern_gmmu.h" 32 #include "gpu/bus/kern_bus.h" 33 #include "gpu/bif/kernel_bif.h" 34 #include "core/locks.h" 35 #include "vgpu/vgpu_util.h" 36 #include "virtualization/kernel_vgpu_mgr.h" 37 #include "vgpu/rpc.h" 38 #include "core/thread_state.h" 39 #include "nvRmReg.h" 40 #include "gpu/fsp/kern_fsp.h" 41 #include "gpu/pmu/kern_pmu.h" 42 #include "gpu/mem_mgr/phys_mem_allocator/numa.h" 43 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 44 #include "kernel/rmapi/rs_utils.h" 45 #include "rmapi/rmapi_utils.h" 46 #include "mmu/gmmu_fmt.h" 47 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 48 #include "class/cl503c.h" 49 #include "class/cl906f.h" // GF100_CHANNEL_GPFIFO 50 #include "os/os.h" 51 #include "gpu/gsp/kernel_gsp.h" 52 #include "gpu/conf_compute/conf_compute.h" 53 #include "platform/sli/sli.h" 54 55 #include "class/cl0050.h" 56 57 static NV_STATUS _memmgrCreateFBSR(MemoryManager *pMemoryManager, NvU32); 58 static NV_STATUS _memmgrCreateChildObjects(MemoryManager *pMemoryManager); 59 static void _memmgrInitRegistryOverrides(OBJGPU *pGpu, MemoryManager *pMemoryManager); 60 static NV_STATUS _memmgrInitMIGMemoryPartitionHeap(OBJGPU *pGpu, MemoryManager *pMemoryManager, 61 NvU32 swizzId, NV_RANGE *pAddrRange, 62 Heap **ppMemoryPartitionHeap); 63 static NV_STATUS _memmgrAllocInternalClientObjects(OBJGPU *pGpu, 64 MemoryManager *pMemoryManager); 65 static void _memmgrFreeInternalClientObjects(MemoryManager *pMemoryManager); 66 67 #define MEMUTILS_CHANNEL_GPFIFO_SIZE (NV906F_GP_ENTRY__SIZE * MEMUTILS_NUM_GPFIFIO_ENTRIES) 68 69 NV_STATUS 70 memmgrConstructEngine_IMPL 71 ( 72 OBJGPU *pGpu, 73 MemoryManager *pMemoryManager, 74 ENGDESCRIPTOR engDesc 75 ) 76 { 77 NV_STATUS rmStatus; 78 79 pMemoryManager->overrideInitHeapMin = 0; 80 pMemoryManager->overrideHeapMax = ~0ULL; 81 pMemoryManager->Ram.fbOverrideSizeMb = ~0ULL; 82 83 // Create the children 84 rmStatus = _memmgrCreateChildObjects(pMemoryManager); 85 if (rmStatus != NV_OK) 86 return rmStatus; 87 88 pMemoryManager->MIGMemoryPartitioningInfo.hClient = NV01_NULL_OBJECT; 89 pMemoryManager->MIGMemoryPartitioningInfo.hDevice = NV01_NULL_OBJECT; 90 pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice = NV01_NULL_OBJECT; 91 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = NV_RANGE_EMPTY; 92 93 return NV_OK; 94 } 95 96 void 97 memmgrDestruct_IMPL 98 ( 99 MemoryManager *pMemoryManager 100 ) 101 { 102 NvU32 i; 103 104 for (i = 0; i < NUM_FBSR_TYPES; i++) 105 { 106 objDelete(pMemoryManager->pFbsr[i]); 107 pMemoryManager->pFbsr[i] = NULL; 108 } 109 110 objDelete(pMemoryManager->pHeap); 111 pMemoryManager->pHeap = NULL; 112 113 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = NV_RANGE_EMPTY; 114 } 115 116 static void 117 _memmgrInitRegistryOverrides(OBJGPU *pGpu, MemoryManager *pMemoryManager) 118 { 119 NvU32 data32; 120 121 // Check for ram size override. 122 if ((osReadRegistryDword(pGpu, NV_REG_STR_OVERRIDE_FB_SIZE, &data32) == NV_OK) && 123 (data32 != 0)) 124 { 125 NV_PRINTF(LEVEL_WARNING, "Regkey %s = %dM\n", 126 NV_REG_STR_OVERRIDE_FB_SIZE, data32); 127 // Used to override heap sizing at create 128 pMemoryManager->Ram.fbOverrideSizeMb = data32; 129 } 130 else 131 { 132 pMemoryManager->Ram.fbOverrideSizeMb = ~0ULL; 133 } 134 135 // 136 // Scrub on Free is enabled by default for GK110+ 137 // The reg key will be used to disable the scrub on free 138 // 139 if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DISABLE_SCRUB_ON_FREE, 140 &data32) == NV_OK) && data32) 141 { 142 pMemoryManager->bScrubOnFreeEnabled = NV_FALSE; 143 } 144 145 if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DISABLE_FAST_SCRUBBER, 146 &data32) == NV_OK) && data32) 147 { 148 pMemoryManager->bFastScrubberEnabled = NV_FALSE; 149 } 150 151 if (NV_OK == osReadRegistryDword(pGpu, NV_REG_STR_RM_SYSMEM_PAGE_SIZE, &data32)) 152 { 153 switch (data32) 154 { 155 case RM_PAGE_SIZE: 156 case RM_PAGE_SIZE_64K: 157 case RM_PAGE_SIZE_HUGE: 158 case RM_PAGE_SIZE_512M: 159 break; 160 default: 161 NV_ASSERT(0); 162 NV_PRINTF(LEVEL_ERROR, 163 "Sysmem page size 0x%x not supported! Defaulting to 4KB\n", 164 data32); 165 data32 = RM_PAGE_SIZE; 166 } 167 pMemoryManager->sysmemPageSize = data32; 168 } 169 else 170 { 171 pMemoryManager->sysmemPageSize = RM_PAGE_SIZE; 172 173 } 174 175 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ALLOW_SYSMEM_LARGE_PAGES, &data32) == NV_OK) 176 { 177 pMemoryManager->bAllowSysmemHugePages = data32 ? NV_TRUE : NV_FALSE; 178 } 179 else 180 { 181 pMemoryManager->bAllowSysmemHugePages = NV_FALSE; 182 } 183 184 // This key should not be used on physical (GSP) RM. 185 if (!RMCFG_FEATURE_PLATFORM_GSP) 186 { 187 // Allow user to increase size of RM reserved heap via a regkey 188 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_INCREASE_RSVD_MEMORY_SIZE_MB, 189 &data32) == NV_OK) 190 { 191 pMemoryManager->rsvdMemorySizeIncrement = (NvU64)data32 << 20; 192 NV_PRINTF(LEVEL_ERROR, 193 "User specified increase in reserved size = %d MBs\n", 194 data32); 195 } 196 } 197 198 if (osReadRegistryDword(pGpu, 199 NV_REG_STR_RM_DISABLE_NONCONTIGUOUS_ALLOCATION, 200 &data32) == NV_OK) 201 { 202 if (data32 == NV_REG_STR_RM_DISABLE_NONCONTIGUOUS_ALLOCATION_TRUE) 203 { 204 pMemoryManager->bAllowNoncontiguousAllocation = NV_FALSE; 205 } 206 } 207 208 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_PAGED_DMA, &data32) == NV_OK) 209 { 210 pMemoryManager->bEnableFbsrPagedDma = !!data32; 211 } 212 213 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_FILE_MODE, &data32) == NV_OK) 214 { 215 if (data32 && RMCFG_FEATURE_PLATFORM_UNIX) 216 { 217 pMemoryManager->bEnableFbsrFileMode = NV_TRUE; 218 } 219 } 220 221 // 222 // Override PMA enable. PDB_PROP_FB_PMA_ENABLED is reconciled with 223 // PDB_PROP_FB_PLATFORM_PMA_SUPPORT to decide whether to enable PMA. 224 // 225 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_PMA, &data32) == NV_OK) 226 { 227 if (data32 == NV_REG_STR_RM_ENABLE_PMA_YES) 228 { 229 pMemoryManager->bPmaEnabled = NV_TRUE; 230 } 231 else 232 { 233 pMemoryManager->bPmaEnabled = NV_FALSE; 234 } 235 } 236 237 if (RMCFG_FEATURE_PLATFORM_WINDOWS && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE)) 238 { 239 pMemoryManager->bFbsrWddmModeEnabled = NV_TRUE; 240 } 241 242 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_WDDM_MODE, &data32) == NV_OK) 243 { 244 pMemoryManager->bFbsrWddmModeEnabled = !!data32; 245 } 246 247 // 248 // Override PMA managed client page tables. 249 // NOTE: This is WAR for bug #s 1946145 and 1971628. 250 // This should be removed as part of heap removal and PMA refactor. 251 // 252 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_PMA_MANAGED_PTABLES, 253 &data32) == NV_OK) 254 { 255 if (data32 == NV_REG_STR_RM_ENABLE_PMA_MANAGED_PTABLES_NO) 256 { 257 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 258 } 259 } 260 261 if (osReadRegistryDword(pGpu, NV_REG_STR_DISABLE_GLOBAL_CE_UTILS, &data32) == NV_OK && 262 data32 == NV_REG_STR_DISABLE_GLOBAL_CE_UTILS_YES) 263 { 264 pMemoryManager->bDisableGlobalCeUtils = NV_TRUE; 265 } 266 267 pMemoryManager->bCePhysicalVidmemAccessNotSupported = gpuIsSelfHosted(pGpu); 268 } 269 270 NV_STATUS 271 memmgrStatePreInitLocked_IMPL 272 ( 273 OBJGPU *pGpu, 274 MemoryManager *pMemoryManager 275 ) 276 { 277 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL(pGpu)) 278 { 279 // 280 // Temporary hack to get OpenRM working without breaking SLI 281 // After fixing CORERM-4078, memmgrInitFbRegions() call should be removed from memsysStateInitLocked() 282 // and only left here 283 // 284 NV_ASSERT_OK_OR_RETURN(memmgrInitFbRegions(pGpu, pMemoryManager)); 285 } 286 287 // Determine the size of reserved memory 288 NV_ASSERT_OK_OR_RETURN(memmgrPreInitReservedMemory_HAL(pGpu, pMemoryManager)); 289 290 return NV_OK; 291 } 292 293 static NV_STATUS 294 memmgrTestCeUtils 295 ( 296 OBJGPU *pGpu, 297 MemoryManager *pMemoryManager 298 ) 299 { 300 MEMORY_DESCRIPTOR *pVidMemDesc = NULL; 301 MEMORY_DESCRIPTOR *pSysMemDesc = NULL; 302 TRANSFER_SURFACE vidSurface = {0}; 303 TRANSFER_SURFACE sysSurface = {0}; 304 NvU32 vidmemData = 0xAABBCCDD; 305 NvU32 sysmemData = 0x11223345; 306 NV_STATUS status; 307 308 NV_ASSERT_OR_RETURN(pMemoryManager->pCeUtils != NULL, NV_ERR_INVALID_STATE); 309 310 if (pMemoryManager->pCeUtils->pLiteKernelChannel != NULL) 311 { 312 // 313 // BUG 4167899: Temporarily skip test in case of lite mode 314 // It sometimes fails when called from acrGatherWprInformation_GM200() 315 // However, ACR is initialized without issues 316 // 317 return NV_OK; 318 } 319 320 NV_ASSERT_OK_OR_GOTO(status, 321 memdescCreate(&pVidMemDesc, pGpu, sizeof vidmemData, RM_PAGE_SIZE, NV_TRUE, ADDR_FBMEM, 322 NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE), 323 failed); 324 memdescTagAlloc(status, 325 NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_19, pVidMemDesc); 326 NV_ASSERT_OK_OR_GOTO(status, status, failed); 327 vidSurface.pMemDesc = pVidMemDesc; 328 329 NV_ASSERT_OK_OR_GOTO(status, 330 memdescCreate(&pSysMemDesc, pGpu, sizeof sysmemData, 0, NV_TRUE, ADDR_SYSMEM, 331 NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE), 332 failed); 333 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_138, 334 pSysMemDesc); 335 NV_ASSERT_OK_OR_GOTO(status, status, failed); 336 sysSurface.pMemDesc = pSysMemDesc; 337 338 NV_ASSERT_OK_OR_GOTO(status, memmgrMemWrite(pMemoryManager, &vidSurface, &vidmemData, sizeof vidmemData, TRANSFER_FLAGS_NONE), failed); 339 NV_ASSERT_OK_OR_GOTO(status, memmgrMemWrite(pMemoryManager, &sysSurface, &sysmemData, sizeof sysmemData, TRANSFER_FLAGS_NONE), failed); 340 NV_ASSERT_OK_OR_GOTO(status, memmgrMemCopy (pMemoryManager, &sysSurface, &vidSurface, sizeof vidmemData, TRANSFER_FLAGS_PREFER_CE), failed); 341 NV_ASSERT_OK_OR_GOTO(status, memmgrMemRead (pMemoryManager, &sysSurface, &sysmemData, sizeof sysmemData, TRANSFER_FLAGS_NONE), failed); 342 NV_ASSERT_TRUE_OR_GOTO(status, sysmemData == vidmemData, NV_ERR_INVALID_STATE, failed); 343 344 failed: 345 memdescFree(pVidMemDesc); 346 memdescDestroy(pVidMemDesc); 347 memdescFree(pSysMemDesc); 348 memdescDestroy(pSysMemDesc); 349 350 return status; 351 } 352 353 NV_STATUS 354 memmgrInitInternalChannels_IMPL 355 ( 356 OBJGPU *pGpu, 357 MemoryManager *pMemoryManager 358 ) 359 { 360 NV_ASSERT_OK_OR_RETURN(memmgrScrubHandlePostSchedulingEnable_HAL(pGpu, pMemoryManager)); 361 362 if (pMemoryManager->bDisableGlobalCeUtils || 363 pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) || 364 pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_ALL_INST_IN_SYSMEM) || 365 pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB) || 366 gpuIsCacheOnlyModeEnabled(pGpu) || 367 (IS_VIRTUAL(pGpu) && !IS_VIRTUAL_WITH_FULL_SRIOV(pGpu)) || 368 !IS_SILICON(pGpu) || 369 IsDFPGA(pGpu)) 370 { 371 NV_PRINTF(LEVEL_INFO, "Skipping global CeUtils creation (unsupported platform)\n"); 372 373 return NV_OK; 374 } 375 376 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_VIRTUALIZATION_MODE_HOST_VGPU) || 377 !memmgrIsPmaInitialized(pMemoryManager) || 378 RMCFG_FEATURE_PLATFORM_GSP || 379 IS_MIG_ENABLED(pGpu) || 380 gpuIsCCorApmFeatureEnabled(pGpu) || 381 IsSLIEnabled(pGpu) || 382 IsUnlinkedSLIEnabled(pGpu) || 383 gpuIsSelfHosted(pGpu) || 384 NVCPU_IS_PPC64LE) 385 { 386 // BUG 4167899: Temporarily skip CeUtils creation on platforms where it fails 387 NV_PRINTF(LEVEL_INFO, "Skipping global CeUtils creation\n"); 388 389 return NV_OK; 390 } 391 392 NV_PRINTF(LEVEL_INFO, "Initializing global CeUtils instance\n"); 393 394 NV_ASSERT_OK_OR_RETURN(memmgrInitCeUtils(pMemoryManager, NV_FALSE)); 395 396 return NV_OK; 397 } 398 399 NV_STATUS 400 memmgrDestroyInternalChannels_IMPL 401 ( 402 OBJGPU *pGpu, 403 MemoryManager *pMemoryManager 404 ) 405 { 406 NV_PRINTF(LEVEL_INFO, "Destroying global CeUtils instance\n"); 407 408 memmgrDestroyCeUtils(pMemoryManager, NV_FALSE); 409 410 NV_ASSERT_OK_OR_RETURN(memmgrScrubHandlePreSchedulingDisable_HAL(pGpu, pMemoryManager)); 411 412 return NV_OK; 413 } 414 415 static NV_STATUS 416 memmgrPostSchedulingEnableHandler 417 ( 418 OBJGPU *pGpu, 419 void *pUnusedData 420 ) 421 { 422 return memmgrInitInternalChannels(pGpu, GPU_GET_MEMORY_MANAGER(pGpu)); 423 } 424 425 static NV_STATUS 426 memmgrPreSchedulingDisableHandler 427 ( 428 OBJGPU *pGpu, 429 void *pUnusedData 430 ) 431 { 432 return memmgrDestroyInternalChannels(pGpu, GPU_GET_MEMORY_MANAGER(pGpu)); 433 } 434 435 NV_STATUS 436 memmgrStateInitLocked_IMPL 437 ( 438 OBJGPU *pGpu, 439 MemoryManager *pMemoryManager 440 ) 441 { 442 NV_STATUS status = NV_OK; 443 NvU32 i; 444 NvBool bDynamicPageOffliningDisable = NV_FALSE; 445 446 NV_ASSERT_OK_OR_RETURN(memmgrInitReservedMemory_HAL(pGpu, pMemoryManager, pMemoryManager->Ram.fbAddrSpaceSizeMb << 20)); 447 448 _memmgrInitRegistryOverrides(pGpu, pMemoryManager); 449 450 // 451 // Enable dynamic page blacklisting at this point before we call CreateHeap 452 // since it internally calls heapGetBlacklistPages which depends on this property 453 // 454 if (!bDynamicPageOffliningDisable) 455 memmgrEnableDynamicPageOfflining_HAL(pGpu, pMemoryManager); 456 457 memmgrScrubRegistryOverrides_HAL(pGpu, pMemoryManager); 458 459 memmgrScrubInit_HAL(pGpu, pMemoryManager); 460 NV_ASSERT_OK_OR_RETURN(kfifoAddSchedulingHandler(pGpu, 461 GPU_GET_KERNEL_FIFO(pGpu), 462 memmgrPostSchedulingEnableHandler, NULL, 463 memmgrPreSchedulingDisableHandler, NULL)); 464 465 // 466 // Allocate framebuffer heap. All memory must be allocated from here to keep the world 467 // consistent (N.B. the heap size has been reduced by the amount of instance memory). 468 // 469 status = memmgrCreateHeap(pMemoryManager); 470 if (status != NV_OK) 471 { 472 return status; 473 } 474 475 // 476 // Just set up the memory pool now (basic init stuff). Actual physical 477 // frames are *NOT* added to the pool at this stage. 478 // 479 status = memmgrPageLevelPoolsCreate(pGpu, pMemoryManager); 480 if (status != NV_OK) 481 { 482 return status; 483 } 484 485 // RMCONFIG: only if FBSR engine is enabled 486 if (RMCFG_MODULE_FBSR) 487 { 488 // 489 // If a configuration is not supported, do not initialize 490 // the corresponding fbsr engine. 491 // 492 if (pMemoryManager->bFbsrWddmModeEnabled) 493 { 494 pMemoryManager->fbsrStartMode = FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED; 495 } 496 else if (pMemoryManager->bEnableFbsrPagedDma) 497 { 498 pMemoryManager->fbsrStartMode = FBSR_TYPE_PAGED_DMA; 499 } 500 else if (pMemoryManager->bEnableFbsrFileMode) 501 { 502 pMemoryManager->fbsrStartMode = FBSR_TYPE_FILE; 503 } 504 else 505 { 506 pMemoryManager->fbsrStartMode = FBSR_TYPE_PERSISTENT; 507 } 508 509 for (i = pMemoryManager->fbsrStartMode; i < NUM_FBSR_TYPES; i++) 510 { 511 if (!pMemoryManager->bPersistentStandbyBuffer && 512 (i == FBSR_TYPE_PERSISTENT)) 513 { 514 continue; 515 } 516 517 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) && 518 (i == FBSR_TYPE_PAGED_DMA || i == FBSR_TYPE_DMA)) 519 { 520 continue; 521 } 522 523 status = fbsrInit_HAL(pGpu, pMemoryManager->pFbsr[i]); 524 525 // 526 // If one fbsr scheme failed, proceed to initializing the other 527 // fallback options. 528 // 529 if (status != NV_OK) 530 { 531 NV_PRINTF(LEVEL_WARNING, 532 "fbsrInit failed for supported type %d suspend-resume scheme\n", 533 i); 534 continue; 535 } 536 } 537 } 538 539 status = _memmgrAllocInternalClientObjects(pGpu, pMemoryManager); 540 if (status != NV_OK) 541 { 542 // 543 // TODO: Bug 3482892: Need a way to roll back StateInit 544 // steps in case of a failure 545 // WAR for now is to cleanup with memmgrStateDestroy(). 546 // 547 memmgrStateDestroy(pGpu, pMemoryManager); 548 return status; 549 } 550 551 return NV_OK; 552 } 553 554 NV_STATUS 555 memmgrVerifyGspDmaOps_IMPL 556 ( 557 OBJGPU *pGpu, 558 MemoryManager *pMemoryManager 559 ) 560 { 561 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 562 NV_STATUS status = NV_OK; 563 MEMORY_DESCRIPTOR *pMemDesc; 564 NvU8 *pTestBuffer; 565 NvU32 testData = 0xdeadbeef; 566 TRANSFER_SURFACE surf = {0}; 567 568 // 569 // Return early if CPU access to CPR vidmem is allowed as GSP DMA 570 // is not needed in this case 571 // 572 if (!kbusIsBarAccessBlocked(pKernelBus)) 573 return NV_OK; 574 575 pTestBuffer = portMemAllocNonPaged(4096); 576 NV_ASSERT_OR_RETURN(pTestBuffer != NULL, NV_ERR_INSUFFICIENT_RESOURCES); 577 578 portMemSet(pTestBuffer, 0, 4096); 579 580 status = memdescCreate(&pMemDesc, pGpu, RM_PAGE_SIZE, RM_PAGE_SIZE, 581 NV_TRUE, ADDR_FBMEM, NV_MEMORY_UNCACHED, 0); 582 NV_ASSERT_OR_RETURN(status == NV_OK, status); 583 584 memdescTagAlloc(status, 585 NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_20, pMemDesc); 586 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 587 588 surf.pMemDesc = pMemDesc; 589 surf.offset = sizeof(NvU32); // Choosing a random offset 590 591 // Write test data to FB using GSP 592 status = memmgrMemWrite(pMemoryManager, &surf, &testData, sizeof(NvU32), 593 TRANSFER_FLAGS_NONE); 594 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 595 596 // Read the same location using GSP and confirm that GSP read is also working fine 597 status = memmgrMemRead(pMemoryManager, &surf, pTestBuffer, sizeof(NvU32), 598 TRANSFER_FLAGS_NONE); 599 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 600 601 if (((NvU32*)pTestBuffer)[0] != testData) 602 { 603 NV_PRINTF(LEVEL_ERROR, "####################################################\n"); 604 NV_PRINTF(LEVEL_ERROR, " Read back of data using GSP shows mismatch\n"); 605 NV_PRINTF(LEVEL_ERROR, " Test data: 0x%x Read Data: 0x%x\n", testData, ((NvU32*)pTestBuffer)[0]); 606 NV_PRINTF(LEVEL_ERROR, "####################################################\n"); 607 status = NV_ERR_INVALID_STATE; 608 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 609 } 610 else 611 { 612 NV_PRINTF(LEVEL_INFO, "####################################################\n"); 613 NV_PRINTF(LEVEL_INFO, " Read back of data using GSP confirms write\n"); 614 NV_PRINTF(LEVEL_INFO, "####################################################\n"); 615 } 616 617 failed: 618 memdescFree(pMemDesc); 619 memdescDestroy(pMemDesc); 620 portMemFree(pTestBuffer); 621 622 return status; 623 } 624 625 NV_STATUS 626 memmgrStateLoad_IMPL 627 ( 628 OBJGPU *pGpu, 629 MemoryManager *pMemoryManager, 630 NvU32 flags 631 ) 632 { 633 // If fbOverrideSizeMb is set, finish setting up the FB parameters now that state init has finished 634 memmgrFinishHandleSizeOverrides_HAL(pGpu, pMemoryManager); 635 636 if ((flags & GPU_STATE_FLAGS_PRESERVING) && 637 !(flags & GPU_STATE_FLAGS_GC6_TRANSITION)) 638 { 639 // 640 // Only do initialization scrubs (i.e. RM reserved region) on 641 // non-GC6 transitions since GC6 cycles leave FB powered. 642 // 643 memmgrScrubInit_HAL(pGpu, pMemoryManager); 644 } 645 646 // Dump FB regions 647 memmgrDumpFbRegions(pGpu, pMemoryManager); 648 649 return NV_OK; 650 } 651 652 NV_STATUS 653 memmgrStatePreUnload_IMPL 654 ( 655 OBJGPU *pGpu, 656 MemoryManager *pMemoryManager, 657 NvU32 flags 658 ) 659 { 660 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 661 662 NV_ASSERT((flags & GPU_STATE_FLAGS_PRESERVING) || pMemoryManager->zbcSurfaces == 0); 663 664 if ((flags & GPU_STATE_FLAGS_PRESERVING)) 665 { 666 // 667 // fifo won't send a PreSchedulingDisable callback on StateUnload 668 // destroy the channel manually, so that a CeUtils lite instance can be created for FBSR 669 // 670 memmgrDestroyCeUtils(pMemoryManager, !IS_VIRTUAL(pGpu)); 671 } 672 673 if (memmgrIsPmaEnabled(pMemoryManager) && 674 memmgrIsPmaSupportedOnPlatform(pMemoryManager) && 675 osNumaOnliningEnabled(pGpu->pOsGpuInfo) && 676 pKernelMemorySystem->memPartitionNumaInfo[0].bInUse) 677 { 678 pmaNumaOfflined(&pMemoryManager->pHeap->pmaObject); 679 } 680 681 return NV_OK; 682 } 683 684 NV_STATUS 685 memmgrStateUnload_IMPL 686 ( 687 OBJGPU *pGpu, 688 MemoryManager *pMemoryManager, 689 NvU32 flags 690 ) 691 { 692 if ((flags & GPU_STATE_FLAGS_PRESERVING) && 693 !(flags & GPU_STATE_FLAGS_GC6_TRANSITION)) 694 { 695 // 696 // Initialiation scrubs only happen during StateLoad on non-GC6 697 // transitions. 698 // 699 memmgrScrubDestroy_HAL(pGpu, pMemoryManager); 700 } 701 702 return NV_OK; 703 } 704 705 void 706 memmgrStateDestroy_IMPL 707 ( 708 OBJGPU *pGpu, 709 MemoryManager *pMemoryManager 710 ) 711 { 712 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 713 Heap *pHeap = MEMORY_MANAGER_GET_HEAP(pMemoryManager); 714 NvU32 i; 715 716 _memmgrFreeInternalClientObjects(pMemoryManager); 717 718 // Destroys the SW state of the page level pools 719 memmgrPageLevelPoolsDestroy(pGpu, pMemoryManager); 720 721 // Destroy the heap entirely, and all associated structures 722 if (pHeap) 723 { 724 kmemsysPreHeapDestruct_HAL(pGpu, pKernelMemorySystem); 725 726 objDelete(pHeap); 727 pMemoryManager->pHeap = NULL; 728 } 729 730 // RMCONFIG: only if FBSR engine is enabled 731 if (RMCFG_MODULE_FBSR) 732 { 733 // Cleanup fbsrReservedRanges 734 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE] != NULL) 735 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE]); 736 737 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE] != NULL) 738 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE]); 739 740 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP] != NULL) 741 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP]); 742 743 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR] != NULL) 744 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR]); 745 746 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR] != NULL) 747 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR]); 748 749 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE] != NULL) 750 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE]); 751 752 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE] = NULL; 753 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE] = NULL; 754 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP] = NULL; 755 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR] = NULL; 756 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR] = NULL; 757 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE] = NULL; 758 759 for (i = 0; i < NUM_FBSR_TYPES; i++) 760 { 761 fbsrDestroy_HAL(pGpu, pMemoryManager->pFbsr[i]); 762 } 763 } 764 if (memmgrIsLocalEgmEnabled(pMemoryManager)) 765 { 766 if (!IS_VIRTUAL_WITH_SRIOV(pGpu)) 767 { 768 kbusUnreserveP2PPeerIds_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), NVBIT(pMemoryManager->localEgmPeerId)); 769 } 770 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 771 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 772 } 773 774 kfifoRemoveSchedulingHandler(pGpu, GPU_GET_KERNEL_FIFO(pGpu), 775 memmgrPostSchedulingEnableHandler, NULL, 776 memmgrPreSchedulingDisableHandler, NULL); 777 memmgrScrubDestroy_HAL(pGpu, pMemoryManager); 778 } 779 780 static NV_STATUS 781 _memmgrCreateChildObjects 782 ( 783 MemoryManager *pMemoryManager 784 ) 785 { 786 NV_STATUS status = NV_OK; 787 788 // RMCONFIG: only if FBSR engine is enabled 789 if (RMCFG_MODULE_FBSR) 790 { 791 NvU32 i; 792 793 // Create FBSR object for every type RM supports. 794 for (i = 0; i < NUM_FBSR_TYPES; i++) 795 { 796 status = _memmgrCreateFBSR(pMemoryManager, i); 797 if (status != NV_OK) 798 { 799 return status; 800 } 801 } 802 } 803 804 return status; 805 } 806 807 NV_STATUS 808 memmgrCreateHeap_IMPL 809 ( 810 MemoryManager *pMemoryManager 811 ) 812 { 813 Heap *newHeap; 814 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 815 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 816 NvU64 rsvdSize; 817 NvU64 size; 818 NV_STATUS status = NV_OK; 819 820 // If we're using FB regions then rsvd memory is already marked as a reserved region 821 if ((pMemoryManager->Ram.numFBRegions == 0) || (IS_VIRTUAL_WITH_SRIOV(pGpu))) 822 { 823 if (pMemoryManager->bReservedMemAtBottom) 824 { 825 // rsvd memory is already accounted for in heapStart 826 rsvdSize = 0; 827 } 828 else 829 { 830 rsvdSize = pMemoryManager->rsvdMemorySize; 831 } 832 } 833 else 834 rsvdSize = 0; 835 836 // for vGPU, add extra FB tax incurred by host RM to reserved size 837 rsvdSize += memmgrGetFbTaxSize_HAL(pGpu, pMemoryManager); 838 839 // 840 // Fix up region descriptions to match with any FB override size 841 // 842 memmgrHandleSizeOverrides_HAL(pGpu, pMemoryManager); 843 844 // 845 // Calculate the FB heap size as the address space size, then deduct any reserved memory 846 // 847 size = pMemoryManager->Ram.fbAddrSpaceSizeMb << 20; 848 size -= NV_MIN(size, rsvdSize); 849 850 if((size != 0) || (pMemoryManager->bScanoutSysmem)) 851 { 852 status = objCreate(&newHeap, pMemoryManager, Heap); 853 if (status != NV_OK) 854 { 855 return status; 856 } 857 858 pMemoryManager->pHeap = newHeap; 859 860 if (memmgrIsPmaEnabled(pMemoryManager) && 861 memmgrIsPmaSupportedOnPlatform(pMemoryManager)) 862 { 863 portMemSet(&pMemoryManager->pHeap->pmaObject, 0, sizeof(pMemoryManager->pHeap->pmaObject)); 864 status = memmgrPmaInitialize(pGpu, pMemoryManager, &pMemoryManager->pHeap->pmaObject); 865 NV_ASSERT_OR_RETURN(status == NV_OK, status); 866 } 867 868 status = heapInit(pGpu, newHeap, 869 pMemoryManager->heapStartOffset, 870 size - pMemoryManager->heapStartOffset, HEAP_TYPE_RM_GLOBAL, GPU_GFID_PF, NULL); 871 NV_ASSERT_OK_OR_RETURN(status); 872 873 if ((memmgrIsPmaInitialized(pMemoryManager)) && (pMemoryManager->pHeap->bHasFbRegions)) 874 { 875 status = memmgrPmaRegisterRegions(pGpu, pMemoryManager, pMemoryManager->pHeap, 876 &pMemoryManager->pHeap->pmaObject); 877 NV_ASSERT_OR_RETURN(status == NV_OK, status); 878 } 879 880 NV_ASSERT_OK_OR_RETURN(memmgrValidateFBEndReservation_HAL(pGpu, pMemoryManager)); 881 882 NV_ASSERT_OK_OR_RETURN(memmgrReserveMemoryForPmu_HAL(pGpu, pMemoryManager)); 883 884 // Reserve vidmem for FSP usage, including FRTS, WPR2 885 status = memmgrReserveMemoryForFsp(pGpu, pMemoryManager); 886 if (status != NV_OK) 887 { 888 NV_PRINTF(LEVEL_ERROR, "Failed to reserve vidmem for WPR and FRTS.\n"); 889 return status; 890 } 891 892 if (!IsSLIEnabled(pGpu)) 893 { 894 // Do the actual blacklisting of pages from the heap 895 if (newHeap->blackListAddresses.count != 0) 896 { 897 status = heapBlackListPages(pGpu, newHeap); 898 899 if (status != NV_OK) 900 { 901 // Warn and continue 902 NV_PRINTF(LEVEL_WARNING, "Error 0x%x creating blacklist\n", 903 status); 904 } 905 } 906 } 907 908 kmemsysPostHeapCreate_HAL(pGpu, pKernelMemorySystem); 909 } 910 911 return status; 912 } 913 914 /* 915 * @brief Gets per-device suballocator. If it is not available, get shared heap. 916 * 917 * @param[in] pMemoryManager MemoryManager pointer 918 */ 919 Heap * 920 memmgrGetDeviceSuballocator_IMPL 921 ( 922 MemoryManager *pMemoryManager, 923 NvBool bForceSubheap 924 ) 925 { 926 927 if (!bForceSubheap) 928 { 929 // If no suballocator found, use heap 930 return MEMORY_MANAGER_GET_HEAP(pMemoryManager); 931 } 932 933 return NULL; 934 } 935 936 static NV_STATUS 937 _memmgrCreateFBSR 938 ( 939 MemoryManager *pMemoryManager, 940 NvU32 type 941 ) 942 { 943 OBJFBSR *pFbsr; 944 NV_STATUS status; 945 946 status = objCreate(&pFbsr, pMemoryManager, OBJFBSR); 947 if (status != NV_OK) 948 { 949 return status; 950 } 951 952 NV_ASSERT(pFbsr); 953 pMemoryManager->pFbsr[type] = pFbsr; 954 955 fbsrObjectInit(pFbsr, type); 956 957 return NV_OK; 958 } 959 960 static void 961 _memmgrFreeInternalClientObjects 962 ( 963 MemoryManager *pMemoryManager 964 ) 965 { 966 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 967 968 if (pMemoryManager->hThirdPartyP2P != 0) 969 { 970 pRmApi->Free(pRmApi, pMemoryManager->hClient, 971 pMemoryManager->hThirdPartyP2P); 972 pMemoryManager->hThirdPartyP2P = 0; 973 } 974 975 if (pMemoryManager->hClient != 0) 976 { 977 rmapiutilFreeClientAndDeviceHandles(pRmApi, 978 &pMemoryManager->hClient, 979 &pMemoryManager->hDevice, 980 &pMemoryManager->hSubdevice); 981 } 982 } 983 984 static NV_STATUS 985 _memmgrAllocInternalClientObjects 986 ( 987 OBJGPU *pGpu, 988 MemoryManager *pMemoryManager 989 ) 990 { 991 NV_STATUS status; 992 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 993 994 status = rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, 995 &pMemoryManager->hClient, 996 &pMemoryManager->hDevice, 997 &pMemoryManager->hSubdevice); 998 if (status != NV_OK) 999 { 1000 goto failed; 1001 } 1002 1003 { 1004 NV503C_ALLOC_PARAMETERS params; 1005 NvHandle hThirdPartyP2P = 0; 1006 1007 NV_ASSERT_OK_OR_GOTO(status, serverutilGenResourceHandle(pMemoryManager->hClient, 1008 &hThirdPartyP2P), 1009 failed); 1010 1011 portMemSet(¶ms, 0, sizeof(params)); 1012 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) 1013 { 1014 params.flags = NV503C_ALLOC_PARAMETERS_FLAGS_TYPE_NVLINK; 1015 } 1016 else 1017 { 1018 params.flags = NV503C_ALLOC_PARAMETERS_FLAGS_TYPE_BAR1; 1019 } 1020 status = pRmApi->AllocWithHandle(pRmApi, 1021 pMemoryManager->hClient, 1022 pMemoryManager->hSubdevice, 1023 hThirdPartyP2P, 1024 NV50_THIRD_PARTY_P2P, 1025 ¶ms, 1026 sizeof(params)); 1027 if (status != NV_OK) 1028 { 1029 NV_PRINTF(LEVEL_WARNING, "Error creating internal ThirdPartyP2P object: %x\n", 1030 status); 1031 pMemoryManager->hThirdPartyP2P = 0; 1032 } 1033 else 1034 { 1035 pMemoryManager->hThirdPartyP2P = hThirdPartyP2P; 1036 } 1037 1038 } 1039 1040 return NV_OK; 1041 1042 failed: 1043 _memmgrFreeInternalClientObjects(pMemoryManager); 1044 1045 return status; 1046 } 1047 1048 /*! 1049 * @brief Determine size of FB RAM which is used for RM internal allocations 1050 * and PMA. 1051 * 1052 * @param[out] pFbUsedSize FB used memory size 1053 * 1054 * @returns NV_OK 1055 */ 1056 NV_STATUS 1057 memmgrGetUsedRamSize_IMPL 1058 ( 1059 OBJGPU *pGpu, 1060 MemoryManager *pMemoryManager, 1061 NvU64 *pFbUsedSize 1062 ) 1063 { 1064 Heap *pHeap = GPU_GET_HEAP(pGpu); 1065 NvU64 heapFreeSpace, heapTotalSpace, pmaFreeSpace; 1066 1067 // 1068 // Determine free memory in FB and substract with total FB memory. 1069 // If PMA is initialized, then use the free memory size in PMA and 1070 // heap otherwise only use heap free memory for calculation. 1071 // 1072 heapGetFree(pHeap, &heapFreeSpace); 1073 heapGetSize(pHeap, &heapTotalSpace); 1074 if (memmgrIsPmaInitialized(pMemoryManager)) 1075 { 1076 pmaGetFreeMemory(&pHeap->pmaObject, &pmaFreeSpace); 1077 *pFbUsedSize = heapTotalSpace - heapFreeSpace - pmaFreeSpace; 1078 } 1079 else 1080 { 1081 *pFbUsedSize = heapTotalSpace - heapFreeSpace; 1082 } 1083 1084 // 1085 // GSP's WPR region has its own save/restore mechanism and does not need 1086 // to be accounted for in total FB size used - which is needed to find out 1087 // how much SYSMEM needs to be allocated to save all FB memory 1088 // 1089 if (IS_GSP_CLIENT(pGpu)) 1090 { 1091 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 1092 NvU64 gspWprRegionSize = pKernelGsp->pWprMeta->gspFwWprEnd - pKernelGsp->pWprMeta->gspFwWprStart; 1093 1094 *pFbUsedSize = *pFbUsedSize - gspWprRegionSize; 1095 } 1096 1097 return NV_OK; 1098 } 1099 1100 NV_STATUS 1101 memmgrAllocHwResources_IMPL 1102 ( 1103 OBJGPU *pGpu, 1104 MemoryManager *pMemoryManager, 1105 FB_ALLOC_INFO *pFbAllocInfo 1106 ) 1107 { 1108 MemoryManager *pMemoryManagerLoop; 1109 FB_ALLOC_INFO *pTempInfo = NULL; 1110 NvU32 skipFlag = (pFbAllocInfo->pageFormat->flags & NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC); 1111 NV_STATUS rmStatus = NV_OK; 1112 1113 pTempInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 1114 if (pTempInfo == NULL) 1115 { 1116 NV_ASSERT(0); 1117 return NV_ERR_NO_MEMORY; 1118 } 1119 1120 // vGPU: 1121 // 1122 // Since vGPU does all real hardware management in the 1123 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 1124 // we can skip the resource allocation step. 1125 // 1126 1127 if (IS_VIRTUAL(pGpu) && !vgpuIsGuestManagedHwAlloc(pGpu)) 1128 { 1129 pFbAllocInfo->pageFormat->flags |= NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC; 1130 } 1131 1132 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1133 { 1134 NV_STATUS tempStatus; 1135 *pTempInfo = *pFbAllocInfo; // struct copy 1136 1137 pMemoryManagerLoop = GPU_GET_MEMORY_MANAGER(pGpu); 1138 1139 tempStatus = memmgrAllocHal_HAL(pGpu, pMemoryManagerLoop, pTempInfo); 1140 // be sure to return an intermediate error 1141 if (NV_OK == rmStatus) 1142 rmStatus = tempStatus; 1143 } 1144 SLI_LOOP_END 1145 1146 *pFbAllocInfo = *pTempInfo; // struct copy 1147 portMemFree(pTempInfo); 1148 1149 pFbAllocInfo->pageFormat->flags &= ~NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC; 1150 pFbAllocInfo->pageFormat->flags |= skipFlag; 1151 1152 return rmStatus; 1153 } 1154 1155 NV_STATUS 1156 memmgrFreeHwResources_IMPL 1157 ( 1158 OBJGPU *pGpu, 1159 MemoryManager *pMemoryManager, 1160 FB_ALLOC_INFO *pFbAllocInfo 1161 ) 1162 { 1163 MemoryManager *pMemoryManagerLoop; 1164 NV_STATUS rmStatus = NV_OK; 1165 RMTIMEOUT timeout; 1166 FB_ALLOC_INFO *pTempInfo = NULL; 1167 1168 pTempInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 1169 if (pTempInfo == NULL) 1170 { 1171 NV_ASSERT(0); 1172 return NV_ERR_NO_MEMORY; 1173 } 1174 1175 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0); 1176 1177 if (IS_VIRTUAL(pGpu) && !vgpuIsGuestManagedHwAlloc(pGpu)) 1178 { 1179 pFbAllocInfo->pageFormat->flags |= NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC; 1180 } 1181 1182 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1183 { 1184 NV_STATUS tempStatus; 1185 pMemoryManagerLoop = GPU_GET_MEMORY_MANAGER(pGpu); 1186 1187 *pTempInfo = *pFbAllocInfo; 1188 1189 tempStatus = memmgrFreeHal_HAL(pGpu, pMemoryManagerLoop, pTempInfo, &timeout); 1190 // be sure to return an intermediate error 1191 if (NV_OK == rmStatus) 1192 rmStatus = tempStatus; 1193 1194 } 1195 SLI_LOOP_END 1196 1197 *pFbAllocInfo = *pTempInfo; 1198 portMemFree(pTempInfo); 1199 1200 return rmStatus; 1201 } 1202 1203 NvBool 1204 memmgrLargePageSupported_IMPL 1205 ( 1206 MemoryManager *pMemoryManager, 1207 NV_ADDRESS_SPACE addrSpace 1208 ) 1209 { 1210 NvBool isSupported = NV_FALSE; 1211 1212 if (addrSpace == ADDR_FBMEM || addrSpace == ADDR_VIRTUAL) 1213 { 1214 isSupported = NV_TRUE; 1215 } 1216 else if (addrSpace == ADDR_SYSMEM) 1217 { 1218 isSupported = (pMemoryManager->sysmemPageSize != RM_PAGE_SIZE); 1219 } 1220 else 1221 { 1222 NV_ASSERT(0); 1223 } 1224 1225 return isSupported; 1226 } 1227 1228 NvBool 1229 memmgrComprSupported_IMPL 1230 ( 1231 MemoryManager *pMemoryManager, 1232 NV_ADDRESS_SPACE addrSpace 1233 ) 1234 { 1235 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 1236 NvBool isSupported = NV_FALSE; 1237 1238 if (GPU_GET_KERNEL_GMMU(pGpu) != NULL) 1239 { 1240 if (memmgrLargePageSupported(pMemoryManager, addrSpace) || 1241 pMemoryManager->bSmallPageCompression) 1242 { 1243 if (addrSpace == ADDR_FBMEM || addrSpace == ADDR_VIRTUAL) 1244 { 1245 isSupported = NV_TRUE; 1246 } 1247 else if (addrSpace == ADDR_SYSMEM) 1248 { 1249 // Compression is allowed on vidmem or unified aperture (vidmem/sysmem is same w.r.t HW) 1250 isSupported = (gpuIsUnifiedMemorySpaceEnabled(pGpu) && 1251 pMemoryManager->bSysmemCompressionSupportDef); 1252 NV_PRINTF(LEVEL_INFO, "isSupported=%s\n", 1253 isSupported ? "NV_TRUE" : "NV_FALSE"); 1254 } 1255 else 1256 { 1257 NV_ASSERT(0); 1258 } 1259 } 1260 } 1261 1262 return isSupported; 1263 } 1264 1265 NV_ADDRESS_SPACE 1266 memmgrAllocGetAddrSpace_IMPL 1267 ( 1268 MemoryManager *pMemoryManager, 1269 NvU32 flags, 1270 NvU32 attr 1271 ) 1272 { 1273 NV_ADDRESS_SPACE addrSpace = ADDR_UNKNOWN; 1274 1275 if (flags & NVOS32_ALLOC_FLAGS_VIRTUAL) 1276 { 1277 addrSpace = ADDR_VIRTUAL; 1278 } 1279 else if (FLD_TEST_DRF(OS32, _ATTR, _LOCATION, _VIDMEM, attr)) 1280 { 1281 addrSpace = ADDR_FBMEM; 1282 } 1283 else 1284 { 1285 // In case location is SYSMEM or ANY, allocate in vidmem if protected flag is set. 1286 if (flags & NVOS32_ALLOC_FLAGS_PROTECTED) 1287 { 1288 addrSpace = ADDR_FBMEM; 1289 } 1290 else 1291 { 1292 addrSpace = ADDR_SYSMEM; 1293 } 1294 } 1295 1296 return addrSpace; 1297 } 1298 1299 NvU32 1300 memmgrGetMappableRamSizeMb_IMPL(MemoryManager *pMemoryManager) 1301 { 1302 return NvU64_LO32(pMemoryManager->Ram.mapRamSizeMb); 1303 } 1304 // 1305 // ZBC clear create/destroy routines. 1306 // 1307 1308 NV_STATUS 1309 memmgrFillMemdescForPhysAttr_IMPL 1310 ( 1311 OBJGPU *pGpu, 1312 MemoryManager *pMemoryManager, 1313 PMEMORY_DESCRIPTOR pMemDesc, 1314 ADDRESS_TRANSLATION addressTranslation, 1315 NvU64 *pOffset, 1316 NvU32 *pMemAperture, 1317 NvU32 *pMemKind, 1318 NvU32 *pZCullId, 1319 NvU32 *pGpuCacheAttr, 1320 NvU32 *pGpuP2PCacheAttr, 1321 NvU64 *contigSegmentSize 1322 ) 1323 { 1324 NvU64 surfOffset = *pOffset, surfBase, surfLimit; 1325 NvU32 zcbitmap; 1326 1327 surfBase = memdescGetPhysAddr(pMemDesc, addressTranslation, 0); 1328 surfLimit = surfBase + pMemDesc->Size - 1; 1329 *pMemKind = memdescGetPteKind(pMemDesc); 1330 1331 *pOffset = memdescGetPhysAddr(pMemDesc, addressTranslation, surfOffset); 1332 1333 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM ) 1334 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_VIDMEM; 1335 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM) 1336 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM; 1337 else if (memdescGetAddressSpace(pMemDesc) == ADDR_EGM) 1338 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM; 1339 else if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL ) 1340 { 1341 // 1342 // XXX we could theoretically find whatever phys mem object is plugged 1343 // in at surfOffset w/in the virt object... that'd mean scanning 1344 // pMemory->DmaMappingList 1345 // 1346 return NV_ERR_NOT_SUPPORTED; 1347 } 1348 else 1349 return NV_ERR_GENERIC; 1350 1351 if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1352 { 1353 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED; 1354 } 1355 else if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1356 { 1357 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED; 1358 } 1359 else 1360 { 1361 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN; 1362 } 1363 1364 if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1365 { 1366 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED; 1367 } 1368 else if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1369 { 1370 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED; 1371 } 1372 else 1373 { 1374 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN; 1375 } 1376 1377 zcbitmap = FB_HWRESID_ZCULL_VAL_FERMI(memdescGetHwResId(pMemDesc)); //bitmap form... need a scalar 1378 for ( *pZCullId = 0; zcbitmap; zcbitmap >>= 1, *pZCullId += 1) {;;;} 1379 *pZCullId -= 1; // side effect if there is no zcull id of setting ~0 1380 1381 *contigSegmentSize = surfLimit - (surfBase + surfOffset) + 1; 1382 1383 if ( !memdescGetContiguity(pMemDesc, addressTranslation)) 1384 { 1385 // XXX overly conservative. we could scan the PTEs to find out if more pages are contig. 1386 NvU64 surfOffsetLimitSame4KBPage = (4*1024)*((surfBase + surfOffset)/(4*1024)) + (4*1024) - 1; 1387 if ( surfLimit >= surfOffsetLimitSame4KBPage ) 1388 *contigSegmentSize = surfOffsetLimitSame4KBPage - (surfBase + surfOffset) + 1; 1389 } 1390 1391 return NV_OK; 1392 } 1393 1394 NvU64 1395 memmgrDeterminePageSize_IMPL 1396 ( 1397 MemoryManager *pMemoryManager, 1398 NvHandle hClient, 1399 NvU64 memSize, 1400 NvU32 memFormat, 1401 NvU32 pageFormatFlags, 1402 NvU32 *pRetAttr, 1403 NvU32 *pRetAttr2 1404 ) 1405 { 1406 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 1407 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 1408 NV_ADDRESS_SPACE addrSpace; 1409 NvBool bIsBigPageSupported; 1410 RM_ATTR_PAGE_SIZE pageSizeAttr; 1411 NvU64 pageSize = 0; 1412 1413 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY) || (pKernelGmmu == NULL)) 1414 { 1415 pageSize = RM_PAGE_SIZE; 1416 } 1417 // Sanity check the arguments. 1418 else if (pRetAttr == NULL || pRetAttr2 == NULL) 1419 { 1420 NV_ASSERT_OR_RETURN(0, 0); 1421 } 1422 else 1423 { 1424 addrSpace = memmgrAllocGetAddrSpace(pMemoryManager, pageFormatFlags, *pRetAttr); 1425 1426 // 1427 // Bug 4270864: Temp hack until sysmem supports higher order allocations. 1428 // We allow EGM to get allocated at higher page size. 1429 // 1430 if (memmgrIsLocalEgmEnabled(pMemoryManager) && 1431 addrSpace == ADDR_SYSMEM && 1432 FLD_TEST_DRF(OS32, _ATTR2, _FIXED_NUMA_NODE_ID, _YES, *pRetAttr2) && 1433 // 1434 // Bug 4270868: MODS has test cases which pass FIXED_NUMA_NODE_ID, 1435 // but invalid node_id. Will remove once MODS tests get fixed. 1436 // 1437 !RMCFG_FEATURE_MODS_FEATURES) 1438 { 1439 bIsBigPageSupported = NV_TRUE; 1440 } 1441 else 1442 { 1443 bIsBigPageSupported = memmgrLargePageSupported(pMemoryManager, addrSpace); 1444 } 1445 pageSizeAttr = dmaNvos32ToPageSizeAttr(*pRetAttr, *pRetAttr2); 1446 1447 // 1448 // Precedence in page size selection 1449 // 1. CACHE_ONLY mode -> SMALL 1450 // 2. !BigPageSupport (Sysmem && GpuSmmuOff ) -> SMALL 1451 // 3. Client page size override -> Use override 1452 // 4. HugePageSupported && size >= HugePageSize -> HUGE 1453 // 5. Block-linear || size >= minSizeForBigPage || hClient || GpuSmmuOn -> BIG 1454 // 6. none of the above -> SMALL 1455 // 1456 // On Tegra, we don't have a carveout/FB in production. So, we're 1457 // not guaranteed to get BIG page sized or contiguous allocations 1458 // from OS. But we need BIG page sized allocations for efficient Big GPU 1459 // operation. We use the SMMU unit within the Tegra Memory Contoller (MC), 1460 // to construct BIG pages from the 4KB small page allocations from OS. 1461 // SMMU will linearize the discontiguous 4KB allocations into what will 1462 // appear to the GPU as a large contiguous physical allocation. 1463 // 1464 // RM will eventually decide whether a SYSMEM allocation needs BIG page 1465 // via GPU SMMU mapping. Right now, we give an option for RM clients to 1466 // force it, via the SMMU_ON_GPU attribute. 1467 // 1468 if (gpuIsCacheOnlyModeEnabled(pGpu)) 1469 { 1470 pageSize = RM_PAGE_SIZE; 1471 } 1472 else if (!bIsBigPageSupported) 1473 { 1474 if (RM_ATTR_PAGE_SIZE_BIG == pageSizeAttr || 1475 RM_ATTR_PAGE_SIZE_HUGE == pageSizeAttr || 1476 RM_ATTR_PAGE_SIZE_512MB == pageSizeAttr) 1477 { 1478 NV_PRINTF(LEVEL_ERROR, 1479 "Big/Huge/512MB page size not supported in sysmem.\n"); 1480 1481 NV_ASSERT_OR_RETURN(0, 0); 1482 } 1483 else 1484 { 1485 pageSize = RM_PAGE_SIZE; 1486 } 1487 } 1488 else 1489 { 1490 switch (pageSizeAttr) 1491 { 1492 case RM_ATTR_PAGE_SIZE_INVALID: 1493 NV_PRINTF(LEVEL_ERROR, "invalid page size attr\n"); 1494 NV_ASSERT_OR_RETURN(0, 0); 1495 1496 case RM_ATTR_PAGE_SIZE_DEFAULT: 1497 { 1498 NvBool bUseDefaultHugePagesize = NV_TRUE; 1499 // WDDMV2 Windows it expect default page size to be 4K /64KB /128KB 1500 if (bUseDefaultHugePagesize && 1501 kgmmuIsHugePageSupported(pKernelGmmu) && 1502 (memSize >= RM_PAGE_SIZE_HUGE) && (addrSpace != ADDR_SYSMEM || 1503 pMemoryManager->sysmemPageSize == RM_PAGE_SIZE_HUGE)) 1504 { 1505 pageSize = RM_PAGE_SIZE_HUGE; 1506 break; 1507 } 1508 else if ((memFormat != NVOS32_ATTR_FORMAT_PITCH) || 1509 (memSize >= kgmmuGetMinBigPageSize(pKernelGmmu)) || hClient || 1510 FLD_TEST_DRF(OS32, _ATTR2, _SMMU_ON_GPU, _ENABLE, *pRetAttr2)) 1511 { 1512 pageSize = kgmmuGetMaxBigPageSize_HAL(pKernelGmmu); 1513 break; 1514 } 1515 1516 pageSize = RM_PAGE_SIZE; 1517 break; 1518 } 1519 1520 case RM_ATTR_PAGE_SIZE_4KB: 1521 pageSize = RM_PAGE_SIZE; 1522 break; 1523 1524 case RM_ATTR_PAGE_SIZE_BIG: 1525 pageSize = kgmmuGetMaxBigPageSize_HAL(pKernelGmmu); 1526 break; 1527 1528 case RM_ATTR_PAGE_SIZE_HUGE: 1529 if (kgmmuIsHugePageSupported(pKernelGmmu)) 1530 { 1531 pageSize = RM_PAGE_SIZE_HUGE; 1532 } 1533 else 1534 { 1535 NV_ASSERT_OR_RETURN(0, 0); 1536 } 1537 break; 1538 1539 case RM_ATTR_PAGE_SIZE_512MB: 1540 if (kgmmuIsPageSize512mbSupported(pKernelGmmu)) 1541 { 1542 pageSize = RM_PAGE_SIZE_512M; 1543 } 1544 else 1545 { 1546 NV_ASSERT_OR_RETURN(0, 0); 1547 } 1548 break; 1549 1550 default: 1551 NV_ASSERT(0); 1552 } 1553 } 1554 } 1555 1556 switch (pageSize) 1557 { 1558 case RM_PAGE_SIZE: 1559 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _4KB, *pRetAttr); 1560 break; 1561 1562 case RM_PAGE_SIZE_64K: 1563 case RM_PAGE_SIZE_128K: 1564 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _BIG, *pRetAttr); 1565 break; 1566 1567 case RM_PAGE_SIZE_HUGE: 1568 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _HUGE, *pRetAttr); 1569 *pRetAttr2 = FLD_SET_DRF(OS32, _ATTR2, _PAGE_SIZE_HUGE, _2MB, *pRetAttr2); 1570 break; 1571 1572 case RM_PAGE_SIZE_512M: 1573 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _HUGE, *pRetAttr); 1574 *pRetAttr2 = FLD_SET_DRF(OS32, _ATTR2, _PAGE_SIZE_HUGE, _512MB, *pRetAttr2); 1575 break; 1576 1577 default: 1578 NV_ASSERT(0); 1579 } 1580 1581 return pageSize; 1582 } 1583 1584 /*! 1585 * Identify if platform's current configuration supports PMA 1586 */ 1587 NV_STATUS 1588 memmgrSetPlatformPmaSupport_IMPL 1589 ( 1590 OBJGPU *pGpu, 1591 MemoryManager *pMemoryManager 1592 ) 1593 { 1594 // 1595 // KMD in WDDM mode will not support pma managed client page tables as 1596 // in both cases client / OS manges it. 1597 // 1598 if (RMCFG_FEATURE_PLATFORM_WINDOWS && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE)) 1599 { 1600 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 1601 } 1602 1603 // 1604 // FB management should use PMA on Unix/Linux/Mods/Windows 1605 // 1606 if (RMCFG_FEATURE_PLATFORM_UNIX 1607 || RMCFG_FEATURE_PLATFORM_MODS 1608 || RMCFG_FEATURE_PLATFORM_WINDOWS) 1609 { 1610 pMemoryManager->bPmaSupportedOnPlatform = NV_TRUE; 1611 } 1612 1613 // 1614 // PMA memory management is not currently supported in non SRIOV VGPU environment. 1615 // The RPC mechanism needs to be expanded to distinguish allocation types. 1616 // Bug #1735412 1617 // 1618 // TODO : Remove these constraints. 1619 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1620 { 1621 pMemoryManager->bPmaSupportedOnPlatform = NV_FALSE; 1622 } 1623 1624 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_VIRTUALIZATION_MODE_HOST_VGPU)) 1625 { 1626 if (pMemoryManager->bVgpuPmaSupport) 1627 { 1628 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 1629 } 1630 else 1631 { 1632 pMemoryManager->bPmaSupportedOnPlatform = NV_FALSE; 1633 } 1634 } 1635 return (NV_OK); 1636 } 1637 1638 /*! 1639 * Allocate console region in CPU-RM based on region table passed from Physical RM 1640 */ 1641 NV_STATUS 1642 memmgrAllocateConsoleRegion_IMPL 1643 ( 1644 OBJGPU *pGpu, 1645 MemoryManager *pMemoryManager, 1646 FB_REGION_DESCRIPTOR *pConsoleFbRegion 1647 ) 1648 { 1649 1650 NV_STATUS status = NV_OK; 1651 NvU32 consoleRegionId = 0x0; 1652 NvU64 regionSize; 1653 1654 if (pMemoryManager->Ram.ReservedConsoleDispMemSize > 0) 1655 { 1656 pConsoleFbRegion->base = pMemoryManager->Ram.fbRegion[consoleRegionId].base; 1657 pConsoleFbRegion->limit = pMemoryManager->Ram.fbRegion[consoleRegionId].limit; 1658 1659 regionSize = pConsoleFbRegion->limit - pConsoleFbRegion->base + 1; 1660 1661 // Once the console is reserved, we don't expect to reserve it again 1662 NV_ASSERT_OR_RETURN(pMemoryManager->Ram.pReservedConsoleMemDesc == NULL, 1663 NV_ERR_STATE_IN_USE); 1664 1665 status = memdescCreate(&pMemoryManager->Ram.pReservedConsoleMemDesc, pGpu, 1666 regionSize, RM_PAGE_SIZE_64K, NV_TRUE, ADDR_FBMEM, 1667 NV_MEMORY_UNCACHED, 1668 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 1669 if (status != NV_OK) 1670 { 1671 pConsoleFbRegion->base = pConsoleFbRegion->limit = 0; 1672 return status; 1673 } 1674 1675 memdescDescribe(pMemoryManager->Ram.pReservedConsoleMemDesc, ADDR_FBMEM, 1676 pConsoleFbRegion->base, regionSize); 1677 memdescSetPageSize(pMemoryManager->Ram.pReservedConsoleMemDesc, 1678 AT_GPU, RM_PAGE_SIZE); 1679 1680 1681 NV_PRINTF(LEVEL_INFO, "Allocating console region of size: %llx, at base : %llx \n ", 1682 regionSize, pConsoleFbRegion->base); 1683 } 1684 1685 return status; 1686 } 1687 1688 void 1689 memmgrReleaseConsoleRegion_IMPL 1690 ( 1691 OBJGPU *pGpu, 1692 MemoryManager *pMemoryManager 1693 ) 1694 { 1695 memdescDestroy(pMemoryManager->Ram.pReservedConsoleMemDesc); 1696 pMemoryManager->Ram.pReservedConsoleMemDesc = NULL; 1697 } 1698 1699 PMEMORY_DESCRIPTOR 1700 memmgrGetReservedConsoleMemDesc_IMPL 1701 ( 1702 OBJGPU *pGpu, 1703 MemoryManager *pMemoryManager 1704 ) 1705 { 1706 return pMemoryManager->Ram.pReservedConsoleMemDesc; 1707 } 1708 1709 /*! 1710 * Reserve FB for allocating BAR2 Page Dirs and Page Tables 1711 */ 1712 void 1713 memmgrReserveBar2BackingStore_IMPL 1714 ( 1715 OBJGPU *pGpu, 1716 MemoryManager *pMemoryManager, 1717 NvU64 *pAddr 1718 ) 1719 { 1720 NvU64 tmpAddr = *pAddr; 1721 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 1722 1723 NvU32 pageDirsSize = kbusGetSizeOfBar2PageDirs_HAL(pGpu, pKernelBus); 1724 NvU32 pageTblsSize = kbusGetSizeOfBar2PageTables_HAL(pGpu, pKernelBus); 1725 1726 // Reserve space for BAR2 Page Dirs 1727 if (pKernelBus->PDEBAR2Aperture == ADDR_FBMEM) 1728 { 1729 tmpAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1730 pKernelBus->bar2[GPU_GFID_PF].pdeBase = tmpAddr; 1731 tmpAddr += pageDirsSize; 1732 } 1733 1734 // Reserve space for BAR2 Page Tables 1735 if (pKernelBus->PTEBAR2Aperture == ADDR_FBMEM) 1736 { 1737 tmpAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1738 pKernelBus->bar2[GPU_GFID_PF].pteBase = tmpAddr; 1739 tmpAddr += pageTblsSize; 1740 } 1741 1742 NV_PRINTF(LEVEL_INFO, "Reserve space for bar2 Page dirs offset = 0x%llx size = 0x%x\n", 1743 pKernelBus->bar2[GPU_GFID_PF].pdeBase, pageDirsSize); 1744 1745 NV_PRINTF(LEVEL_INFO, "Reserve space for bar2 Page tables offset = 0x%llx size = 0x%x\n", 1746 pKernelBus->bar2[GPU_GFID_PF].pteBase, pageTblsSize); 1747 1748 *pAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1749 } 1750 1751 /*! 1752 * Calculate the Vista reserved memory requirement per FB region for mixed type/density 1753 */ 1754 void 1755 memmgrCalcReservedFbSpace_IMPL 1756 ( 1757 OBJGPU *pGpu, 1758 MemoryManager *pMemoryManager 1759 ) 1760 { 1761 NvU64 rsvdFastSize = 0; 1762 NvU64 rsvdSlowSize = 0; 1763 NvU64 rsvdISOSize = 0; 1764 NvU32 i; 1765 NvU32 idxISORegion = 0; 1766 NvU32 idxFastRegion = 0; 1767 NvU32 idxSlowRegion = 0; 1768 NvBool bAllocProtected = NV_FALSE; 1769 1770 bAllocProtected = gpuIsCCFeatureEnabled(pGpu); 1771 1772 // 1773 // This is a hack solely for Vista (on Vista the OS controls the majority of heap). 1774 // Linux and Mac don't have reserved memory and doesn't use this function. 1775 // 1776 // On Vista, Fermi's instance memory is not reserved by RM anymore. 1777 // KMD has to reserve enough instance memory for driver private data. 1778 // This function does the calculation of needed space. See bug 642233. 1779 // While it returns the result in Mb, the calculation is made with byte 1780 // 1781 1782 // If we have no usable memory then we can't reserve any. 1783 if (!pMemoryManager->Ram.fbUsableMemSize) 1784 return; 1785 1786 memmgrCalcReservedFbSpaceHal_HAL(pGpu, pMemoryManager, &rsvdFastSize, &rsvdSlowSize, &rsvdISOSize); 1787 1788 // If we have regions defined, fill in the per-segment reserved memory requirement 1789 if (pMemoryManager->Ram.numFBRegions > 0) 1790 { 1791 FB_REGION_DESCRIPTOR *pFbRegion = NULL; 1792 NvU64 regionSize = 0; 1793 1794 // 1795 // Find the fastest and ISO regions. This search makes a soft assumption that 1796 // region #0 is not reserved, fastest, and supports ISO -- that would be stupid 1797 // 1798 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 1799 { 1800 pFbRegion = &pMemoryManager->Ram.fbRegion[i]; 1801 regionSize = (pFbRegion->limit - pFbRegion->base +1); 1802 1803 // Check only non-reserved regions (which are typically unpopulated blackholes in address space) 1804 if ((!pFbRegion->bRsvdRegion) && 1805 (bAllocProtected || !pFbRegion->bProtected) && 1806 (regionSize >= (rsvdFastSize + rsvdSlowSize + rsvdISOSize))) 1807 { 1808 // Find the fastest region 1809 if ((pFbRegion->performance > pMemoryManager->Ram.fbRegion[idxFastRegion].performance) 1810 || pMemoryManager->Ram.fbRegion[idxFastRegion].bRsvdRegion 1811 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxFastRegion].bProtected)) 1812 { 1813 idxFastRegion = i; 1814 } 1815 // Find the slowest region 1816 if ((pFbRegion->performance < pMemoryManager->Ram.fbRegion[idxSlowRegion].performance) 1817 || pMemoryManager->Ram.fbRegion[idxSlowRegion].bRsvdRegion 1818 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxSlowRegion].bProtected)) 1819 { 1820 idxSlowRegion = i; 1821 } 1822 // Find the fastest ISO region 1823 if (pFbRegion->bSupportISO) 1824 { 1825 if ((!pMemoryManager->Ram.fbRegion[idxISORegion].bSupportISO) || 1826 (pFbRegion->performance > pMemoryManager->Ram.fbRegion[idxISORegion].performance) 1827 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxISORegion].bProtected)) 1828 { 1829 idxISORegion = i; 1830 } 1831 } 1832 } 1833 } 1834 1835 // There should *ALWAYS* be a region that supports ISO, even if we have no display 1836 NV_ASSERT(pMemoryManager->Ram.fbRegion[idxISORegion].bSupportISO); 1837 1838 // There should *ALWAYS* be a non-reserved region that is faster than reserved and supports ISO 1839 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxISORegion].bRsvdRegion); 1840 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxFastRegion].bRsvdRegion); 1841 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxSlowRegion].bRsvdRegion); 1842 1843 // 1844 // Whenever Hopper CC is enabled, it is mandatory to put allocations 1845 // like page tables, CBC and fault buffers in CPR region. Cannot put 1846 // reserved memory in protected region in non CC cases 1847 // 1848 if (!bAllocProtected) 1849 { 1850 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxISORegion].bProtected); 1851 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxFastRegion].bProtected); 1852 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxSlowRegion].bProtected); 1853 } 1854 1855 // 1856 // Vista expects to be able to VidHeapControl allocate a cursor in ISO 1857 // 1858 // For mixed density reserved memory should be split between "fast" and 1859 // "slow" memory. Fast memory should also support ISO. The policy to 1860 // prefer "slow" vs "fast" memory is platform dependent. 1861 // 1862 pMemoryManager->Ram.fbRegion[idxISORegion].rsvdSize += rsvdISOSize; 1863 pMemoryManager->Ram.fbRegion[idxSlowRegion].rsvdSize += rsvdSlowSize; 1864 pMemoryManager->Ram.fbRegion[idxFastRegion].rsvdSize += rsvdFastSize; 1865 } 1866 } 1867 1868 /*! 1869 * Init channel size 1870 * 1871 * @param[in] pChannel OBJCHANNEL pointer 1872 * @param[in] numCopyBlocks Number of copies that should fit in the push buffer 1873 * 1874 * @returns NV_STATUS 1875 */ 1876 void 1877 memmgrMemUtilsSetupChannelBufferSizes_IMPL 1878 ( 1879 MemoryManager *pMemoryManager, 1880 OBJCHANNEL *pChannel, 1881 NvU32 numCopyBlocks 1882 ) 1883 { 1884 // set channel specific sizes 1885 pChannel->channelPbSize = numCopyBlocks * MEMUTILS_SIZE_PER_BLOCK_INBYTES; 1886 pChannel->channelNotifierSize = MEMUTILS_CHANNEL_NOTIFIER_SIZE; 1887 pChannel->channelNumGpFifioEntries = MEMUTILS_NUM_GPFIFIO_ENTRIES; 1888 pChannel->methodSizePerBlock = MEMUTILS_SIZE_PER_BLOCK_INBYTES; 1889 pChannel->channelSize = pChannel->channelPbSize + MEMUTILS_CHANNEL_GPFIFO_SIZE + MEMUTILS_CHANNEL_SEMAPHORE_SIZE; 1890 pChannel->semaOffset = pChannel->channelPbSize + MEMUTILS_CHANNEL_GPFIFO_SIZE; 1891 pChannel->finishPayloadOffset = pChannel->semaOffset + 4; 1892 } 1893 1894 NV_STATUS memmgrFree_IMPL 1895 ( 1896 OBJGPU *pGpu, 1897 MemoryManager *pMemoryManager, 1898 Heap *pHeap, 1899 NvHandle hClient, 1900 NvHandle hDevice, 1901 NvHandle hVASpace, 1902 NvU32 owner, 1903 MEMORY_DESCRIPTOR *pMemDesc 1904 ) 1905 { 1906 NvU64 offsetAlign; 1907 NV_STATUS status; 1908 NvU32 pmaFreeFlag = 0; 1909 1910 // IRQL TEST: must be running at equivalent of passive-level 1911 IRQL_ASSERT_AND_RETURN(!osIsRaisedIRQL()); 1912 1913 if (pMemDesc == NULL) 1914 return NV_ERR_INVALID_ARGUMENT; 1915 1916 offsetAlign = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1917 1918 if (owner == NVOS32_BLOCK_TYPE_FREE) 1919 return NV_ERR_INVALID_ARGUMENT; 1920 1921 // Virtual heap allocs are tagged vitual and always own the memdesc 1922 if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL) 1923 { 1924 OBJVASPACE *pVAS = NULL; 1925 RsClient *pClient; 1926 1927 status = serverGetClientUnderLock(&g_resServ, hClient, &pClient); 1928 if (status != NV_OK) 1929 return status; 1930 1931 status = vaspaceGetByHandleOrDeviceDefault(pClient, hDevice, hVASpace, &pVAS); 1932 if (status != NV_OK) 1933 return status; 1934 1935 status = vaspaceFree(pVAS, offsetAlign); 1936 memdescDestroy(pMemDesc); 1937 return status; 1938 } 1939 1940 // Free up the memory allocated by PMA. 1941 if (pMemDesc->pPmaAllocInfo) 1942 { 1943 FB_ALLOC_INFO *pFbAllocInfo = NULL; 1944 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 1945 OBJGPU *pMemdescOwnerGpu = NULL; 1946 1947 // 1948 // A memdesc can be duped under a peer device. In that case, before 1949 // freeing FB make sure the GPU which owns the memdesc is available. 1950 // Otherwise, just assert, destroy the memdesc and return NV_OK to 1951 // make sure rest of the clean up happens correctly as we are on 1952 // destroy path. 1953 // Note this is just a WAR till ressrv bring in cleanup of dup objects 1954 // on GPU tear down. 1955 // RS-TODO: Nuke this check once the cleanup is implemented. 1956 // 1957 if (pGpu != pMemDesc->pGpu) 1958 { 1959 if (!gpumgrIsGpuPointerValid(pMemDesc->pGpu)) 1960 { 1961 // 1962 // This should never happen. GPU tear down should always clear 1963 // the duped memory list after resource server implements it. 1964 // For now just assert! 1965 // 1966 NV_ASSERT(0); 1967 memdescDestroy(pMemDesc); 1968 goto pma_free_exit; 1969 } 1970 } 1971 1972 pMemdescOwnerGpu = pMemDesc->pGpu; 1973 1974 // 1975 // Similar to the above WAR, if portMem alocations fail for any reason, 1976 // just assert and return NV_OK to ensure that the rest of the clean up 1977 // happens correctly. 1978 // 1979 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 1980 if (pFbAllocInfo == NULL) 1981 { 1982 NV_ASSERT(0); 1983 goto pma_free_exit; 1984 } 1985 1986 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 1987 if (pFbAllocPageFormat == NULL) { 1988 NV_ASSERT(0); 1989 goto pma_free_exit; 1990 } 1991 1992 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 1993 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 1994 pFbAllocInfo->hClient = hClient; 1995 pFbAllocInfo->hDevice = hDevice; 1996 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 1997 1998 // 1999 // Do not release any HW resources associated with this allocation 2000 // until the last reference to the allocation is freed. Passing 2001 // hwresid = 0 and format = pitch to memmgrFreeHwResources will ensure 2002 // that no comptags/zcull/zbc resources are freed. 2003 // 2004 if (pMemDesc->RefCount == 1) 2005 { 2006 pFbAllocInfo->hwResId = memdescGetHwResId(pMemDesc); 2007 pFbAllocInfo->format = memdescGetPteKind(pMemDesc); 2008 } 2009 else 2010 { 2011 pFbAllocInfo->hwResId = 0; 2012 pFbAllocInfo->format = 0; 2013 } 2014 pFbAllocInfo->offset = offsetAlign; 2015 pFbAllocInfo->size = pMemDesc->Size; 2016 2017 // Free any HW resources allocated. 2018 memmgrFreeHwResources(pMemdescOwnerGpu, 2019 GPU_GET_MEMORY_MANAGER(pMemdescOwnerGpu), pFbAllocInfo); 2020 2021 if (pMemDesc->pPmaAllocInfo != NULL) 2022 { 2023 // Disabling scrub on free for non compressible surfaces 2024 if (RMCFG_FEATURE_PLATFORM_MODS && 2025 !memmgrIsKind_HAL(GPU_GET_MEMORY_MANAGER(pMemdescOwnerGpu), 2026 FB_IS_KIND_COMPRESSIBLE, 2027 memdescGetPteKind(pMemDesc))) 2028 { 2029 pmaFreeFlag = PMA_FREE_SKIP_SCRUB; 2030 } 2031 2032 vidmemPmaFree(pMemdescOwnerGpu, pHeap, pMemDesc->pPmaAllocInfo, pmaFreeFlag); 2033 NV_PRINTF(LEVEL_INFO, "Freeing PMA allocation\n"); 2034 } 2035 2036 pma_free_exit: 2037 portMemFree(pFbAllocInfo); 2038 portMemFree(pFbAllocPageFormat); 2039 memdescDestroy(pMemDesc); 2040 2041 return NV_OK; 2042 } 2043 2044 return heapFree(pGpu, pHeap, hClient, hDevice, owner, pMemDesc); 2045 } 2046 2047 NV_STATUS 2048 memmgrSetPartitionableMem_IMPL 2049 ( 2050 OBJGPU *pGpu, 2051 MemoryManager *pMemoryManager 2052 ) 2053 { 2054 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 2055 NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS params = {0}; 2056 Heap *pHeap = GPU_GET_HEAP(pGpu); 2057 NvU64 bottomRsvdSize = 0; 2058 NvU64 topRsvdSize = 0; 2059 NvU32 bottomRegionIdx = 0xFFFF; 2060 NvU32 topRegionIdx = 0xFFFF; 2061 NvU32 i; 2062 NvU64 size; 2063 NvU64 base; 2064 NvU64 offset; 2065 NvU64 freeMem; 2066 2067 // 2068 // Find out the first and the last region for which internal heap or 2069 // bRsvdRegion is true. In Ampere we should never have more than two 2070 // discontigous RM reserved region 2071 // To-Do - Bug 2301972 - Make sure that reserved memory is aligned to VMMU 2072 // segments 2073 // 2074 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 2075 { 2076 if (pMemoryManager->Ram.fbRegion[i].bInternalHeap || 2077 pMemoryManager->Ram.fbRegion[i].bRsvdRegion) 2078 { 2079 NvU64 rsvdSize = (pMemoryManager->Ram.fbRegion[i].limit - 2080 pMemoryManager->Ram.fbRegion[i].base + 1); 2081 2082 // Check if this is bottom reserved region 2083 if (pMemoryManager->Ram.fbRegion[i].base == 0) 2084 { 2085 bottomRegionIdx = i; 2086 bottomRsvdSize += rsvdSize; 2087 } 2088 else if (i > 0 && (pMemoryManager->Ram.fbRegion[i-1].bInternalHeap || 2089 pMemoryManager->Ram.fbRegion[i-1].bRsvdRegion) && 2090 (pMemoryManager->Ram.fbRegion[i].base == pMemoryManager->Ram.fbRegion[i - 1].limit + 1)) 2091 { 2092 // See if this is the contigous region with previous discovery 2093 if (bottomRegionIdx == (i - 1)) 2094 { 2095 // Contigous bottom region 2096 bottomRsvdSize += rsvdSize; 2097 } 2098 else 2099 { 2100 // Contigous top region 2101 topRsvdSize += rsvdSize; 2102 } 2103 } 2104 else 2105 { 2106 // 2107 // Make sure we don't have discontigous reserved regions as 2108 // they are not supported by HW also and we need to support 2109 // these by using blacklisting mechanism. 2110 // 2111 if (topRegionIdx != 0xFFFF) 2112 { 2113 NV_PRINTF(LEVEL_ERROR, 2114 "More than two discontigous rsvd regions found. " 2115 "Rsvd region base - 0x%llx, Rsvd region Size - 0x%llx\n", 2116 pMemoryManager->Ram.fbRegion[i].base, rsvdSize); 2117 NV_ASSERT(0); 2118 return NV_ERR_INVALID_STATE; 2119 } 2120 2121 topRegionIdx = i; 2122 topRsvdSize += rsvdSize; 2123 } 2124 } 2125 } 2126 2127 // 2128 // Sanity check against the biggest available memory chunk. Pick the smallest 2129 // of biggest available memory chunk or calculated total - reserved memory as 2130 // in vGPU we are still using OBJHEAP and there are some allocations which 2131 // happens at the top of the heap before we program this register 2132 // 2133 if (!memmgrIsPmaInitialized(pMemoryManager)) 2134 { 2135 NvU64 bytesTotal; 2136 const NvU64 vgpuHeapWarSize = 256 *1024 * 1024; 2137 NV_ASSERT_OK_OR_RETURN(heapInfo(pHeap, &freeMem, &bytesTotal, &base, 2138 &offset, &size)); 2139 2140 // 2141 // offset is the starting address of biggest empty block whose size is 2142 // returned and we care about the base of largest empty block 2143 // 2144 base = offset; 2145 2146 // 2147 // WAR - Bug-2383259 - TilL PMA is not enabled in vGPU-Host 2148 // we need to delay reserve some memory at the top to full fill lazy 2149 // allocations like FECS and GPCCS uCode. Leave 256MB at the top for 2150 // such lazy allocations 2151 // 2152 if (size > vgpuHeapWarSize) 2153 { 2154 size -= vgpuHeapWarSize; 2155 } 2156 } 2157 else 2158 { 2159 PMA_REGION_DESCRIPTOR *pFirstPmaRegionDesc = NULL; 2160 NvU32 numPmaRegions; 2161 NvU32 pmaConfig = PMA_QUERY_NUMA_ONLINED; 2162 2163 NV_ASSERT_OK_OR_RETURN(pmaGetRegionInfo(&pHeap->pmaObject, 2164 &numPmaRegions, &pFirstPmaRegionDesc)); 2165 2166 base = pFirstPmaRegionDesc->base; 2167 pmaGetFreeMemory(&pHeap->pmaObject, &freeMem); 2168 pmaGetTotalMemory(&pHeap->pmaObject, &size); 2169 2170 NV_ASSERT_OK(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfig)); 2171 2172 // 2173 // MIG won't be used alongside APM and hence the check below is of no use 2174 // Even if we enable the check for APM the check will fail given that after 2175 // enabling "scrub on free" using virtual CE writes, memory gets consumed by 2176 // page tables backing the scrubber channel virtual mappings and hence the 2177 // calculation below no longer holds good 2178 // In case of HCC, structures like PB, GPFIFO and USERD for scrubber and golden 2179 // channels are required to be in CPR vidmem. This changes the calculation below 2180 // We can ignore this for the non-MIG case. 2181 // 2182 // When FB memory is onlined as NUMA node, kernel can directly alloc FB memory 2183 // and hence free memory can not be expected to be same as total memory. 2184 // 2185 if ((!gpuIsCCorApmFeatureEnabled(pGpu) || IS_MIG_ENABLED(pGpu)) && 2186 !(pmaConfig & PMA_QUERY_NUMA_ONLINED)) 2187 { 2188 NvU64 maxUsedPmaSize = 2 * RM_PAGE_SIZE_128K; 2189 // 2190 // PMA should be completely free at this point, otherwise we risk 2191 // not setting the right partitionable range (pmaGetLargestFree's 2192 // offset argument is not implemented as of this writing, so we 2193 // only get the base address of the region that contains it). There 2194 // is a known allocation from the top-level scrubber/CeUtils channel that 2195 // is expected to be no larger than 128K. Issue a warning for any 2196 // other uses. 2197 // 2198 if ((size > maxUsedPmaSize) && 2199 (freeMem < (size - maxUsedPmaSize))) 2200 { 2201 NV_PRINTF(LEVEL_ERROR, 2202 "Assumption that PMA is empty (after accounting for the top-level scrubber and CeUtils) is not met!\n"); 2203 NV_PRINTF(LEVEL_ERROR, 2204 " free space = 0x%llx bytes, total space = 0x%llx bytes\n", 2205 freeMem, size); 2206 NV_ASSERT_OR_RETURN(freeMem >= (size - maxUsedPmaSize), 2207 NV_ERR_INVALID_STATE); 2208 } 2209 } 2210 } 2211 2212 if (size == 0) 2213 { 2214 NV_PRINTF(LEVEL_ERROR, 2215 "No partitionable memory. MIG memory partitioning can't be enabled.\n"); 2216 return NV_OK; 2217 } 2218 2219 if (base != bottomRsvdSize) 2220 { 2221 NV_PRINTF(LEVEL_ERROR, 2222 "Partitionable memory start - 0x%llx not aligned with RM reserved " 2223 "region base-end - 0x%llx\n", base, bottomRsvdSize); 2224 return NV_ERR_INVALID_STATE; 2225 } 2226 2227 params.partitionableMemSize = size; 2228 params.bottomRsvdSize = bottomRsvdSize; 2229 params.topRsvdSize = topRsvdSize; 2230 2231 // Call physical MemorySystem to align and program the partitionable range 2232 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2233 pRmApi->Control(pRmApi, 2234 pGpu->hInternalClient, 2235 pGpu->hInternalSubdevice, 2236 NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM, 2237 ¶ms, 2238 sizeof(params))); 2239 2240 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = 2241 rangeMake(params.partitionableStartAddr, params.partitionableEndAddr); 2242 2243 // 2244 // Make sure the created range is a valid range. 2245 // rangeIsEmpty checks lo > hi, which should be good enough to catch 2246 // inverted range case. 2247 // 2248 NV_ASSERT_OR_RETURN(!rangeIsEmpty(pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange), 2249 NV_ERR_INVALID_STATE); 2250 2251 if (!KBUS_CPU_VISIBLE_BAR12_DISABLED(pGpu)) 2252 { 2253 NV_ASSERT_OK_OR_RETURN(memmgrSetMIGPartitionableBAR1Range(pGpu, pMemoryManager)); 2254 } 2255 2256 if (IS_GSP_CLIENT(pGpu)) 2257 { 2258 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2259 2260 // 2261 // The Physical RM initializes its AMAPLIB context via 2262 // memsysSetPartitionableMem_HAL(). The GSP Client RM has a separate 2263 // AMAPLIB context that must also be initialized. 2264 // 2265 kmemsysReadMIGMemoryCfg_HAL(pGpu, pKernelMemorySystem); 2266 } 2267 2268 return NV_OK; 2269 } 2270 2271 NV_STATUS 2272 memmgrFillComprInfo_IMPL 2273 ( 2274 OBJGPU *pGpu, 2275 MemoryManager *pMemoryManager, 2276 NvU64 pageSize, 2277 NvU32 pageCount, 2278 NvU32 kind, 2279 NvU64 surfOffset, 2280 NvU32 compTagStartOffset, 2281 COMPR_INFO *pComprInfo 2282 ) 2283 { 2284 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 2285 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 2286 2287 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2288 2289 pComprInfo->kind = kind; 2290 2291 if (!memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 2292 return NV_OK; 2293 2294 // TODO: We will have to support compression on vGPU HOST for AC 2295 NV_ASSERT(compTagStartOffset != ~(NvU32)0); 2296 2297 pComprInfo->compPageShift = pMemorySystemConfig->comprPageShift; 2298 pComprInfo->compTagLineMin = compTagStartOffset; 2299 pComprInfo->compPageIndexLo = (NvU32)(surfOffset >> pComprInfo->compPageShift); 2300 pComprInfo->compPageIndexHi = (NvU32)((surfOffset + pageSize * pageCount - 1) >> pComprInfo->compPageShift); 2301 pComprInfo->compTagLineMultiplier = 1; 2302 2303 return NV_OK; 2304 } 2305 2306 NV_STATUS 2307 memmgrGetKindComprForGpu_KERNEL 2308 ( 2309 MemoryManager *pMemoryManager, 2310 MEMORY_DESCRIPTOR *pMemDesc, 2311 OBJGPU *pMappingGpu, 2312 NvU64 offset, 2313 NvU32 *pKind, 2314 COMPR_INFO *pComprInfo 2315 ) 2316 { 2317 NvU32 ctagId = FB_HWRESID_CTAGID_VAL_FERMI(memdescGetHwResId(pMemDesc)); 2318 NvU32 kind = memdescGetPteKindForGpu(pMemDesc, pMappingGpu); 2319 const MEMORY_SYSTEM_STATIC_CONFIG *pMappingMemSysConfig = 2320 kmemsysGetStaticConfig(pMappingGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pMappingGpu)); 2321 2322 // Compression is not supported on memory not backed by a GPU 2323 if (pMemDesc->pGpu != NULL && memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind) && 2324 (ctagId == 0 || ctagId == FB_HWRESID_CTAGID_VAL_FERMI(-1))) 2325 { 2326 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2327 2328 pComprInfo->kind = kind; 2329 pComprInfo->compPageShift = pMappingMemSysConfig->comprPageShift; 2330 pComprInfo->bPhysBasedComptags = NV_TRUE; 2331 pComprInfo->compTagLineMin = 1; 2332 } 2333 else 2334 { 2335 if (ctagId == FB_HWRESID_CTAGID_VAL_FERMI(0xcdcdcdcd)) 2336 { 2337 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2338 2339 pComprInfo->kind = memmgrGetUncompressedKind_HAL(pMappingGpu, pMemoryManager, kind, NV_TRUE); 2340 } 2341 else 2342 { 2343 memmgrFillComprInfoUncompressed(pMemoryManager, kind, pComprInfo); 2344 } 2345 } 2346 2347 *pKind = pComprInfo->kind; 2348 2349 return NV_OK; 2350 } 2351 2352 NV_STATUS 2353 memmgrGetKindComprFromMemDesc_IMPL 2354 ( 2355 MemoryManager *pMemoryManager, 2356 MEMORY_DESCRIPTOR *pMemDesc, 2357 NvU64 offset, 2358 NvU32 *kind, 2359 COMPR_INFO *pComprInfo 2360 ) 2361 { 2362 return memmgrGetKindComprForGpu_HAL(pMemoryManager, pMemDesc, pMemDesc->pGpu, 2363 offset, kind, pComprInfo); 2364 } 2365 2366 void 2367 memmgrSetMIGPartitionableMemoryRange_IMPL 2368 ( 2369 OBJGPU *pGpu, 2370 MemoryManager *pMemoryManager, 2371 NV_RANGE range 2372 ) 2373 { 2374 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = range; 2375 } 2376 2377 NV_RANGE 2378 memmgrGetMIGPartitionableMemoryRange_IMPL 2379 ( 2380 OBJGPU *pGpu, 2381 MemoryManager *pMemoryManager 2382 ) 2383 { 2384 return pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange; 2385 } 2386 2387 /* 2388 * @brief Sets total partitionable BAR1 2389 */ 2390 NV_STATUS 2391 memmgrSetMIGPartitionableBAR1Range_IMPL 2392 ( 2393 OBJGPU *pGpu, 2394 MemoryManager *pMemoryManager 2395 ) 2396 { 2397 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 2398 OBJVASPACE *pBar1VAS = kbusGetBar1VASpace_HAL(pGpu, pKernelBus); 2399 OBJEHEAP *pVASHeap; 2400 NvU64 largestFreeOffset = 0; 2401 NvU64 largestFreeSize = 0; 2402 NvU64 partitionableBar1Start; 2403 NvU64 partitionableBar1End; 2404 2405 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)) 2406 return NV_OK; 2407 2408 NV_ASSERT_OR_RETURN(pBar1VAS != NULL, NV_ERR_INVALID_STATE); 2409 pVASHeap = vaspaceGetHeap(pBar1VAS); 2410 2411 // Get partitionable BAR1 range 2412 pVASHeap->eheapInfo(pVASHeap, NULL, NULL, &largestFreeOffset, &largestFreeSize, NULL, NULL); 2413 2414 // 2415 // We are not considering alignment here because VA space is reserved/allocated in chunks of pages 2416 // so largestFreeOffset should be already aligned. 2417 // 2418 partitionableBar1Start = largestFreeOffset; 2419 partitionableBar1End = largestFreeOffset + largestFreeSize - 1; 2420 NV_ASSERT_OR_RETURN(partitionableBar1Start >= vaspaceGetVaStart(pBar1VAS), NV_ERR_INVALID_STATE); 2421 NV_ASSERT_OR_RETURN(partitionableBar1End <= vaspaceGetVaLimit(pBar1VAS), NV_ERR_INVALID_STATE); 2422 2423 pMemoryManager->MIGMemoryPartitioningInfo.partitionableBar1Range = rangeMake(partitionableBar1Start, partitionableBar1End); 2424 return NV_OK; 2425 } 2426 2427 NV_RANGE 2428 memmgrGetMIGPartitionableBAR1Range_IMPL 2429 ( 2430 OBJGPU *pGpu, 2431 MemoryManager *pMemoryManager 2432 ) 2433 { 2434 return pMemoryManager->MIGMemoryPartitioningInfo.partitionableBar1Range; 2435 } 2436 2437 NV_STATUS 2438 memmgrAllocMIGGPUInstanceMemory_VF 2439 ( 2440 OBJGPU *pGpu, 2441 MemoryManager *pMemoryManager, 2442 NvU32 swizzId, 2443 NvHandle *phMemory, 2444 NV_RANGE *pAddrRange, 2445 Heap **ppMemoryPartitionHeap 2446 ) 2447 { 2448 // For vGpu we have a static memory allocation 2449 *phMemory = NV01_NULL_OBJECT; 2450 *pAddrRange = pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange; 2451 *ppMemoryPartitionHeap = GPU_GET_HEAP(pGpu); 2452 2453 return NV_OK; 2454 } 2455 2456 // Function to allocate memory for a GPU instance 2457 NV_STATUS 2458 memmgrAllocMIGGPUInstanceMemory_PF 2459 ( 2460 OBJGPU *pGpu, 2461 MemoryManager *pMemoryManager, 2462 NvU32 swizzId, 2463 NvHandle *phMemory, 2464 NV_RANGE *pAddrRange, 2465 Heap **ppMemoryPartitionHeap 2466 ) 2467 { 2468 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2469 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2470 NV_STATUS rmStatus = NV_OK; 2471 NvHandle hMemory = 0; 2472 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2473 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2474 2475 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2476 NV_ASSERT_OK_OR_RETURN(kmemsysGetMIGGPUInstanceMemInfo(pGpu, pKernelMemorySystem, swizzId, pAddrRange)); 2477 2478 // 2479 // Only allocate memory for non swizzID-0 GPU instances as swizzID-0 owns full 2480 // gpu and there is no need to pre-reserve memory for that and non 2481 // coherent systems. In coherent NUMA systems, NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE 2482 // is not supported and the memory comes from the MIG partition memory 2483 // NUMA node. 2484 // 2485 if (kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2486 { 2487 if(bNumaEnabled) 2488 { 2489 NvS32 numaNodeId; 2490 NvU64 partitionBaseAddr = pAddrRange->lo; 2491 NvU64 partitionSize = rangeLength(*pAddrRange); 2492 2493 if (kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager) == 0x0) 2494 { 2495 // Remove swizz Id 0 / baremetal GPU memory NUMA node 2496 pmaNumaOfflined(&GPU_GET_HEAP(pGpu)->pmaObject); 2497 kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, 0); 2498 } 2499 2500 // 2501 // The memory gets removed in memmgrFreeMIGGPUInstanceMemory if 2502 // there is any failure after adding the memory. 2503 // 2504 NV_ASSERT_OK_OR_RETURN(kmemsysNumaAddMemory_HAL(pGpu, 2505 pKernelMemorySystem, 2506 swizzId, 2507 partitionBaseAddr, 2508 partitionSize, 2509 &numaNodeId)); 2510 } 2511 else 2512 { 2513 // 2514 // Allocate memory using vidHeapControl 2515 // 2516 // vidHeapControl calls should happen outside GPU locks 2517 // This is a PMA requirement as memory allocation calls may invoke eviction 2518 // which UVM could get stuck behind GPU lock 2519 // See Bug 1735851-#24 2520 // 2521 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 2522 2523 // Allocate gpfifo entries 2524 NV_MEMORY_ALLOCATION_PARAMS memAllocParams; 2525 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2526 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2527 memAllocParams.type = NVOS32_TYPE_IMAGE; 2528 memAllocParams.size = rangeLength(*pAddrRange); 2529 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _VIDMEM); 2530 memAllocParams.attr |= DRF_DEF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS); 2531 memAllocParams.attr |= DRF_DEF(OS32, _ATTR, _PAGE_SIZE, _DEFAULT); 2532 memAllocParams.attr2 = DRF_DEF(OS32, _ATTR2, _PAGE_OFFLINING, _OFF); // free the offlined pages 2533 memAllocParams.flags |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 2534 memAllocParams.rangeLo = 0; 2535 memAllocParams.rangeHi = 0; 2536 memAllocParams.offset = pAddrRange->lo; // Offset needed if fixed address allocation 2537 memAllocParams.hVASpace = 0; // Physical allocation 2538 memAllocParams.internalflags = NVOS32_ALLOC_INTERNAL_FLAGS_SKIP_SCRUB; 2539 2540 rmStatus = pRmApi->Alloc(pRmApi, 2541 pMemoryManager->MIGMemoryPartitioningInfo.hClient, 2542 pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice, 2543 &hMemory, 2544 NV01_MEMORY_LOCAL_USER, 2545 &memAllocParams, 2546 sizeof(memAllocParams)); 2547 2548 // Reaquire the GPU locks 2549 if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_MEM) != NV_OK) 2550 { 2551 NV_PRINTF(LEVEL_ERROR, "failed to grab RM-Lock\n"); 2552 DBG_BREAKPOINT(); 2553 rmStatus = NV_ERR_GENERIC; 2554 goto cleanup; 2555 } 2556 2557 if (rmStatus != NV_OK) 2558 { 2559 NV_PRINTF(LEVEL_ERROR, 2560 "Unable to allocate physical memory for GPU instance.\n"); 2561 return rmStatus; 2562 } 2563 } 2564 } 2565 rmStatus = _memmgrInitMIGMemoryPartitionHeap(pGpu, pMemoryManager, swizzId, pAddrRange, ppMemoryPartitionHeap); 2566 if (rmStatus != NV_OK) 2567 { 2568 NV_PRINTF(LEVEL_ERROR, "Unable to initialize memory partition heap\n"); 2569 goto cleanup; 2570 } 2571 2572 NV_PRINTF(LEVEL_INFO, 2573 "Allocated memory partition heap for swizzId - %d with StartAddr - 0x%llx, endAddr - 0x%llx.\n", 2574 swizzId, pAddrRange->lo, pAddrRange->hi); 2575 2576 *phMemory = hMemory; 2577 return rmStatus; 2578 2579 cleanup: 2580 pRmApi->Free(pRmApi, pMemoryManager->MIGMemoryPartitioningInfo.hClient, hMemory); 2581 2582 return rmStatus; 2583 } 2584 2585 // Function to initialize heap for managing MIG partition memory 2586 static NV_STATUS 2587 _memmgrInitMIGMemoryPartitionHeap 2588 ( 2589 OBJGPU *pGpu, 2590 MemoryManager *pMemoryManager, 2591 NvU32 swizzId, 2592 NV_RANGE *pAddrRange, 2593 Heap **ppMemoryPartitionHeap 2594 ) 2595 { 2596 NV_STATUS status = NV_OK; 2597 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2598 Heap *pMemoryPartitionHeap = NULL; 2599 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2600 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2601 NvU64 partitionBaseAddr = pAddrRange->lo; 2602 NvU64 partitionSize = rangeLength(*pAddrRange); 2603 2604 // Use default heap for swizzID-0 as we don't prereserve memory for swizzID-0 2605 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2606 if (!kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2607 { 2608 *ppMemoryPartitionHeap = pMemoryManager->pHeap; 2609 return NV_OK; 2610 } 2611 else 2612 { 2613 *ppMemoryPartitionHeap = NULL; 2614 } 2615 2616 NV_ASSERT_OK_OR_GOTO( 2617 status, 2618 objCreate(ppMemoryPartitionHeap, pMemoryManager, Heap), 2619 fail); 2620 2621 pMemoryPartitionHeap = *ppMemoryPartitionHeap; 2622 2623 if (memmgrIsPmaEnabled(pMemoryManager) && 2624 memmgrIsPmaSupportedOnPlatform(pMemoryManager)) 2625 { 2626 portMemSet(&pMemoryPartitionHeap->pmaObject, 0, sizeof(pMemoryPartitionHeap->pmaObject)); 2627 NV_ASSERT_OK_OR_GOTO( 2628 status, 2629 memmgrPmaInitialize(pGpu, pMemoryManager, &pMemoryPartitionHeap->pmaObject), 2630 fail); 2631 2632 if (bNumaEnabled) 2633 { 2634 NV_ASSERT_OR_GOTO(pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse, fail); 2635 partitionBaseAddr = pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset; 2636 partitionSize = pKernelMemorySystem->memPartitionNumaInfo[swizzId].size; 2637 2638 // 2639 // The base and size passed here is the FB base and size and 2640 // not the partition's. pmaNumaOnlined requires the FB base and 2641 // size to convert between FB local address and SPA. 2642 // memmgrPmaRegisterRegions is where the partition's base and size 2643 // is reported to PMA. 2644 // 2645 NV_ASSERT_OK_OR_GOTO( 2646 status, 2647 pmaNumaOnlined(&pMemoryPartitionHeap->pmaObject, 2648 pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId, 2649 pKernelMemorySystem->coherentCpuFbBase, 2650 pKernelMemorySystem->numaOnlineSize), 2651 fail); 2652 } 2653 } 2654 2655 NV_ASSERT_OK_OR_GOTO( 2656 status, 2657 heapInit(pGpu, pMemoryPartitionHeap, partitionBaseAddr, 2658 partitionSize, 2659 HEAP_TYPE_PARTITION_LOCAL, 2660 GPU_GFID_PF, 2661 NULL), 2662 fail); 2663 2664 if (memmgrIsPmaInitialized(pMemoryManager) && 2665 (pMemoryPartitionHeap->bHasFbRegions)) 2666 { 2667 NV_ASSERT_OK_OR_GOTO( 2668 status, 2669 memmgrPmaRegisterRegions(pGpu, pMemoryManager, pMemoryPartitionHeap, 2670 &pMemoryPartitionHeap->pmaObject), 2671 fail); 2672 } 2673 2674 if (!IsSLIEnabled(pGpu)) 2675 { 2676 // Do the actual blacklisting of pages from the heap 2677 if (pMemoryPartitionHeap->blackListAddresses.count != 0) 2678 { 2679 status = heapBlackListPages(pGpu, pMemoryPartitionHeap); 2680 2681 if (status != NV_OK) 2682 { 2683 // Warn and continue 2684 NV_PRINTF(LEVEL_WARNING, "Error 0x%x creating blacklist\n", 2685 status); 2686 } 2687 } 2688 } 2689 2690 return NV_OK; 2691 2692 fail: 2693 2694 if (pMemoryPartitionHeap != NULL) 2695 { 2696 objDelete(pMemoryPartitionHeap); 2697 *ppMemoryPartitionHeap = NULL; 2698 } 2699 2700 return status; 2701 } 2702 2703 // Function to free GPU instance memory 2704 NV_STATUS 2705 memmgrFreeMIGGPUInstanceMemory_IMPL 2706 ( 2707 OBJGPU *pGpu, 2708 MemoryManager *pMemoryManager, 2709 NvU32 swizzId, 2710 NvHandle hMemory, 2711 Heap **ppMemoryPartitionHeap 2712 ) 2713 { 2714 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2715 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2716 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2717 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2718 2719 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2720 2721 // Nothing to do for swizzId 0 as we neither allocate memory nor allocate new heap object 2722 if (!kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2723 return NV_OK; 2724 2725 objDelete(*ppMemoryPartitionHeap); 2726 *ppMemoryPartitionHeap = NULL; 2727 2728 if (bNumaEnabled) 2729 { 2730 kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, swizzId); 2731 2732 if (kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager) == 0x0) 2733 { 2734 NvS32 numaNodeId; 2735 2736 // Add back the baremetal GPU memory NUMA node. 2737 NV_ASSERT_OK_OR_RETURN(kmemsysNumaAddMemory_HAL(pGpu, 2738 pKernelMemorySystem, 2739 0, 2740 pKernelMemorySystem->numaOnlineBase, 2741 pKernelMemorySystem->numaOnlineSize, 2742 &numaNodeId)); 2743 // Baremetal NUMA node id should be same as pGpu->numaNodeId 2744 NV_ASSERT_OR_RETURN(numaNodeId == pGpu->numaNodeId, NV_ERR_INVALID_STATE); 2745 NV_ASSERT_OK_OR_RETURN(pmaNumaOnlined(&GPU_GET_HEAP(pGpu)->pmaObject, 2746 pGpu->numaNodeId, 2747 pKernelMemorySystem->coherentCpuFbBase, 2748 pKernelMemorySystem->numaOnlineSize)); 2749 } 2750 } 2751 2752 // Free allocated memory 2753 if (!bNumaEnabled && (hMemory != NV01_NULL_OBJECT)) 2754 { 2755 pRmApi->Free(pRmApi, pMemoryManager->MIGMemoryPartitioningInfo.hClient, hMemory); 2756 } 2757 return NV_OK; 2758 } 2759 2760 void memmgrComprInfoDisableCompression_IMPL 2761 ( 2762 MemoryManager *pMemoryManager, 2763 COMPR_INFO *pComprInfo 2764 ) 2765 { 2766 memmgrFillComprInfoUncompressed(pMemoryManager, pComprInfo->kind, pComprInfo); 2767 } 2768 2769 void memmgrFillComprInfoUncompressed_IMPL 2770 ( 2771 MemoryManager *pMemoryManager, 2772 NvU32 kind, 2773 COMPR_INFO *pComprInfo 2774 ) 2775 { 2776 if (memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 2777 kind = memmgrGetUncompressedKind_HAL(ENG_GET_GPU(pMemoryManager), pMemoryManager, kind, NV_FALSE); 2778 2779 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2780 pComprInfo->kind = kind; 2781 } 2782 2783 /*! 2784 * @brief Creates the SW state of the page level pools. 2785 * 2786 * @param pGpu 2787 * @param pMemoryManager 2788 * 2789 * @returns On success, returns NV_OK. 2790 * On failure, returns error code. 2791 */ 2792 NV_STATUS 2793 memmgrPageLevelPoolsCreate_IMPL 2794 ( 2795 OBJGPU *pGpu, 2796 MemoryManager *pMemoryManager 2797 ) 2798 { 2799 NV_STATUS status = NV_OK; 2800 2801 if (RMCFG_FEATURE_PMA && 2802 memmgrIsPmaInitialized(pMemoryManager) && 2803 memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2804 { 2805 Heap *pHeap = GPU_GET_HEAP(pGpu); 2806 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 2807 const GMMU_FMT *pFmt = NULL; 2808 2809 pFmt = kgmmuFmtGet(pKernelGmmu, GMMU_FMT_VERSION_DEFAULT, 0); 2810 NV_ASSERT_OR_RETURN(NULL != pFmt, NV_ERR_INVALID_ARGUMENT); 2811 2812 status = rmMemPoolSetup((void *)&pHeap->pmaObject, &pMemoryManager->pPageLevelReserve, 2813 (pFmt->version == GMMU_FMT_VERSION_1) ? POOL_CONFIG_GMMU_FMT_1 : POOL_CONFIG_GMMU_FMT_2); 2814 2815 NV_ASSERT(NV_OK == status); 2816 2817 // 2818 // Allocate the pool in CPR in case of Confidential Compute 2819 // When Hopper Confidential Compute is enabled, page tables 2820 // cannot be in non-CPR region 2821 // 2822 if (gpuIsCCFeatureEnabled(pGpu) && (status == NV_OK)) 2823 { 2824 rmMemPoolAllocateProtectedMemory(pMemoryManager->pPageLevelReserve, NV_TRUE); 2825 } 2826 } 2827 return status; 2828 } 2829 2830 /*! 2831 * @brief Destroys the SW state of the page level pools. 2832 * 2833 * @param pGpu 2834 * @param pMemoryManager 2835 * 2836 * @returns 2837 */ 2838 void 2839 memmgrPageLevelPoolsDestroy_IMPL 2840 ( 2841 OBJGPU *pGpu, 2842 MemoryManager *pMemoryManager 2843 ) 2844 { 2845 if (RMCFG_FEATURE_PMA && 2846 memmgrIsPmaInitialized(pMemoryManager) && 2847 memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2848 { 2849 rmMemPoolDestroy(pMemoryManager->pPageLevelReserve); 2850 pMemoryManager->pPageLevelReserve = NULL; 2851 } 2852 } 2853 2854 /*! 2855 * @brief Gets page level pool to use 2856 * 2857 * @param pGpu 2858 * @param pMemoryManager 2859 * @param[in] hClient client handle 2860 * @param[out] ppMemPoolInfo page level pool 2861 * 2862 * @returns On success, returns NV_OK. 2863 * On failure, returns error code. 2864 */ 2865 NV_STATUS 2866 memmgrPageLevelPoolsGetInfo_IMPL 2867 ( 2868 OBJGPU *pGpu, 2869 MemoryManager *pMemoryManager, 2870 Device *pDevice, 2871 RM_POOL_ALLOC_MEM_RESERVE_INFO **ppMemPoolInfo 2872 ) 2873 { 2874 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2875 NvBool bMemPartitioningEnabled = (pKernelMIGManager != NULL) && kmigmgrIsMIGMemPartitioningEnabled(pGpu, pKernelMIGManager); 2876 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemPool = NULL; 2877 NV_ASSERT_OR_RETURN(ppMemPoolInfo != NULL, NV_ERR_INVALID_ARGUMENT); 2878 2879 if (!memmgrIsPmaInitialized(pMemoryManager) || 2880 !memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2881 { 2882 return NV_ERR_INVALID_STATE; 2883 } 2884 2885 // If memory partitioning is enabled, then use per-partition pool allocator 2886 if (bMemPartitioningEnabled) 2887 { 2888 MIG_INSTANCE_REF ref; 2889 NV_ASSERT_OK_OR_RETURN( 2890 kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager, pDevice, &ref)); 2891 pMemPool = ref.pKernelMIGGpuInstance->pPageTableMemPool; 2892 } 2893 else 2894 { 2895 pMemPool = pMemoryManager->pPageLevelReserve; 2896 } 2897 NV_ASSERT_OR_RETURN(pMemPool != NULL, NV_ERR_INVALID_STATE); 2898 2899 *ppMemPoolInfo = pMemPool; 2900 return NV_OK; 2901 } 2902 2903 /*! 2904 * @brief Initialize the PMA object 2905 * 2906 * @param pGpu 2907 * @param pMemoryManager 2908 * @param[in] pPma Pointer to the PMA object to init 2909 * 2910 * @returns On success, returns NV_OK. 2911 * On failure, returns error code. 2912 */ 2913 NV_STATUS 2914 memmgrPmaInitialize_IMPL 2915 ( 2916 OBJGPU *pGpu, 2917 MemoryManager *pMemoryManager, 2918 PMA *pPma 2919 ) 2920 { 2921 NvU32 pmaInitFlags = PMA_INIT_NONE; 2922 NV_STATUS status = NV_OK; 2923 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2924 2925 NV_ASSERT(memmgrIsPmaEnabled(pMemoryManager) && 2926 memmgrIsPmaSupportedOnPlatform(pMemoryManager)); 2927 2928 if (memmgrIsPmaForcePersistence(pMemoryManager)) 2929 { 2930 pmaInitFlags |= PMA_INIT_FORCE_PERSISTENCE; 2931 } 2932 2933 if (memmgrIsScrubOnFreeEnabled(pMemoryManager)) 2934 { 2935 pmaInitFlags |= PMA_INIT_SCRUB_ON_FREE; 2936 } 2937 2938 // Disable client page table management on SLI. 2939 if (IsSLIEnabled(pGpu)) 2940 { 2941 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 2942 } 2943 2944 if (bNumaEnabled) 2945 { 2946 NV_PRINTF(LEVEL_INFO, "Initializing PMA with NUMA flag.\n"); 2947 pmaInitFlags |= PMA_INIT_NUMA; 2948 2949 if (gpuIsSelfHosted(pGpu)) 2950 { 2951 NV_PRINTF(LEVEL_INFO, "Initializing PMA with NUMA_AUTO_ONLINE flag.\n"); 2952 pmaInitFlags |= PMA_INIT_NUMA_AUTO_ONLINE; 2953 } 2954 } 2955 2956 status = pmaInitialize(pPma, pmaInitFlags); 2957 if (status != NV_OK) 2958 { 2959 NV_PRINTF(LEVEL_ERROR, "Failed to initialize PMA!\n"); 2960 return status; 2961 } 2962 2963 if (bNumaEnabled) 2964 { 2965 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2966 2967 NvU32 numaSkipReclaimVal = NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_DEFAULT; 2968 2969 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE, &numaSkipReclaimVal) == NV_OK) 2970 { 2971 if (numaSkipReclaimVal > NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_MAX) 2972 { 2973 numaSkipReclaimVal = NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_MAX; 2974 } 2975 } 2976 pmaNumaSetReclaimSkipThreshold(pPma, numaSkipReclaimVal); 2977 2978 // Full FB memory is added and onlined already 2979 if (pKernelMemorySystem->memPartitionNumaInfo[0].bInUse) 2980 { 2981 NV_ASSERT_OK_OR_RETURN(pmaNumaOnlined(pPma, pGpu->numaNodeId, 2982 pKernelMemorySystem->coherentCpuFbBase, 2983 pKernelMemorySystem->numaOnlineSize)); 2984 } 2985 2986 } 2987 2988 return NV_OK; 2989 } 2990 2991 NV_STATUS 2992 memmgrInitFbRegions_IMPL 2993 ( 2994 OBJGPU *pGpu, 2995 MemoryManager *pMemoryManager 2996 ) 2997 { 2998 NV_ASSERT_OR_RETURN(pMemoryManager->Ram.numFBRegions == 0, NV_ERR_INVALID_STATE); 2999 3000 // Don't setup regions if FB is broken and we aren't using L2 cache as "FB". 3001 if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) && 3002 !gpuIsCacheOnlyModeEnabled(pGpu))) 3003 { 3004 // 3005 // Bug 594534: Don't read/write in the FBIO/FBPA space when FB is broken. 3006 // Indicate 32MB FB Memory instead, which is a bit of a hack since zero 3007 // would be more accurate, but zero breaks things. 3008 // 3009 3010 // When ZeroFB + L2Cache mode is enabled, we'll set fbAddrSpaceSizeMb 3011 // appropriately in memmgrInitBaseFbRegions_HAL. 3012 if (!gpuIsCacheOnlyModeEnabled(pGpu)) 3013 { 3014 pMemoryManager->Ram.mapRamSizeMb = pMemoryManager->Ram.fbAddrSpaceSizeMb = 32; 3015 NV_PRINTF(LEVEL_ERROR, 3016 "Bug 594534: HACK: Report 32MB of framebuffer instead of reading registers.\n"); 3017 3018 } 3019 3020 return NV_OK; 3021 } 3022 3023 NV_ASSERT_OK_OR_RETURN(memmgrInitBaseFbRegions_HAL(pGpu, pMemoryManager)); 3024 3025 NV_ASSERT_OK_OR_RETURN(memmgrInitFbRegionsHal_HAL(pGpu, pMemoryManager)); 3026 3027 // 3028 // Build a list of regions sorted by allocation priority 3029 // (highest to lowest). Used for allocations using ObjHeap. 3030 // 3031 memmgrRegenerateFbRegionPriority(pGpu, pMemoryManager); 3032 3033 if (RMCFG_FEATURE_PLATFORM_WINDOWS) 3034 { 3035 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_EXTERNAL_HEAP_CONTROL)) 3036 { 3037 // KMD in WDDM mode 3038 if (pMemoryManager->bMixedDensityFbp) 3039 { 3040 // 3041 // For mixed memory on LDDM platforms, when we are using kernel-managed 3042 // heap (not TCC mode), we want to prefer allocating in slow memory to conserve 3043 // fast memory for applications. 3044 // 3045 pMemoryManager->bPreferSlowRegion = NV_TRUE; 3046 } 3047 } 3048 } 3049 3050 NV_ASSERT_OK_OR_RETURN(memmgrSetPlatformPmaSupport(pGpu, pMemoryManager)); 3051 3052 return NV_OK; 3053 } 3054 3055 /*! 3056 * @brief Register regions to the PMA object 3057 * 3058 * @param pGpu 3059 * @param pMemoryManager 3060 * @param[in] pPma Pointer to the PMA object to register with 3061 * 3062 * @returns On success, returns NV_OK. 3063 * On failure, returns error code. 3064 */ 3065 NV_STATUS 3066 memmgrPmaRegisterRegions_IMPL 3067 ( 3068 OBJGPU *pGpu, 3069 MemoryManager *pMemoryManager, 3070 Heap *pHeap, 3071 PMA *pPma 3072 ) 3073 { 3074 HEAP_TYPE_INTERNAL heapType = pHeap->heapType; 3075 PMA_REGION_DESCRIPTOR pmaRegion; 3076 NvU32 pmaRegionIdx = 0; 3077 NvU32 i; 3078 PMA_BLACKLIST_ADDRESS *pBlacklistPages = NULL; 3079 NvU32 blRegionCount = 0; 3080 NvU32 blPageIndex; 3081 NvU32 blackListCount; 3082 NvU64 base, size; 3083 NV_STATUS status = NV_OK; 3084 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 3085 3086 blackListCount = pHeap->blackListAddresses.count; 3087 base = pHeap->base; 3088 size = pHeap->total; 3089 3090 // 3091 // If there are blacklisted pages, prepare a staging buffer to pass the 3092 // per-region blacklisted pages to PMA 3093 // 3094 if (blackListCount > 0) 3095 { 3096 pBlacklistPages = portMemAllocNonPaged( 3097 sizeof(PMA_BLACKLIST_ADDRESS) * blackListCount); 3098 if (pBlacklistPages == NULL) 3099 { 3100 NV_PRINTF(LEVEL_ERROR, 3101 "Could not allocate memory for blackList!\n"); 3102 status = NV_ERR_NO_MEMORY; 3103 goto _pmaInitFailed; 3104 } 3105 } 3106 3107 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 3108 { 3109 // 3110 // Skip all regions that are completely outside the heap boundry 3111 // OR marked as internal(used for internal RM allocations) 3112 // OR marked as reserved(used for console, display, link training buffer etc.) 3113 // 3114 if ((pMemoryManager->Ram.fbRegion[i].limit < base || 3115 pMemoryManager->Ram.fbRegion[i].base >= (base + size)) || 3116 (pMemoryManager->Ram.fbRegion[i].bInternalHeap) || 3117 (pMemoryManager->Ram.fbRegion[i].bRsvdRegion)) 3118 { 3119 continue; 3120 } 3121 3122 NV_PRINTF(LEVEL_INFO, 3123 "PMA: Register FB region[%d] %llx..%llx EXTERNAL\n", i, 3124 pMemoryManager->Ram.fbRegion[i].base, pMemoryManager->Ram.fbRegion[i].limit); 3125 3126 pmaRegion.base = pMemoryManager->Ram.fbRegion[i].base; 3127 pmaRegion.limit = pMemoryManager->Ram.fbRegion[i].limit; 3128 3129 // Check if the base of managed memory is not based at FB region base. 3130 if (pmaRegion.base < base) 3131 { 3132 pmaRegion.base = base; 3133 } 3134 3135 // check if limit of managed memory is less than FB region limit 3136 if (pmaRegion.limit >= (base + size)) 3137 { 3138 pmaRegion.limit = base + size - 1; 3139 } 3140 3141 pmaRegion.performance = pMemoryManager->Ram.fbRegion[i].performance; 3142 pmaRegion.bSupportCompressed = pMemoryManager->Ram.fbRegion[i].bSupportCompressed; 3143 pmaRegion.bSupportISO = pMemoryManager->Ram.fbRegion[i].bSupportISO; 3144 pmaRegion.bProtected = pMemoryManager->Ram.fbRegion[i].bProtected; 3145 3146 // 3147 // Now we know the region, find if it has any blacklisted pages 3148 // TODO: Try to coalesce to unique 64K pages 3149 // 3150 blRegionCount = 0; 3151 if (pBlacklistPages != NULL) 3152 { 3153 for (blPageIndex = 0; blPageIndex < blackListCount; blPageIndex++) 3154 { 3155 if ((pHeap->blackListAddresses.data[blPageIndex].address 3156 != NV2080_CTRL_FB_OFFLINED_PAGES_INVALID_ADDRESS) && 3157 (pHeap->blackListAddresses.data[blPageIndex].address >= pmaRegion.base) && 3158 (pHeap->blackListAddresses.data[blPageIndex].address <= pmaRegion.limit)) 3159 { 3160 // Collect the region's blacklisted pages 3161 pBlacklistPages[blRegionCount].physOffset = pHeap->blackListAddresses.data[blPageIndex].address; 3162 3163 pBlacklistPages[blRegionCount].bIsDynamic = 3164 ((pHeap->blackListAddresses.data[blPageIndex].type == 3165 NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_MULTIPLE_SBE) || 3166 (pHeap->blackListAddresses.data[blPageIndex].type == 3167 NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE)); 3168 3169 blRegionCount++; 3170 } 3171 } 3172 } 3173 3174 NV_PRINTF(LEVEL_INFO, 3175 "Register FB region %llx..%llx of size %llx with PMA\n", 3176 pmaRegion.base, pmaRegion.limit, 3177 pmaRegion.limit - pmaRegion.base + 1); 3178 // 3179 // Register the region for PMA management, and note if asynchronous 3180 // scrubbing is enabled. Synchronous scrubbing is done before 3181 // heap/PMA is initialized, but asynchronously scrubbed pages will 3182 // need to be unmarked once they are scrubbed. 3183 // 3184 status = pmaRegisterRegion(pPma, pmaRegionIdx, 3185 memmgrEccScrubInProgress_HAL(pGpu, pMemoryManager), 3186 &pmaRegion, blRegionCount, 3187 ((blRegionCount==0) ? NULL : pBlacklistPages)); 3188 if (status != NV_OK) 3189 { 3190 NV_PRINTF(LEVEL_ERROR, 3191 "failed to register FB region %llx..%llx with PMA\n", 3192 pmaRegion.base, pmaRegion.limit); 3193 DBG_BREAKPOINT(); 3194 goto _pmaInitFailed; 3195 } 3196 pmaRegionIdx++; 3197 } 3198 3199 // 3200 // bug #200354346, make sure the RM reserved region(s) are 3201 // scrubbed during the region creation itself. Top Down scrubber, 3202 // skips the RM reserved region(s) because the assumption is, they 3203 // are pre-scrubbed. 3204 // 3205 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 3206 memmgrScrubInternalRegions_HAL(pGpu, pMemoryManager); 3207 3208 _pmaInitFailed: 3209 portMemFree(pBlacklistPages); 3210 3211 if ((status == NV_OK) && (pKernelMemorySystem->fbOverrideStartKb != 0)) 3212 { 3213 NvU64 allocSize = NV_ALIGN_UP(((NvU64)pKernelMemorySystem->fbOverrideStartKb << 10), PMA_GRANULARITY); 3214 NvU32 numPages = (NvU32)(allocSize >> PMA_PAGE_SHIFT); 3215 PMA_ALLOCATION_OPTIONS allocOptions = {0}; 3216 3217 allocOptions.flags = PMA_ALLOCATE_CONTIGUOUS; 3218 allocOptions.flags |= PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE; 3219 allocOptions.physBegin = 0; 3220 allocOptions.physEnd = allocSize - 1; 3221 3222 // This is intentionally thrown away 3223 NvU64 *pPages = NULL; 3224 pPages = portMemAllocNonPaged(numPages * sizeof(NvU64)); 3225 if (pPages != NULL) 3226 { 3227 // Accommodate the regkey override for FB start 3228 status = pmaAllocatePages(pPma, numPages, _PMA_64KB, &allocOptions, pPages); 3229 portMemFree(pPages); 3230 } 3231 } 3232 if (status != NV_OK) 3233 { 3234 if (memmgrIsPmaInitialized(pMemoryManager)) 3235 { 3236 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 3237 { 3238 memmgrSetPmaInitialized(pMemoryManager, NV_FALSE); 3239 } 3240 pmaDestroy(pPma); 3241 } 3242 } 3243 3244 return status; 3245 } 3246 3247 /*! 3248 * @brief Allocate internal handles for MIG partition memory allocation 3249 */ 3250 NV_STATUS 3251 memmgrAllocMIGMemoryAllocationInternalHandles_IMPL 3252 ( 3253 OBJGPU *pGpu, 3254 MemoryManager *pMemoryManager 3255 ) 3256 { 3257 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 3258 3259 NV_ASSERT_OR_RETURN(pMemoryManager->MIGMemoryPartitioningInfo.hClient == NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 3260 NV_ASSERT_OK_OR_RETURN( 3261 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, 3262 &pMemoryManager->MIGMemoryPartitioningInfo.hClient, 3263 &pMemoryManager->MIGMemoryPartitioningInfo.hDevice, 3264 &pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice)); 3265 3266 return NV_OK; 3267 } 3268 3269 /*! 3270 * @brief Free internal handles used to support MIG memory partitioning 3271 */ 3272 void 3273 memmgrFreeMIGMemoryAllocationInternalHandles_IMPL 3274 ( 3275 OBJGPU *pGpu, 3276 MemoryManager *pMemoryManager 3277 ) 3278 { 3279 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 3280 3281 rmapiutilFreeClientAndDeviceHandles(pRmApi, 3282 &pMemoryManager->MIGMemoryPartitioningInfo.hClient, 3283 &pMemoryManager->MIGMemoryPartitioningInfo.hDevice, 3284 &pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice); 3285 } 3286 3287 /*! 3288 * @brief Gets free memory (client visible) for all valid GPU instances 3289 */ 3290 void 3291 memmgrGetFreeMemoryForAllMIGGPUInstances_IMPL 3292 ( 3293 OBJGPU *pGpu, 3294 MemoryManager *pMemoryManager, 3295 NvU64 *pBytes 3296 ) 3297 { 3298 NvU64 val = 0; 3299 Heap *pHeap = NULL; 3300 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3301 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance; 3302 3303 *pBytes = 0; 3304 3305 FOR_EACH_VALID_GPU_INSTANCE(pGpu, pKernelMIGManager, pKernelMIGGPUInstance) 3306 { 3307 NV_ASSERT(pKernelMIGGPUInstance->pMemoryPartitionHeap != NULL); 3308 pHeap = pKernelMIGGPUInstance->pMemoryPartitionHeap; 3309 3310 if (memmgrIsPmaInitialized(pMemoryManager)) 3311 pmaGetFreeMemory(&pHeap->pmaObject, &val); 3312 else 3313 heapGetFree(pHeap, &val); 3314 3315 *pBytes += val; 3316 } 3317 FOR_EACH_VALID_GPU_INSTANCE_END(); 3318 } 3319 3320 /*! 3321 * @brief Gets total memory for all valid GPU instances 3322 * 3323 * @param pGpu 3324 * @param pMemoryManager 3325 * @param[out] pBytes pointer to the total memory 3326 * 3327 */ 3328 void 3329 memmgrGetTotalMemoryForAllMIGGPUInstances_IMPL 3330 ( 3331 OBJGPU *pGpu, 3332 MemoryManager *pMemoryManager, 3333 NvU64 *pBytes 3334 ) 3335 { 3336 NvU64 val = 0; 3337 Heap *pHeap = NULL; 3338 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3339 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance; 3340 3341 *pBytes = 0; 3342 3343 FOR_EACH_VALID_GPU_INSTANCE(pGpu, pKernelMIGManager, pKernelMIGGPUInstance) 3344 { 3345 NV_ASSERT(pKernelMIGGPUInstance->pMemoryPartitionHeap != NULL); 3346 pHeap = pKernelMIGGPUInstance->pMemoryPartitionHeap; 3347 3348 if (memmgrIsPmaInitialized(pMemoryManager)) 3349 pmaGetTotalMemory(&pHeap->pmaObject, &val); 3350 else 3351 heapGetSize(pHeap, &val); 3352 3353 *pBytes += val; 3354 } 3355 FOR_EACH_VALID_GPU_INSTANCE_END(); 3356 } 3357 3358 void 3359 memmgrGetTopLevelScrubberStatus_IMPL 3360 ( 3361 OBJGPU *pGpu, 3362 MemoryManager *pMemoryManager, 3363 NvBool *pbTopLevelScrubberEnabled, 3364 NvBool *pbTopLevelScrubberConstructed 3365 ) 3366 { 3367 NvBool bTopLevelScrubberEnabled = NV_FALSE; 3368 NvBool bTopLevelScrubberConstructed = NV_FALSE; 3369 NvU32 pmaConfigs = PMA_QUERY_SCRUB_ENABLED | PMA_QUERY_SCRUB_VALID; 3370 3371 if (memmgrIsPmaInitialized(pMemoryManager)) 3372 { 3373 Heap *pHeap = GPU_GET_HEAP(pGpu); 3374 NV_ASSERT_OK(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfigs)); 3375 bTopLevelScrubberEnabled = (pmaConfigs & PMA_QUERY_SCRUB_ENABLED) != 0x0; 3376 bTopLevelScrubberConstructed = (pmaConfigs & PMA_QUERY_SCRUB_VALID) != 0x0; 3377 } 3378 3379 if (pbTopLevelScrubberEnabled != NULL) 3380 *pbTopLevelScrubberEnabled = bTopLevelScrubberEnabled; 3381 if (pbTopLevelScrubberConstructed != NULL) 3382 *pbTopLevelScrubberConstructed = bTopLevelScrubberConstructed; 3383 } 3384 3385 /*! 3386 * @brief Return the full address range for the partition assigend for the vGPU. 3387 * 3388 * @param[in] pGpu 3389 * @param[in] pMemoryManager 3390 * @param[out] base reference to the base address of the partition 3391 * @param[out] size reference to the overall size of the partition 3392 */ 3393 static void 3394 _memmgrGetFullMIGAddrRange 3395 ( 3396 OBJGPU *pGpu, 3397 MemoryManager *pMemoryManager, 3398 NvU64 *base, 3399 NvU64 *size 3400 ) 3401 { 3402 NvU32 i; 3403 NvU64 lo, hi; 3404 3405 *base = 0; 3406 *size = 0; 3407 if (pMemoryManager->Ram.numFBRegions == 0) 3408 { 3409 return; 3410 } 3411 3412 lo = pMemoryManager->Ram.fbRegion[0].base; 3413 hi = pMemoryManager->Ram.fbRegion[0].limit; 3414 3415 for (i = 1; i < pMemoryManager->Ram.numFBRegions; i++) 3416 { 3417 if (pMemoryManager->Ram.fbRegion[i].base < lo) 3418 { 3419 lo = pMemoryManager->Ram.fbRegion[i].base; 3420 } 3421 3422 if (pMemoryManager->Ram.fbRegion[i].limit > hi) 3423 { 3424 hi = pMemoryManager->Ram.fbRegion[i].limit; 3425 } 3426 } 3427 3428 *base = lo; 3429 *size = hi - lo + 1; 3430 } 3431 3432 /*! 3433 * @brief Discover MIG partitionable memory range based on PMA status 3434 */ 3435 NV_STATUS 3436 memmgrDiscoverMIGPartitionableMemoryRange_VF 3437 ( 3438 OBJGPU *pGpu, 3439 MemoryManager *pMemoryManager, 3440 NV_RANGE *pMemoryRange 3441 ) 3442 { 3443 NvU64 size; 3444 NvU64 base; 3445 3446 // Set memory information 3447 if (!memmgrIsPmaInitialized(pMemoryManager)) 3448 { 3449 Heap *pHeap = GPU_GET_HEAP(pGpu); 3450 NvU64 freeMem; 3451 NvU64 bytesTotal; 3452 NvU64 offset; 3453 3454 NV_ASSERT_OK_OR_RETURN(heapInfo(pHeap, &freeMem, &bytesTotal, &base, 3455 &offset, &size)); 3456 3457 // 3458 // offset is the starting address of biggest empty block whose size is 3459 // returned and we care about the base of largest empty block 3460 // 3461 base = offset; 3462 } 3463 else 3464 { 3465 // 3466 // In the case of vGPU, pmaGetLargestFree only returns the user-visible 3467 // PMA region and not the reserved/internal regions that constitute the 3468 // overall partition size assigned to the vGPU. 3469 // This is misleading as pMemoryManager->partitionableMemoryRange is expected to 3470 // represent the actual partition size. 3471 // 3472 _memmgrGetFullMIGAddrRange(pGpu, pMemoryManager, &base, &size); 3473 } 3474 3475 *pMemoryRange = rangeMake(base, base + size - 1); 3476 3477 return NV_OK; 3478 } 3479 3480 NV_STATUS 3481 memmgrValidateFBEndReservation_PF 3482 ( 3483 OBJGPU *pGpu, 3484 MemoryManager *pMemoryManager 3485 ) 3486 { 3487 NV_STATUS status; 3488 3489 NV_ASSERT_TRUE_OR_GOTO(status, 3490 (pGpu != NULL) && 3491 (pMemoryManager != NULL), 3492 NV_ERR_INVALID_ARGUMENT, 3493 memmgrValidateFBEndReservation_PF_exit); 3494 3495 // If we reserved more memory from RM than we previously estimated 3496 if (pMemoryManager->rsvdMemorySize > memmgrGetFBEndReserveSizeEstimate_HAL(pGpu, pMemoryManager)) 3497 { 3498 NV_PRINTF(LEVEL_ERROR, 3499 "End of FB reservation was not enough (%u vs %u). Failing to boot.\n", 3500 memmgrGetFBEndReserveSizeEstimate_HAL(pGpu, pMemoryManager), 3501 pMemoryManager->rsvdMemorySize); 3502 3503 NV_ASSERT_OK_OR_GOTO(status, 3504 NV_ERR_INSUFFICIENT_RESOURCES, 3505 memmgrValidateFBEndReservation_PF_exit); 3506 } 3507 3508 memmgrValidateFBEndReservation_PF_exit: 3509 return status; 3510 } 3511 3512 NV_STATUS 3513 memmgrReserveMemoryForPmu_MONOLITHIC 3514 ( 3515 OBJGPU *pGpu, 3516 MemoryManager *pMemoryManager 3517 ) 3518 { 3519 NV_STATUS status = NV_OK; 3520 3521 return status; 3522 } 3523 3524 3525 NV_STATUS 3526 memmgrReserveMemoryForFsp_IMPL 3527 ( 3528 OBJGPU *pGpu, 3529 MemoryManager *pMemoryManager 3530 ) 3531 { 3532 KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu); 3533 3534 // 3535 // If we sent FSP commands to boot ACR, we need to allocate the surfaces 3536 // used by FSP and ACR as WPR/FRTS here from the reserved heap 3537 // 3538 if (pKernelFsp && (!pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_DISABLE_FRTS_VIDMEM) && 3539 (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_BOOT_COMMAND_OK)))) 3540 { 3541 3542 // For GSP-RM flow, we don't need to allocate WPR since it is handled by CPU 3543 if (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM)) 3544 { 3545 return NV_OK; 3546 } 3547 3548 } 3549 return NV_OK; 3550 } 3551 3552 NvU64 3553 memmgrGetVgpuHostRmReservedFb_KERNEL 3554 ( 3555 OBJGPU *pGpu, 3556 MemoryManager *pMemoryManager, 3557 NvU32 vgpuTypeId 3558 ) 3559 { 3560 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3561 NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS params = {0}; 3562 3563 params.vgpuTypeId = vgpuTypeId; 3564 // Send to GSP to get amount of FB reserved for the host 3565 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, 3566 pGpu->hInternalClient, 3567 pGpu->hInternalSubdevice, 3568 NV2080_CTRL_CMD_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB, 3569 ¶ms, 3570 sizeof(params))); 3571 return params.hostReservedFb; 3572 } 3573 3574 /*! 3575 * @brief Memory Manager State post load 3576 * 3577 * @param[in] pGpu GPU pointer 3578 * @param[in/out] pMemoryManager MemoryManager pointer 3579 * @param[in] flags State transition flags 3580 * 3581 * @returns On success, returns NV_OK. 3582 * On failure, returns error code. 3583 */ 3584 NV_STATUS 3585 memmgrStatePostLoad_IMPL 3586 ( 3587 OBJGPU *pGpu, 3588 MemoryManager *pMemoryManager, 3589 NvU32 flags 3590 ) 3591 { 3592 if (memmgrIsLocalEgmSupported(pMemoryManager)) 3593 { 3594 NvU64 egmPhysAddr, egmSize; 3595 NvS32 egmNodeId; 3596 NvU32 data32; 3597 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 3598 3599 pMemoryManager->localEgmNodeId = -1; 3600 if (gpuIsSelfHosted(pGpu) && 3601 pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP) && // EGM can be enabled only in C2C path. 3602 (osGetEgmInfo(pGpu, &egmPhysAddr, &egmSize, &egmNodeId) == NV_OK) && 3603 (egmSize != 0)) 3604 { 3605 pMemoryManager->localEgmBasePhysAddr = egmPhysAddr; 3606 pMemoryManager->localEgmSize = egmSize; 3607 pMemoryManager->localEgmNodeId = egmNodeId; 3608 // 3609 // Using fixed Peer ID 7 for local EGM so that vGPU 3610 // migration doesn't fail because of peer id conflict in 3611 // the new host system. 3612 // 3613 pMemoryManager->localEgmPeerId = 7; 3614 pMemoryManager->bLocalEgmEnabled = NV_TRUE; 3615 } 3616 3617 // 3618 // regkey can override the production flow values. 3619 // Note that this could cause an issue with vGPU migration 3620 // if one host system uses regkey to override the EGM peer id 3621 // and other host system doesn't. 3622 // 3623 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_LOCAL_EGM_PEER_ID, &data32) == NV_OK) 3624 { 3625 pMemoryManager->bLocalEgmEnabled = NV_TRUE; 3626 pMemoryManager->localEgmPeerId = data32; 3627 } 3628 } 3629 3630 // 3631 // Reserve the peerID used for local EGM so that the peerID isn't 3632 // resused for other peer Gpus. 3633 // 3634 if (memmgrIsLocalEgmEnabled(pMemoryManager)) 3635 { 3636 if (kbusReserveP2PPeerIds_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), NVBIT(pMemoryManager->localEgmPeerId)) == NV_OK) 3637 { 3638 if (!IS_VIRTUAL_WITH_SRIOV(pGpu)) 3639 { 3640 NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS params = { 0 }; 3641 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3642 NV_STATUS status; 3643 3644 params.egmPeerId = pMemoryManager->localEgmPeerId; 3645 3646 // Call physical HSHUB to program the EGM PeerId settings. 3647 3648 status = pRmApi->Control(pRmApi, 3649 pGpu->hInternalClient, 3650 pGpu->hInternalSubdevice, 3651 NV2080_CTRL_CMD_INTERNAL_HSHUB_EGM_CONFIG, 3652 ¶ms, 3653 sizeof(params)); 3654 if (status != NV_OK) 3655 { 3656 NV_PRINTF(LEVEL_ERROR, "HSHUB programming failed for EGM Peer ID: %u\n", 3657 pMemoryManager->localEgmPeerId); 3658 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 3659 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3660 return status; 3661 } 3662 } 3663 } 3664 else 3665 { 3666 NV_PRINTF(LEVEL_ERROR, 3667 "Peer ID specified for local EGM already in use!\n"); 3668 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 3669 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3670 return NV_ERR_INVALID_ARGUMENT; 3671 } 3672 } 3673 else 3674 { 3675 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3676 } 3677 return NV_OK; 3678 } 3679 3680 NV_STATUS 3681 memmgrInitCeUtils_IMPL 3682 ( 3683 MemoryManager *pMemoryManager, 3684 NvBool bFifoLite 3685 ) 3686 { 3687 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 3688 NV0050_ALLOCATION_PARAMETERS ceUtilsParams = {0}; 3689 3690 NV_ASSERT_OR_RETURN(pMemoryManager->pCeUtils == NULL, NV_ERR_INVALID_STATE); 3691 3692 if (!bFifoLite && pMemoryManager->pCeUtilsSuspended != NULL) 3693 { 3694 pMemoryManager->pCeUtils = pMemoryManager->pCeUtilsSuspended; 3695 pMemoryManager->pCeUtilsSuspended = NULL; 3696 return NV_OK; 3697 } 3698 3699 if (bFifoLite) 3700 ceUtilsParams.flags |= DRF_DEF(0050_CEUTILS, _FLAGS, _FIFO_LITE, _TRUE); 3701 3702 if (pMemoryManager->bCePhysicalVidmemAccessNotSupported) 3703 ceUtilsParams.flags |= DRF_DEF(0050_CEUTILS, _FLAGS, _VIRTUAL_MODE, _TRUE); 3704 3705 NV_ASSERT_OK_OR_RETURN(objCreate(&pMemoryManager->pCeUtils, pMemoryManager, CeUtils, ENG_GET_GPU(pMemoryManager), NULL, &ceUtilsParams)); 3706 3707 NV_STATUS status = memmgrTestCeUtils(pGpu, pMemoryManager); 3708 NV_ASSERT_OK(status); 3709 if (status != NV_OK) 3710 { 3711 memmgrDestroyCeUtils(pMemoryManager, NV_FALSE); 3712 } 3713 3714 return status; 3715 } 3716 3717 void 3718 memmgrDestroyCeUtils_IMPL 3719 ( 3720 MemoryManager *pMemoryManager, 3721 NvBool bSuspendCeUtils 3722 ) 3723 { 3724 if (bSuspendCeUtils) 3725 { 3726 NV_ASSERT_OR_RETURN_VOID(pMemoryManager->pCeUtilsSuspended == NULL); 3727 pMemoryManager->pCeUtilsSuspended = pMemoryManager->pCeUtils; 3728 } 3729 else 3730 { 3731 objDelete(pMemoryManager->pCeUtils); 3732 } 3733 pMemoryManager->pCeUtils = NULL; 3734 } 3735