1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "gpu/mem_mgr/mem_mgr.h" 25 #include "gpu/mem_mgr/heap.h" 26 #include "gpu/mem_sys/kern_mem_sys.h" 27 #include "gpu/mem_mgr/mem_utils.h" 28 #include "gpu/mem_mgr/ce_utils.h" 29 #include "mem_mgr/video_mem.h" 30 #include "gpu/mem_mgr/fbsr.h" 31 #include "gpu/mmu/kern_gmmu.h" 32 #include "gpu/bus/kern_bus.h" 33 #include "gpu/bif/kernel_bif.h" 34 #include "core/locks.h" 35 #include "virtualization/kernel_vgpu_mgr.h" 36 #include "vgpu/rpc.h" 37 #include "core/thread_state.h" 38 #include "nvRmReg.h" 39 #include "gpu/fsp/kern_fsp.h" 40 #include "gpu/mem_mgr/phys_mem_allocator/numa.h" 41 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 42 #include "kernel/rmapi/rs_utils.h" 43 #include "rmapi/rmapi_utils.h" 44 #include "mmu/gmmu_fmt.h" 45 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 46 #include "class/cl503c.h" 47 #include "class/cl906f.h" // GF100_CHANNEL_GPFIFO 48 #include "os/os.h" 49 #include "gpu/gsp/kernel_gsp.h" 50 #include "gpu/conf_compute/conf_compute.h" 51 52 #include "class/cl0050.h" 53 54 static NV_STATUS _memmgrCreateFBSR(MemoryManager *pMemoryManager, NvU32); 55 static NV_STATUS _memmgrCreateChildObjects(MemoryManager *pMemoryManager); 56 static void _memmgrInitRegistryOverrides(OBJGPU *pGpu, MemoryManager *pMemoryManager); 57 static NV_STATUS _memmgrInitMIGMemoryPartitionHeap(OBJGPU *pGpu, MemoryManager *pMemoryManager, 58 NvU32 swizzId, NV_RANGE *pAddrRange, 59 Heap **ppMemoryPartitionHeap); 60 static NV_STATUS _memmgrAllocInternalClientObjects(OBJGPU *pGpu, 61 MemoryManager *pMemoryManager); 62 static void _memmgrFreeInternalClientObjects(MemoryManager *pMemoryManager); 63 64 #define MEMUTILS_CHANNEL_GPFIFO_SIZE (NV906F_GP_ENTRY__SIZE * MEMUTILS_NUM_GPFIFIO_ENTRIES) 65 66 NV_STATUS 67 memmgrConstructEngine_IMPL 68 ( 69 OBJGPU *pGpu, 70 MemoryManager *pMemoryManager, 71 ENGDESCRIPTOR engDesc 72 ) 73 { 74 NV_STATUS rmStatus; 75 76 pMemoryManager->overrideInitHeapMin = 0; 77 pMemoryManager->overrideHeapMax = ~0ULL; 78 79 // Create the children 80 rmStatus = _memmgrCreateChildObjects(pMemoryManager); 81 if (rmStatus != NV_OK) 82 return rmStatus; 83 84 pMemoryManager->MIGMemoryPartitioningInfo.hClient = NV01_NULL_OBJECT; 85 pMemoryManager->MIGMemoryPartitioningInfo.hDevice = NV01_NULL_OBJECT; 86 pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice = NV01_NULL_OBJECT; 87 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = NV_RANGE_EMPTY; 88 89 return NV_OK; 90 } 91 92 void 93 memmgrDestruct_IMPL 94 ( 95 MemoryManager *pMemoryManager 96 ) 97 { 98 NvU32 i; 99 100 for (i = 0; i < NUM_FBSR_TYPES; i++) 101 { 102 objDelete(pMemoryManager->pFbsr[i]); 103 pMemoryManager->pFbsr[i] = NULL; 104 } 105 106 objDelete(pMemoryManager->pHeap); 107 pMemoryManager->pHeap = NULL; 108 109 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = NV_RANGE_EMPTY; 110 } 111 112 static void 113 _memmgrInitRegistryOverrides(OBJGPU *pGpu, MemoryManager *pMemoryManager) 114 { 115 NvU32 data32; 116 117 // Check for ram size override. 118 pMemoryManager->Ram.fbOverrideSizeMb = (NvU64)~0; 119 if ((osReadRegistryDword(pGpu, NV_REG_STR_OVERRIDE_FB_SIZE, &data32) == NV_OK) && 120 (data32 != 0)) 121 { 122 NV_PRINTF(LEVEL_WARNING, "Regkey %s = %dM\n", 123 NV_REG_STR_OVERRIDE_FB_SIZE, data32); 124 // Used to override heap sizing at create 125 pMemoryManager->Ram.fbOverrideSizeMb = data32; 126 } 127 128 // 129 // Scrub on Free is enabled by default for GK110+ 130 // The reg key will be used to disable the scrub on free 131 // 132 if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DISABLE_SCRUB_ON_FREE, 133 &data32) == NV_OK) && data32) 134 { 135 pMemoryManager->bScrubOnFreeEnabled = NV_FALSE; 136 } 137 138 if ((osReadRegistryDword(pGpu, NV_REG_STR_RM_DISABLE_FAST_SCRUBBER, 139 &data32) == NV_OK) && data32) 140 { 141 pMemoryManager->bFastScrubberEnabled = NV_FALSE; 142 } 143 144 if (NV_OK == osReadRegistryDword(pGpu, NV_REG_STR_RM_SYSMEM_PAGE_SIZE, &data32)) 145 { 146 switch (data32) 147 { 148 case RM_PAGE_SIZE: 149 case RM_PAGE_SIZE_64K: 150 case RM_PAGE_SIZE_HUGE: 151 case RM_PAGE_SIZE_512M: 152 break; 153 default: 154 NV_ASSERT(0); 155 NV_PRINTF(LEVEL_ERROR, 156 "Sysmem page size 0x%x not supported! Defaulting to 4KB\n", 157 data32); 158 data32 = RM_PAGE_SIZE; 159 } 160 pMemoryManager->sysmemPageSize = data32; 161 } 162 else 163 { 164 pMemoryManager->sysmemPageSize = RM_PAGE_SIZE; 165 166 } 167 168 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ALLOW_SYSMEM_LARGE_PAGES, &data32) == NV_OK) 169 { 170 pMemoryManager->bAllowSysmemHugePages = data32 ? NV_TRUE : NV_FALSE; 171 } 172 else 173 { 174 pMemoryManager->bAllowSysmemHugePages = NV_FALSE; 175 } 176 177 // This key should not be used on physical (GSP) RM. 178 if (!RMCFG_FEATURE_PLATFORM_GSP) 179 { 180 // Allow user to increase size of RM reserved heap via a regkey 181 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_INCREASE_RSVD_MEMORY_SIZE_MB, 182 &data32) == NV_OK) 183 { 184 pMemoryManager->rsvdMemorySizeIncrement = (NvU64)data32 << 20; 185 NV_PRINTF(LEVEL_ERROR, 186 "User specified increase in reserved size = %d MBs\n", 187 data32); 188 } 189 } 190 191 if (osReadRegistryDword(pGpu, 192 NV_REG_STR_RM_DISABLE_NONCONTIGUOUS_ALLOCATION, 193 &data32) == NV_OK) 194 { 195 if (data32 == NV_REG_STR_RM_DISABLE_NONCONTIGUOUS_ALLOCATION_TRUE) 196 { 197 pMemoryManager->bAllowNoncontiguousAllocation = NV_FALSE; 198 } 199 } 200 201 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_PAGED_DMA, &data32) == NV_OK) 202 { 203 pMemoryManager->bEnableFbsrPagedDma = !!data32; 204 } 205 206 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_FILE_MODE, &data32) == NV_OK) 207 { 208 if (data32 && RMCFG_FEATURE_PLATFORM_UNIX) 209 { 210 pMemoryManager->bEnableFbsrFileMode = NV_TRUE; 211 } 212 } 213 214 // 215 // Override PMA enable. PDB_PROP_FB_PMA_ENABLED is reconciled with 216 // PDB_PROP_FB_PLATFORM_PMA_SUPPORT to decide whether to enable PMA. 217 // 218 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_PMA, &data32) == NV_OK) 219 { 220 if (data32 == NV_REG_STR_RM_ENABLE_PMA_YES) 221 { 222 pMemoryManager->bPmaEnabled = NV_TRUE; 223 } 224 else 225 { 226 pMemoryManager->bPmaEnabled = NV_FALSE; 227 } 228 } 229 230 if (RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE)) 231 { 232 pMemoryManager->bFbsrWddmModeEnabled = NV_TRUE; 233 } 234 235 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_FBSR_WDDM_MODE, &data32) == NV_OK) 236 { 237 pMemoryManager->bFbsrWddmModeEnabled = !!data32; 238 } 239 240 // 241 // Override PMA managed client page tables. 242 // NOTE: This is WAR for bug #s 1946145 and 1971628. 243 // This should be removed as part of heap removal and PMA refactor. 244 // 245 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_PMA_MANAGED_PTABLES, 246 &data32) == NV_OK) 247 { 248 if (data32 == NV_REG_STR_RM_ENABLE_PMA_MANAGED_PTABLES_NO) 249 { 250 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 251 } 252 } 253 254 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_ADDRTREE, &data32) == NV_OK) 255 { 256 if (data32 == NV_REG_STR_RM_ENABLE_ADDRTREE_YES) 257 { 258 pMemoryManager->bPmaAddrTree = NV_TRUE; 259 NV_PRINTF(LEVEL_ERROR, "Enabled address tree for PMA via regkey.\n"); 260 } 261 } 262 else if (RMCFG_FEATURE_PLATFORM_MODS) 263 { 264 pMemoryManager->bPmaAddrTree = NV_TRUE; 265 NV_PRINTF(LEVEL_ERROR, "Enabled address tree for PMA for MODS.\n"); 266 } 267 } 268 269 NV_STATUS 270 memmgrStatePreInitLocked_IMPL 271 ( 272 OBJGPU *pGpu, 273 MemoryManager *pMemoryManager 274 ) 275 { 276 if (IS_GSP_CLIENT(pGpu)) 277 { 278 // 279 // Temporary hack to get OpenRM working without breaking SLI 280 // After fixing CORERM-4078, memmgrInitFbRegions() call should be removed from memsysStateInitLocked() 281 // and only left here 282 // 283 NV_ASSERT_OK_OR_RETURN(memmgrInitFbRegions(pGpu, pMemoryManager)); 284 } 285 286 // Determine the size of reserved memory 287 NV_ASSERT_OK_OR_RETURN(memmgrPreInitReservedMemory_HAL(pGpu, pMemoryManager)); 288 289 return NV_OK; 290 } 291 292 NV_STATUS 293 memmgrStateInitLocked_IMPL 294 ( 295 OBJGPU *pGpu, 296 MemoryManager *pMemoryManager 297 ) 298 { 299 NV_STATUS status = NV_OK; 300 NvU32 i; 301 NvBool bDynamicPageOffliningDisable = NV_FALSE; 302 303 NV_ASSERT_OK_OR_RETURN(memmgrInitReservedMemory_HAL(pGpu, pMemoryManager, pMemoryManager->Ram.fbAddrSpaceSizeMb << 20)); 304 305 _memmgrInitRegistryOverrides(pGpu, pMemoryManager); 306 307 // 308 // Enable dynamic page blacklisting at this point before we call CreateHeap 309 // since it internally calls heapGetBlacklistPages which depends on this property 310 // 311 if (!bDynamicPageOffliningDisable) 312 memmgrEnableDynamicPageOfflining_HAL(pGpu, pMemoryManager); 313 314 memmgrScrubRegistryOverrides_HAL(pGpu, pMemoryManager); 315 memmgrScrubInit_HAL(pGpu, pMemoryManager); 316 317 // 318 // Allocate framebuffer heap. All memory must be allocated from here to keep the world 319 // consistent (N.B. the heap size has been reduced by the amount of instance memory). 320 // 321 status = memmgrCreateHeap(pMemoryManager); 322 if (status != NV_OK) 323 { 324 return status; 325 } 326 327 // 328 // Just set up the memory pool now (basic init stuff). Actual physical 329 // frames are *NOT* added to the pool at this stage. 330 // 331 status = memmgrPageLevelPoolsCreate(pGpu, pMemoryManager); 332 if (status != NV_OK) 333 { 334 return status; 335 } 336 337 // RMCONFIG: only if FBSR engine is enabled 338 if (RMCFG_MODULE_FBSR) 339 { 340 // 341 // If a configuration is not supported, do not initialize 342 // the corresponding fbsr engine. 343 // 344 if (pMemoryManager->bFbsrWddmModeEnabled) 345 { 346 pMemoryManager->fbsrStartMode = FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED; 347 } 348 else if (pMemoryManager->bEnableFbsrPagedDma) 349 { 350 pMemoryManager->fbsrStartMode = FBSR_TYPE_PAGED_DMA; 351 } 352 else if (pMemoryManager->bEnableFbsrFileMode) 353 { 354 pMemoryManager->fbsrStartMode = FBSR_TYPE_FILE; 355 } 356 else 357 { 358 pMemoryManager->fbsrStartMode = FBSR_TYPE_PERSISTENT; 359 } 360 361 for (i = pMemoryManager->fbsrStartMode; i < NUM_FBSR_TYPES; i++) 362 { 363 if (!pMemoryManager->bPersistentStandbyBuffer && 364 (i == FBSR_TYPE_PERSISTENT)) 365 { 366 continue; 367 } 368 369 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) && 370 (i == FBSR_TYPE_PAGED_DMA || i == FBSR_TYPE_DMA)) 371 { 372 continue; 373 } 374 375 status = fbsrInit_HAL(pGpu, pMemoryManager->pFbsr[i]); 376 377 // 378 // If one fbsr scheme failed, proceed to initializing the other 379 // fallback options. 380 // 381 if (status != NV_OK) 382 { 383 NV_PRINTF(LEVEL_WARNING, 384 "fbsrInit failed for supported type %d suspend-resume scheme\n", 385 i); 386 continue; 387 } 388 } 389 } 390 391 status = _memmgrAllocInternalClientObjects(pGpu, pMemoryManager); 392 if (status != NV_OK) 393 { 394 // 395 // TODO: Bug 3482892: Need a way to roll back StateInit 396 // steps in case of a failure 397 // WAR for now is to cleanup with memmgrStateDestroy(). 398 // 399 memmgrStateDestroy(pGpu, pMemoryManager); 400 return status; 401 } 402 403 return NV_OK; 404 } 405 406 NV_STATUS 407 memmgrVerifyGspDmaOps_IMPL 408 ( 409 OBJGPU *pGpu, 410 MemoryManager *pMemoryManager 411 ) 412 { 413 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 414 NV_STATUS status = NV_OK; 415 MEMORY_DESCRIPTOR *pMemDesc; 416 NvU8 *pTestBuffer; 417 NvU32 testData = 0xdeadbeef; 418 TRANSFER_SURFACE surf = {0}; 419 420 // 421 // Return early if CPU access to CPR vidmem is allowed as GSP DMA 422 // is not needed in this case 423 // 424 if (!kbusIsBarAccessBlocked(pKernelBus)) 425 return NV_OK; 426 427 pTestBuffer = portMemAllocNonPaged(4096); 428 NV_ASSERT_OR_RETURN(pTestBuffer != NULL, NV_ERR_INSUFFICIENT_RESOURCES); 429 430 portMemSet(pTestBuffer, 0, 4096); 431 432 status = memdescCreate(&pMemDesc, pGpu, RM_PAGE_SIZE, RM_PAGE_SIZE, 433 NV_TRUE, ADDR_FBMEM, NV_MEMORY_UNCACHED, 0); 434 NV_ASSERT_OR_RETURN(status == NV_OK, status); 435 436 status = memdescAlloc(pMemDesc); 437 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 438 439 surf.pMemDesc = pMemDesc; 440 surf.offset = sizeof(NvU32); // Choosing a random offset 441 442 // Write test data to FB using GSP 443 status = memmgrMemWrite(pMemoryManager, &surf, &testData, sizeof(NvU32), 444 TRANSFER_FLAGS_NONE); 445 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 446 447 // Read the same location using GSP and confirm that GSP read is also working fine 448 status = memmgrMemRead(pMemoryManager, &surf, pTestBuffer, sizeof(NvU32), 449 TRANSFER_FLAGS_NONE); 450 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 451 452 if (((NvU32*)pTestBuffer)[0] != testData) 453 { 454 NV_PRINTF(LEVEL_ERROR, "####################################################\n"); 455 NV_PRINTF(LEVEL_ERROR, " Read back of data using GSP shows mismatch\n"); 456 NV_PRINTF(LEVEL_ERROR, " Test data: 0x%x Read Data: 0x%x\n", testData, ((NvU32*)pTestBuffer)[0]); 457 NV_PRINTF(LEVEL_ERROR, "####################################################\n"); 458 status = NV_ERR_INVALID_STATE; 459 NV_ASSERT_OR_GOTO(status == NV_OK, failed); 460 } 461 else 462 { 463 NV_PRINTF(LEVEL_INFO, "####################################################\n"); 464 NV_PRINTF(LEVEL_INFO, " Read back of data using GSP confirms write\n"); 465 NV_PRINTF(LEVEL_INFO, "####################################################\n"); 466 } 467 468 failed: 469 memdescFree(pMemDesc); 470 memdescDestroy(pMemDesc); 471 portMemFree(pTestBuffer); 472 473 return status; 474 } 475 476 NV_STATUS 477 memmgrStateLoad_IMPL 478 ( 479 OBJGPU *pGpu, 480 MemoryManager *pMemoryManager, 481 NvU32 flags 482 ) 483 { 484 // If fbOverrideSizeMb is set, finish setting up the FB parameters now that state init has finished 485 memmgrFinishHandleSizeOverrides_HAL(pGpu, pMemoryManager); 486 487 if ((flags & GPU_STATE_FLAGS_PRESERVING) && 488 !(flags & GPU_STATE_FLAGS_GC6_TRANSITION)) 489 { 490 // 491 // Only do initialization scrubs (i.e. RM reserved region) on 492 // non-GC6 transitions since GC6 cycles leave FB powered. 493 // 494 memmgrScrubInit_HAL(pGpu, pMemoryManager); 495 } 496 497 // Dump FB regions 498 memmgrDumpFbRegions(pGpu, pMemoryManager); 499 500 return NV_OK; 501 } 502 503 NV_STATUS 504 memmgrStatePreUnload_IMPL 505 ( 506 OBJGPU *pGpu, 507 MemoryManager *pMemoryManager, 508 NvU32 flags 509 ) 510 { 511 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 512 513 NV_ASSERT((flags & GPU_STATE_FLAGS_PRESERVING) || pMemoryManager->zbcSurfaces == 0); 514 515 if (memmgrIsPmaEnabled(pMemoryManager) && 516 memmgrIsPmaSupportedOnPlatform(pMemoryManager) && 517 osNumaOnliningEnabled(pGpu->pOsGpuInfo) && 518 pKernelMemorySystem->memPartitionNumaInfo[0].bInUse) 519 { 520 pmaNumaOfflined(&pMemoryManager->pHeap->pmaObject); 521 } 522 523 return NV_OK; 524 } 525 526 NV_STATUS 527 memmgrStateUnload_IMPL 528 ( 529 OBJGPU *pGpu, 530 MemoryManager *pMemoryManager, 531 NvU32 flags 532 ) 533 { 534 if ((flags & GPU_STATE_FLAGS_PRESERVING) && 535 !(flags & GPU_STATE_FLAGS_GC6_TRANSITION)) 536 { 537 // 538 // Initialiation scrubs only happen during StateLoad on non-GC6 539 // transitions. 540 // 541 memmgrScrubDestroy_HAL(pGpu, pMemoryManager); 542 } 543 544 return NV_OK; 545 } 546 547 void 548 memmgrStateDestroy_IMPL 549 ( 550 OBJGPU *pGpu, 551 MemoryManager *pMemoryManager 552 ) 553 { 554 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 555 Heap *pHeap = MEMORY_MANAGER_GET_HEAP(pMemoryManager); 556 NvU32 i; 557 558 _memmgrFreeInternalClientObjects(pMemoryManager); 559 560 // Destroys the SW state of the page level pools 561 memmgrPageLevelPoolsDestroy(pGpu, pMemoryManager); 562 563 // Destroy the heap entirely, and all associated structures 564 if (pHeap) 565 { 566 kmemsysPreHeapDestruct_HAL(pGpu, pKernelMemorySystem); 567 568 objDelete(pHeap); 569 pMemoryManager->pHeap = NULL; 570 } 571 572 // RMCONFIG: only if FBSR engine is enabled 573 if (RMCFG_MODULE_FBSR) 574 { 575 // Cleanup fbsrReservedRanges 576 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE] != NULL) 577 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE]); 578 579 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE] != NULL) 580 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE]); 581 582 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP] != NULL) 583 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP]); 584 585 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR] != NULL) 586 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR]); 587 588 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR] != NULL) 589 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR]); 590 591 if (pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE] != NULL) 592 memdescDestroy(pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE]); 593 594 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_BEFORE_BAR2PTE] = NULL; 595 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_AFTER_BAR2PTE] = NULL; 596 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_HEAP] = NULL; 597 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_NON_WPR] = NULL; 598 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_GSP_WPR] = NULL; 599 pMemoryManager->fbsrReservedRanges[FBSR_RESERVED_INST_MEMORY_VGA_WORKSPACE] = NULL; 600 601 for (i = 0; i < NUM_FBSR_TYPES; i++) 602 { 603 fbsrDestroy_HAL(pGpu, pMemoryManager->pFbsr[i]); 604 } 605 } 606 if (memmgrIsLocalEgmEnabled(pMemoryManager)) 607 { 608 if (!IS_VIRTUAL_WITH_SRIOV(pGpu)) 609 { 610 kbusUnreserveP2PPeerIds_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), NVBIT(pMemoryManager->localEgmPeerId)); 611 } 612 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 613 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 614 } 615 616 memmgrScrubDestroy_HAL(pGpu, pMemoryManager); 617 } 618 619 static NV_STATUS 620 _memmgrCreateChildObjects 621 ( 622 MemoryManager *pMemoryManager 623 ) 624 { 625 NV_STATUS status = NV_OK; 626 627 // RMCONFIG: only if FBSR engine is enabled 628 if (RMCFG_MODULE_FBSR) 629 { 630 NvU32 i; 631 632 // Create FBSR object for every type RM supports. 633 for (i = 0; i < NUM_FBSR_TYPES; i++) 634 { 635 status = _memmgrCreateFBSR(pMemoryManager, i); 636 if (status != NV_OK) 637 { 638 return status; 639 } 640 } 641 } 642 643 return status; 644 } 645 646 NV_STATUS 647 memmgrCreateHeap_IMPL 648 ( 649 MemoryManager *pMemoryManager 650 ) 651 { 652 Heap *newHeap; 653 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 654 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 655 NvU64 rsvdSize; 656 NvU64 size; 657 NV_STATUS status = NV_OK; 658 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 659 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 660 661 // If we're using FB regions then rsvd memory is already marked as a reserved region 662 if ((pMemoryManager->Ram.numFBRegions == 0) || (IS_VIRTUAL_WITH_SRIOV(pGpu))) 663 { 664 if (pMemorySystemConfig->bReservedMemAtBottom) 665 { 666 // rsvd memory is already accounted for in heapStart 667 rsvdSize = 0; 668 } 669 else 670 { 671 rsvdSize = pMemoryManager->rsvdMemorySize; 672 } 673 } 674 else 675 rsvdSize = 0; 676 677 // for vGPU, add extra FB tax incurred by host RM to reserved size 678 rsvdSize += memmgrGetFbTaxSize_HAL(pGpu, pMemoryManager); 679 680 // 681 // Fix up region descriptions to match with any FB override size 682 // 683 memmgrHandleSizeOverrides_HAL(pGpu, pMemoryManager); 684 685 // 686 // Calculate the FB heap size as the address space size, then deduct any reserved memory 687 // 688 size = pMemoryManager->Ram.fbAddrSpaceSizeMb << 20; 689 size -= NV_MIN(size, rsvdSize); 690 691 if((size != 0) || (pMemoryManager->bScanoutSysmem)) 692 { 693 status = objCreate(&newHeap, pMemoryManager, Heap); 694 if (status != NV_OK) 695 { 696 return status; 697 } 698 699 pMemoryManager->pHeap = newHeap; 700 701 if (memmgrIsPmaEnabled(pMemoryManager) && 702 memmgrIsPmaSupportedOnPlatform(pMemoryManager)) 703 { 704 portMemSet(&pMemoryManager->pHeap->pmaObject, 0, sizeof(pMemoryManager->pHeap->pmaObject)); 705 status = memmgrPmaInitialize(pGpu, pMemoryManager, &pMemoryManager->pHeap->pmaObject); 706 NV_ASSERT_OR_RETURN(status == NV_OK, status); 707 } 708 709 status = heapInit(pGpu, newHeap, 710 pMemoryManager->heapStartOffset, 711 size - pMemoryManager->heapStartOffset, HEAP_TYPE_RM_GLOBAL, GPU_GFID_PF, NULL); 712 NV_ASSERT_OR_RETURN(NV_OK == status, status); 713 714 if ((memmgrIsPmaInitialized(pMemoryManager)) && (pMemoryManager->pHeap->bHasFbRegions)) 715 { 716 status = memmgrPmaRegisterRegions(pGpu, pMemoryManager, pMemoryManager->pHeap, 717 &pMemoryManager->pHeap->pmaObject); 718 NV_ASSERT_OR_RETURN(status == NV_OK, status); 719 } 720 721 // Reserve vidmem for FSP usage, including FRTS, WPR2 722 status = memmgrReserveMemoryForFsp(pGpu, pMemoryManager); 723 if (status != NV_OK) 724 { 725 NV_PRINTF(LEVEL_ERROR, "Failed to reserve vidmem for WPR and FRTS.\n"); 726 return status; 727 } 728 729 if (!IsSLIEnabled(pGpu)) 730 { 731 // Do the actual blacklisting of pages from the heap 732 if (newHeap->blackListAddresses.count != 0) 733 { 734 status = heapBlackListPages(pGpu, newHeap); 735 736 if (status != NV_OK) 737 { 738 // Warn and continue 739 NV_PRINTF(LEVEL_WARNING, "Error 0x%x creating blacklist\n", 740 status); 741 } 742 } 743 } 744 745 kmemsysPostHeapCreate_HAL(pGpu, pKernelMemorySystem); 746 } 747 748 return status; 749 } 750 751 /* 752 * @brief Gets per-device suballocator. If it is not available, get shared heap. 753 * 754 * @param[in] pMemoryManager MemoryManager pointer 755 */ 756 Heap * 757 memmgrGetDeviceSuballocator_IMPL 758 ( 759 MemoryManager *pMemoryManager, 760 NvBool bForceSubheap 761 ) 762 { 763 764 if (!bForceSubheap) 765 { 766 // If no suballocator found, use heap 767 return MEMORY_MANAGER_GET_HEAP(pMemoryManager); 768 } 769 770 return NULL; 771 } 772 773 static NV_STATUS 774 _memmgrCreateFBSR 775 ( 776 MemoryManager *pMemoryManager, 777 NvU32 type 778 ) 779 { 780 OBJFBSR *pFbsr; 781 NV_STATUS status; 782 783 status = objCreate(&pFbsr, pMemoryManager, OBJFBSR); 784 if (status != NV_OK) 785 { 786 return status; 787 } 788 789 NV_ASSERT(pFbsr); 790 pMemoryManager->pFbsr[type] = pFbsr; 791 792 fbsrObjectInit(pFbsr, type); 793 794 return NV_OK; 795 } 796 797 static void 798 _memmgrFreeInternalClientObjects 799 ( 800 MemoryManager *pMemoryManager 801 ) 802 { 803 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 804 805 if (pMemoryManager->hThirdPartyP2P != 0) 806 { 807 pRmApi->Free(pRmApi, pMemoryManager->hClient, 808 pMemoryManager->hThirdPartyP2P); 809 pMemoryManager->hThirdPartyP2P = 0; 810 } 811 812 if (pMemoryManager->hClient != 0) 813 { 814 rmapiutilFreeClientAndDeviceHandles(pRmApi, 815 &pMemoryManager->hClient, 816 &pMemoryManager->hDevice, 817 &pMemoryManager->hSubdevice); 818 } 819 } 820 821 static NV_STATUS 822 _memmgrAllocInternalClientObjects 823 ( 824 OBJGPU *pGpu, 825 MemoryManager *pMemoryManager 826 ) 827 { 828 NV_STATUS status; 829 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 830 831 status = rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, 832 &pMemoryManager->hClient, 833 &pMemoryManager->hDevice, 834 &pMemoryManager->hSubdevice); 835 if (status != NV_OK) 836 { 837 goto failed; 838 } 839 840 { 841 NV503C_ALLOC_PARAMETERS params; 842 NvHandle hThirdPartyP2P = 0; 843 844 NV_ASSERT_OK_OR_GOTO(status, serverutilGenResourceHandle(pMemoryManager->hClient, 845 &hThirdPartyP2P), 846 failed); 847 848 portMemSet(¶ms, 0, sizeof(params)); 849 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING)) 850 { 851 params.flags = NV503C_ALLOC_PARAMETERS_FLAGS_TYPE_NVLINK; 852 } 853 else 854 { 855 params.flags = NV503C_ALLOC_PARAMETERS_FLAGS_TYPE_BAR1; 856 } 857 status = pRmApi->AllocWithHandle(pRmApi, 858 pMemoryManager->hClient, 859 pMemoryManager->hSubdevice, 860 hThirdPartyP2P, 861 NV50_THIRD_PARTY_P2P, 862 ¶ms, 863 sizeof(params)); 864 if (status != NV_OK) 865 { 866 NV_PRINTF(LEVEL_WARNING, "Error creating internal ThirdPartyP2P object: %x\n", 867 status); 868 pMemoryManager->hThirdPartyP2P = 0; 869 } 870 else 871 { 872 pMemoryManager->hThirdPartyP2P = hThirdPartyP2P; 873 } 874 875 } 876 877 return NV_OK; 878 879 failed: 880 _memmgrFreeInternalClientObjects(pMemoryManager); 881 882 return status; 883 } 884 885 /*! 886 * @brief Determine size of FB RAM which is used for RM internal allocations 887 * and PMA. 888 * 889 * @param[out] pFbUsedSize FB used memory size 890 * 891 * @returns NV_OK 892 */ 893 NV_STATUS 894 memmgrGetUsedRamSize_IMPL 895 ( 896 OBJGPU *pGpu, 897 MemoryManager *pMemoryManager, 898 NvU64 *pFbUsedSize 899 ) 900 { 901 Heap *pHeap = GPU_GET_HEAP(pGpu); 902 NvU64 heapFreeSpace, heapTotalSpace, pmaFreeSpace; 903 904 // 905 // Determine free memory in FB and substract with total FB memory. 906 // If PMA is initialized, then use the free memory size in PMA and 907 // heap otherwise only use heap free memory for calculation. 908 // 909 heapGetFree(pHeap, &heapFreeSpace); 910 heapGetSize(pHeap, &heapTotalSpace); 911 if (memmgrIsPmaInitialized(pMemoryManager)) 912 { 913 pmaGetFreeMemory(&pHeap->pmaObject, &pmaFreeSpace); 914 *pFbUsedSize = heapTotalSpace - heapFreeSpace - pmaFreeSpace; 915 } 916 else 917 { 918 *pFbUsedSize = heapTotalSpace - heapFreeSpace; 919 } 920 921 // 922 // GSP's WPR region has its own save/restore mechanism and does not need 923 // to be accounted for in total FB size used - which is needed to find out 924 // how much SYSMEM needs to be allocated to save all FB memory 925 // 926 if (IS_GSP_CLIENT(pGpu)) 927 { 928 KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu); 929 NvU64 gspWprRegionSize = pKernelGsp->pWprMeta->gspFwWprEnd - pKernelGsp->pWprMeta->gspFwWprStart; 930 931 *pFbUsedSize = *pFbUsedSize - gspWprRegionSize; 932 } 933 934 return NV_OK; 935 } 936 937 NV_STATUS 938 memmgrAllocHwResources_IMPL 939 ( 940 OBJGPU *pGpu, 941 MemoryManager *pMemoryManager, 942 FB_ALLOC_INFO *pFbAllocInfo 943 ) 944 { 945 MemoryManager *pMemoryManagerLoop; 946 FB_ALLOC_INFO *pTempInfo = NULL; 947 NvU32 skipFlag = (pFbAllocInfo->pageFormat->flags & NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC); 948 NV_STATUS rmStatus = NV_OK; 949 950 pTempInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 951 if (pTempInfo == NULL) 952 { 953 NV_ASSERT(0); 954 return NV_ERR_NO_MEMORY; 955 } 956 957 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 958 { 959 NV_STATUS tempStatus; 960 *pTempInfo = *pFbAllocInfo; // struct copy 961 962 pMemoryManagerLoop = GPU_GET_MEMORY_MANAGER(pGpu); 963 964 tempStatus = memmgrAllocHal_HAL(pGpu, pMemoryManagerLoop, pTempInfo); 965 // be sure to return an intermediate error 966 if (NV_OK == rmStatus) 967 rmStatus = tempStatus; 968 } 969 SLI_LOOP_END 970 971 *pFbAllocInfo = *pTempInfo; // struct copy 972 portMemFree(pTempInfo); 973 974 pFbAllocInfo->pageFormat->flags &= ~NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC; 975 pFbAllocInfo->pageFormat->flags |= skipFlag; 976 977 return rmStatus; 978 } 979 980 NV_STATUS 981 memmgrFreeHwResources_IMPL 982 ( 983 OBJGPU *pGpu, 984 MemoryManager *pMemoryManager, 985 FB_ALLOC_INFO *pFbAllocInfo 986 ) 987 { 988 MemoryManager *pMemoryManagerLoop; 989 NV_STATUS rmStatus = NV_OK; 990 RMTIMEOUT timeout; 991 FB_ALLOC_INFO *pTempInfo = NULL; 992 993 pTempInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 994 if (pTempInfo == NULL) 995 { 996 NV_ASSERT(0); 997 return NV_ERR_NO_MEMORY; 998 } 999 1000 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0); 1001 1002 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1003 { 1004 NV_STATUS tempStatus; 1005 pMemoryManagerLoop = GPU_GET_MEMORY_MANAGER(pGpu); 1006 1007 *pTempInfo = *pFbAllocInfo; 1008 1009 tempStatus = memmgrFreeHal_HAL(pGpu, pMemoryManagerLoop, pTempInfo, &timeout); 1010 // be sure to return an intermediate error 1011 if (NV_OK == rmStatus) 1012 rmStatus = tempStatus; 1013 1014 } 1015 SLI_LOOP_END 1016 1017 *pFbAllocInfo = *pTempInfo; 1018 portMemFree(pTempInfo); 1019 1020 return rmStatus; 1021 } 1022 1023 NvBool 1024 memmgrLargePageSupported_IMPL 1025 ( 1026 MemoryManager *pMemoryManager, 1027 NV_ADDRESS_SPACE addrSpace 1028 ) 1029 { 1030 NvBool isSupported = NV_FALSE; 1031 1032 if (addrSpace == ADDR_FBMEM || addrSpace == ADDR_VIRTUAL) 1033 { 1034 isSupported = NV_TRUE; 1035 } 1036 else if (addrSpace == ADDR_SYSMEM) 1037 { 1038 isSupported = (pMemoryManager->sysmemPageSize != RM_PAGE_SIZE); 1039 } 1040 else 1041 { 1042 NV_ASSERT(0); 1043 } 1044 1045 return isSupported; 1046 } 1047 1048 NvBool 1049 memmgrComprSupported_IMPL 1050 ( 1051 MemoryManager *pMemoryManager, 1052 NV_ADDRESS_SPACE addrSpace 1053 ) 1054 { 1055 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 1056 NvBool isSupported = NV_FALSE; 1057 1058 if (GPU_GET_KERNEL_GMMU(pGpu) != NULL) 1059 { 1060 if (memmgrLargePageSupported(pMemoryManager, addrSpace) || 1061 pMemoryManager->bSmallPageCompression) 1062 { 1063 if (addrSpace == ADDR_FBMEM || addrSpace == ADDR_VIRTUAL) 1064 { 1065 isSupported = NV_TRUE; 1066 } 1067 else if (addrSpace == ADDR_SYSMEM) 1068 { 1069 // Compression is allowed on vidmem or unified aperture (vidmem/sysmem is same w.r.t HW) 1070 isSupported = (gpuIsUnifiedMemorySpaceEnabled(pGpu) && 1071 pMemoryManager->bSysmemCompressionSupportDef); 1072 NV_PRINTF(LEVEL_ERROR, "isSupported=%s\n", 1073 isSupported ? "NV_TRUE" : "NV_FALSE"); 1074 } 1075 else 1076 { 1077 NV_ASSERT(0); 1078 } 1079 } 1080 } 1081 1082 return isSupported; 1083 } 1084 1085 NV_ADDRESS_SPACE 1086 memmgrAllocGetAddrSpace_IMPL 1087 ( 1088 MemoryManager *pMemoryManager, 1089 NvU32 flags, 1090 NvU32 attr 1091 ) 1092 { 1093 NV_ADDRESS_SPACE addrSpace = ADDR_UNKNOWN; 1094 1095 if (flags & NVOS32_ALLOC_FLAGS_VIRTUAL) 1096 { 1097 addrSpace = ADDR_VIRTUAL; 1098 } 1099 else if (FLD_TEST_DRF(OS32, _ATTR, _LOCATION, _VIDMEM, attr)) 1100 { 1101 addrSpace = ADDR_FBMEM; 1102 } 1103 else 1104 { 1105 // In case location is SYSMEM or ANY, allocate in vidmem if protected flag is set. 1106 if (flags & NVOS32_ALLOC_FLAGS_PROTECTED) 1107 { 1108 addrSpace = ADDR_FBMEM; 1109 } 1110 else 1111 { 1112 addrSpace = ADDR_SYSMEM; 1113 } 1114 } 1115 1116 return addrSpace; 1117 } 1118 1119 NvU32 1120 memmgrGetMappableRamSizeMb_IMPL(MemoryManager *pMemoryManager) 1121 { 1122 return NvU64_LO32(pMemoryManager->Ram.mapRamSizeMb); 1123 } 1124 // 1125 // ZBC clear create/destroy routines. 1126 // 1127 1128 NV_STATUS 1129 memmgrFillMemdescForPhysAttr_IMPL 1130 ( 1131 OBJGPU *pGpu, 1132 MemoryManager *pMemoryManager, 1133 PMEMORY_DESCRIPTOR pMemDesc, 1134 ADDRESS_TRANSLATION addressTranslation, 1135 NvU64 *pOffset, 1136 NvU32 *pMemAperture, 1137 NvU32 *pMemKind, 1138 NvU32 *pZCullId, 1139 NvU32 *pGpuCacheAttr, 1140 NvU32 *pGpuP2PCacheAttr, 1141 NvU64 *contigSegmentSize 1142 ) 1143 { 1144 NvU64 surfOffset = *pOffset, surfBase, surfLimit; 1145 NvU32 zcbitmap; 1146 1147 surfBase = memdescGetPhysAddr(pMemDesc, addressTranslation, 0); 1148 surfLimit = surfBase + pMemDesc->Size - 1; 1149 *pMemKind = memdescGetPteKind(pMemDesc); 1150 1151 *pOffset = memdescGetPhysAddr(pMemDesc, addressTranslation, surfOffset); 1152 1153 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM ) 1154 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_VIDMEM; 1155 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM) 1156 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM; 1157 else if (memdescGetAddressSpace(pMemDesc) == ADDR_EGM) 1158 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM; 1159 else if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL ) 1160 { 1161 // 1162 // XXX we could theoretically find whatever phys mem object is plugged 1163 // in at surfOffset w/in the virt object... that'd mean scanning 1164 // pMemory->DmaMappingList 1165 // 1166 return NV_ERR_NOT_SUPPORTED; 1167 } 1168 else 1169 return NV_ERR_GENERIC; 1170 1171 if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1172 { 1173 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED; 1174 } 1175 else if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1176 { 1177 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED; 1178 } 1179 else 1180 { 1181 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN; 1182 } 1183 1184 if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_CACHED) 1185 { 1186 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED; 1187 } 1188 else if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) 1189 { 1190 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED; 1191 } 1192 else 1193 { 1194 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN; 1195 } 1196 1197 zcbitmap = FB_HWRESID_ZCULL_VAL_FERMI(memdescGetHwResId(pMemDesc)); //bitmap form... need a scalar 1198 for ( *pZCullId = 0; zcbitmap; zcbitmap >>= 1, *pZCullId += 1) {;;;} 1199 *pZCullId -= 1; // side effect if there is no zcull id of setting ~0 1200 1201 *contigSegmentSize = surfLimit - (surfBase + surfOffset) + 1; 1202 1203 if ( !memdescGetContiguity(pMemDesc, addressTranslation)) 1204 { 1205 // XXX overly conservative. we could scan the PTEs to find out if more pages are contig. 1206 NvU64 surfOffsetLimitSame4KBPage = (4*1024)*((surfBase + surfOffset)/(4*1024)) + (4*1024) - 1; 1207 if ( surfLimit >= surfOffsetLimitSame4KBPage ) 1208 *contigSegmentSize = surfOffsetLimitSame4KBPage - (surfBase + surfOffset) + 1; 1209 } 1210 1211 return NV_OK; 1212 } 1213 1214 NvU64 1215 memmgrDeterminePageSize_IMPL 1216 ( 1217 MemoryManager *pMemoryManager, 1218 NvHandle hClient, 1219 NvU64 memSize, 1220 NvU32 memFormat, 1221 NvU32 pageFormatFlags, 1222 NvU32 *pRetAttr, 1223 NvU32 *pRetAttr2 1224 ) 1225 { 1226 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 1227 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 1228 NV_ADDRESS_SPACE addrSpace; 1229 NvBool bIsBigPageSupported; 1230 RM_ATTR_PAGE_SIZE pageSizeAttr; 1231 NvU64 pageSize = 0; 1232 1233 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY) || (pKernelGmmu == NULL)) 1234 { 1235 pageSize = RM_PAGE_SIZE; 1236 } 1237 // Sanity check the arguments. 1238 else if (pRetAttr == NULL || pRetAttr2 == NULL) 1239 { 1240 NV_ASSERT_OR_RETURN(0, 0); 1241 } 1242 else 1243 { 1244 addrSpace = memmgrAllocGetAddrSpace(pMemoryManager, pageFormatFlags, *pRetAttr); 1245 1246 bIsBigPageSupported = memmgrLargePageSupported(pMemoryManager, addrSpace); 1247 pageSizeAttr = dmaNvos32ToPageSizeAttr(*pRetAttr, *pRetAttr2); 1248 1249 // 1250 // Precedence in page size selection 1251 // 1. CACHE_ONLY mode -> SMALL 1252 // 2. !BigPageSupport (Sysmem && GpuSmmuOff ) -> SMALL 1253 // 3. Client page size override -> Use override 1254 // 4. HugePageSupported && size >= HugePageSize -> HUGE 1255 // 5. Block-linear || size >= minSizeForBigPage || hClient || GpuSmmuOn -> BIG 1256 // 6. none of the above -> SMALL 1257 // 1258 // On Tegra, we don't have a carveout/FB in production. So, we're 1259 // not guaranteed to get BIG page sized or contiguous allocations 1260 // from OS. But we need BIG page sized allocations for efficient Big GPU 1261 // operation. We use the SMMU unit within the Tegra Memory Contoller (MC), 1262 // to construct BIG pages from the 4KB small page allocations from OS. 1263 // SMMU will linearize the discontiguous 4KB allocations into what will 1264 // appear to the GPU as a large contiguous physical allocation. 1265 // 1266 // RM will eventually decide whether a SYSMEM allocation needs BIG page 1267 // via GPU SMMU mapping. Right now, we give an option for RM clients to 1268 // force it, via the SMMU_ON_GPU attribute. 1269 // 1270 if (gpuIsCacheOnlyModeEnabled(pGpu)) 1271 { 1272 pageSize = RM_PAGE_SIZE; 1273 } 1274 else if (!bIsBigPageSupported) 1275 { 1276 if (RM_ATTR_PAGE_SIZE_BIG == pageSizeAttr || 1277 RM_ATTR_PAGE_SIZE_HUGE == pageSizeAttr || 1278 RM_ATTR_PAGE_SIZE_512MB == pageSizeAttr) 1279 { 1280 NV_PRINTF(LEVEL_ERROR, 1281 "Big/Huge/512MB page size not supported in sysmem.\n"); 1282 1283 NV_ASSERT_OR_RETURN(0, 0); 1284 } 1285 else 1286 { 1287 pageSize = RM_PAGE_SIZE; 1288 } 1289 } 1290 else 1291 { 1292 switch (pageSizeAttr) 1293 { 1294 case RM_ATTR_PAGE_SIZE_INVALID: 1295 NV_PRINTF(LEVEL_ERROR, "invalid page size attr\n"); 1296 NV_ASSERT_OR_RETURN(0, 0); 1297 1298 case RM_ATTR_PAGE_SIZE_DEFAULT: 1299 { 1300 NvBool bUseDefaultHugePagesize = NV_TRUE; 1301 // WDDMV2 Windows it expect default page size to be 4K /64KB /128KB 1302 if (bUseDefaultHugePagesize && 1303 kgmmuIsHugePageSupported(pKernelGmmu) && 1304 (memSize >= RM_PAGE_SIZE_HUGE) && (addrSpace != ADDR_SYSMEM || 1305 pMemoryManager->sysmemPageSize == RM_PAGE_SIZE_HUGE)) 1306 { 1307 pageSize = RM_PAGE_SIZE_HUGE; 1308 break; 1309 } 1310 else if ((memFormat != NVOS32_ATTR_FORMAT_PITCH) || 1311 (memSize >= kgmmuGetMinBigPageSize(pKernelGmmu)) || hClient || 1312 FLD_TEST_DRF(OS32, _ATTR2, _SMMU_ON_GPU, _ENABLE, *pRetAttr2)) 1313 { 1314 pageSize = kgmmuGetMaxBigPageSize_HAL(pKernelGmmu); 1315 break; 1316 } 1317 1318 pageSize = RM_PAGE_SIZE; 1319 break; 1320 } 1321 1322 case RM_ATTR_PAGE_SIZE_4KB: 1323 pageSize = RM_PAGE_SIZE; 1324 break; 1325 1326 case RM_ATTR_PAGE_SIZE_BIG: 1327 pageSize = kgmmuGetMaxBigPageSize_HAL(pKernelGmmu); 1328 break; 1329 1330 case RM_ATTR_PAGE_SIZE_HUGE: 1331 if (kgmmuIsHugePageSupported(pKernelGmmu)) 1332 { 1333 pageSize = RM_PAGE_SIZE_HUGE; 1334 } 1335 else 1336 { 1337 NV_ASSERT_OR_RETURN(0, 0); 1338 } 1339 break; 1340 1341 case RM_ATTR_PAGE_SIZE_512MB: 1342 if (kgmmuIsPageSize512mbSupported(pKernelGmmu)) 1343 { 1344 pageSize = RM_PAGE_SIZE_512M; 1345 } 1346 else 1347 { 1348 NV_ASSERT_OR_RETURN(0, 0); 1349 } 1350 break; 1351 1352 default: 1353 NV_ASSERT(0); 1354 } 1355 } 1356 } 1357 1358 switch (pageSize) 1359 { 1360 case RM_PAGE_SIZE: 1361 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _4KB, *pRetAttr); 1362 break; 1363 1364 case RM_PAGE_SIZE_64K: 1365 case RM_PAGE_SIZE_128K: 1366 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _BIG, *pRetAttr); 1367 break; 1368 1369 case RM_PAGE_SIZE_HUGE: 1370 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _HUGE, *pRetAttr); 1371 *pRetAttr2 = FLD_SET_DRF(OS32, _ATTR2, _PAGE_SIZE_HUGE, _2MB, *pRetAttr2); 1372 break; 1373 1374 case RM_PAGE_SIZE_512M: 1375 *pRetAttr = FLD_SET_DRF(OS32, _ATTR, _PAGE_SIZE, _HUGE, *pRetAttr); 1376 *pRetAttr2 = FLD_SET_DRF(OS32, _ATTR2, _PAGE_SIZE_HUGE, _512MB, *pRetAttr2); 1377 break; 1378 1379 default: 1380 NV_ASSERT(0); 1381 } 1382 1383 return pageSize; 1384 } 1385 1386 /*! 1387 * Identify if platform's current configuration supports PMA 1388 */ 1389 NV_STATUS 1390 memmgrSetPlatformPmaSupport_IMPL 1391 ( 1392 OBJGPU *pGpu, 1393 MemoryManager *pMemoryManager 1394 ) 1395 { 1396 // 1397 // KMD in WDDM mode will not support pma managed client page tables as 1398 // in both cases client / OS manges it. 1399 // 1400 if (RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM && !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_TCC_MODE)) 1401 { 1402 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 1403 } 1404 1405 // 1406 // FB management should use PMA on Unix/Linux/Mods/Windows 1407 // 1408 if (RMCFG_FEATURE_PLATFORM_UNIX 1409 || RMCFG_FEATURE_PLATFORM_MODS 1410 || RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM) 1411 { 1412 pMemoryManager->bPmaSupportedOnPlatform = NV_TRUE; 1413 } 1414 1415 // 1416 // PMA memory management is not currently supported in non SRIOV VGPU environment. 1417 // The RPC mechanism needs to be expanded to distinguish allocation types. 1418 // Bug #1735412 1419 // 1420 // TODO : Remove these constraints. 1421 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1422 { 1423 pMemoryManager->bPmaSupportedOnPlatform = NV_FALSE; 1424 } 1425 1426 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_VIRTUALIZATION_MODE_HOST_VGPU)) 1427 { 1428 if (pMemoryManager->bVgpuPmaSupport) 1429 { 1430 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 1431 } 1432 else 1433 { 1434 pMemoryManager->bPmaSupportedOnPlatform = NV_FALSE; 1435 } 1436 } 1437 return (NV_OK); 1438 } 1439 1440 /*! 1441 * Allocate console region in CPU-RM based on region table passed from Physical RM 1442 */ 1443 NV_STATUS 1444 memmgrAllocateConsoleRegion_IMPL 1445 ( 1446 OBJGPU *pGpu, 1447 MemoryManager *pMemoryManager, 1448 FB_REGION_DESCRIPTOR *pConsoleFbRegion 1449 ) 1450 { 1451 1452 NV_STATUS status = NV_OK; 1453 NvU32 consoleRegionId = 0x0; 1454 NvU64 regionSize; 1455 1456 if (pMemoryManager->Ram.ReservedConsoleDispMemSize > 0) 1457 { 1458 pConsoleFbRegion->base = pMemoryManager->Ram.fbRegion[consoleRegionId].base; 1459 pConsoleFbRegion->limit = pMemoryManager->Ram.fbRegion[consoleRegionId].limit; 1460 1461 regionSize = pConsoleFbRegion->limit - pConsoleFbRegion->base + 1; 1462 1463 // Once the console is reserved, we don't expect to reserve it again 1464 NV_ASSERT_OR_RETURN(pMemoryManager->Ram.pReservedConsoleMemDesc == NULL, 1465 NV_ERR_STATE_IN_USE); 1466 1467 status = memdescCreate(&pMemoryManager->Ram.pReservedConsoleMemDesc, pGpu, 1468 regionSize, RM_PAGE_SIZE_64K, NV_TRUE, ADDR_FBMEM, 1469 NV_MEMORY_UNCACHED, 1470 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 1471 if (status != NV_OK) 1472 { 1473 pConsoleFbRegion->base = pConsoleFbRegion->limit = 0; 1474 return status; 1475 } 1476 1477 memdescDescribe(pMemoryManager->Ram.pReservedConsoleMemDesc, ADDR_FBMEM, 1478 pConsoleFbRegion->base, regionSize); 1479 memdescSetPageSize(pMemoryManager->Ram.pReservedConsoleMemDesc, 1480 AT_GPU, RM_PAGE_SIZE); 1481 1482 1483 NV_PRINTF(LEVEL_INFO, "Allocating console region of size: %llx, at base : %llx \n ", 1484 regionSize, pConsoleFbRegion->base); 1485 } 1486 1487 return status; 1488 } 1489 1490 void 1491 memmgrReleaseConsoleRegion_IMPL 1492 ( 1493 OBJGPU *pGpu, 1494 MemoryManager *pMemoryManager 1495 ) 1496 { 1497 memdescDestroy(pMemoryManager->Ram.pReservedConsoleMemDesc); 1498 pMemoryManager->Ram.pReservedConsoleMemDesc = NULL; 1499 } 1500 1501 PMEMORY_DESCRIPTOR 1502 memmgrGetReservedConsoleMemDesc_IMPL 1503 ( 1504 OBJGPU *pGpu, 1505 MemoryManager *pMemoryManager 1506 ) 1507 { 1508 return pMemoryManager->Ram.pReservedConsoleMemDesc; 1509 } 1510 1511 /*! 1512 * Reserve FB for allocating BAR2 Page Dirs and Page Tables 1513 */ 1514 void 1515 memmgrReserveBar2BackingStore_IMPL 1516 ( 1517 OBJGPU *pGpu, 1518 MemoryManager *pMemoryManager, 1519 NvU64 *pAddr 1520 ) 1521 { 1522 NvU64 tmpAddr = *pAddr; 1523 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 1524 1525 NvU32 pageDirsSize = kbusGetSizeOfBar2PageDirs_HAL(pGpu, pKernelBus); 1526 NvU32 pageTblsSize = kbusGetSizeOfBar2PageTables_HAL(pGpu, pKernelBus); 1527 1528 // Reserve space for BAR2 Page Dirs 1529 if (pKernelBus->PDEBAR2Aperture == ADDR_FBMEM) 1530 { 1531 tmpAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1532 pKernelBus->bar2[GPU_GFID_PF].pdeBase = tmpAddr; 1533 tmpAddr += pageDirsSize; 1534 } 1535 1536 // Reserve space for BAR2 Page Tables 1537 if (pKernelBus->PTEBAR2Aperture == ADDR_FBMEM) 1538 { 1539 tmpAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1540 pKernelBus->bar2[GPU_GFID_PF].pteBase = tmpAddr; 1541 tmpAddr += pageTblsSize; 1542 } 1543 1544 NV_PRINTF(LEVEL_INFO, "Reserve space for bar2 Page dirs offset = 0x%llx size = 0x%x\n", 1545 pKernelBus->bar2[GPU_GFID_PF].pdeBase, pageDirsSize); 1546 1547 NV_PRINTF(LEVEL_INFO, "Reserve space for bar2 Page tables offset = 0x%llx size = 0x%x\n", 1548 pKernelBus->bar2[GPU_GFID_PF].pteBase, pageTblsSize); 1549 1550 *pAddr = NV_ROUNDUP(tmpAddr, RM_PAGE_SIZE); 1551 } 1552 1553 /*! 1554 * Calculate the Vista reserved memory requirement per FB region for mixed type/density 1555 */ 1556 void 1557 memmgrCalcReservedFbSpace_IMPL 1558 ( 1559 OBJGPU *pGpu, 1560 MemoryManager *pMemoryManager 1561 ) 1562 { 1563 NvU64 rsvdFastSize = 0; 1564 NvU64 rsvdSlowSize = 0; 1565 NvU64 rsvdISOSize = 0; 1566 NvU32 i; 1567 NvU32 idxISORegion = 0; 1568 NvU32 idxFastRegion = 0; 1569 NvU32 idxSlowRegion = 0; 1570 NvBool bAllocProtected = NV_FALSE; 1571 1572 bAllocProtected = gpuIsCCFeatureEnabled(pGpu); 1573 1574 // 1575 // This is a hack solely for Vista (on Vista the OS controls the majority of heap). 1576 // Linux and Mac don't have reserved memory and doesn't use this function. 1577 // 1578 // On Vista, Fermi's instance memory is not reserved by RM anymore. 1579 // KMD has to reserve enough instance memory for driver private data. 1580 // This function does the calculation of needed space. See bug 642233. 1581 // While it returns the result in Mb, the calculation is made with byte 1582 // 1583 1584 // If we have no usable memory then we can't reserve any. 1585 if (!pMemoryManager->Ram.fbUsableMemSize) 1586 return; 1587 1588 // If reserved memory requirements have already been calculated, don't do it again. 1589 if (pMemoryManager->bLddmReservedMemoryCalculated) 1590 return; 1591 1592 memmgrCalcReservedFbSpaceHal_HAL(pGpu, pMemoryManager, &rsvdFastSize, &rsvdSlowSize, &rsvdISOSize); 1593 1594 // If we have regions defined, fill in the per-segment reserved memory requirement 1595 if (pMemoryManager->Ram.numFBRegions > 0) 1596 { 1597 FB_REGION_DESCRIPTOR *pFbRegion = NULL; 1598 NvU64 regionSize = 0; 1599 1600 // 1601 // Find the fastest and ISO regions. This search makes a soft assumption that 1602 // region #0 is not reserved, fastest, and supports ISO -- that would be stupid 1603 // 1604 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 1605 { 1606 pFbRegion = &pMemoryManager->Ram.fbRegion[i]; 1607 regionSize = (pFbRegion->limit - pFbRegion->base +1); 1608 1609 // Check only non-reserved regions (which are typically unpopulated blackholes in address space) 1610 if ((!pFbRegion->bRsvdRegion) && 1611 (bAllocProtected || !pFbRegion->bProtected) && 1612 (regionSize >= (rsvdFastSize + rsvdSlowSize + rsvdISOSize))) 1613 { 1614 // Find the fastest region 1615 if ((pFbRegion->performance > pMemoryManager->Ram.fbRegion[idxFastRegion].performance) 1616 || pMemoryManager->Ram.fbRegion[idxFastRegion].bRsvdRegion 1617 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxFastRegion].bProtected)) 1618 { 1619 idxFastRegion = i; 1620 } 1621 // Find the slowest region 1622 if ((pFbRegion->performance < pMemoryManager->Ram.fbRegion[idxSlowRegion].performance) 1623 || pMemoryManager->Ram.fbRegion[idxSlowRegion].bRsvdRegion 1624 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxSlowRegion].bProtected)) 1625 { 1626 idxSlowRegion = i; 1627 } 1628 // Find the fastest ISO region 1629 if (pFbRegion->bSupportISO) 1630 { 1631 if ((!pMemoryManager->Ram.fbRegion[idxISORegion].bSupportISO) || 1632 (pFbRegion->performance > pMemoryManager->Ram.fbRegion[idxISORegion].performance) 1633 || (!bAllocProtected && pMemoryManager->Ram.fbRegion[idxISORegion].bProtected)) 1634 { 1635 idxISORegion = i; 1636 } 1637 } 1638 } 1639 } 1640 1641 // There should *ALWAYS* be a region that supports ISO, even if we have no display 1642 NV_ASSERT(pMemoryManager->Ram.fbRegion[idxISORegion].bSupportISO); 1643 1644 // There should *ALWAYS* be a non-reserved region that is faster than reserved and supports ISO 1645 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxISORegion].bRsvdRegion); 1646 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxFastRegion].bRsvdRegion); 1647 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxSlowRegion].bRsvdRegion); 1648 1649 // 1650 // Whenever Hopper CC is enabled, it is mandatory to put allocations 1651 // like page tables, CBC and fault buffers in CPR region. Cannot put 1652 // reserved memory in protected region in non CC cases 1653 // 1654 if (!bAllocProtected) 1655 { 1656 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxISORegion].bProtected); 1657 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxFastRegion].bProtected); 1658 NV_ASSERT(!pMemoryManager->Ram.fbRegion[idxSlowRegion].bProtected); 1659 } 1660 1661 // 1662 // Vista expects to be able to VidHeapControl allocate a cursor in ISO 1663 // 1664 // For mixed density reserved memory should be split between "fast" and 1665 // "slow" memory. Fast memory should also support ISO. The policy to 1666 // prefer "slow" vs "fast" memory is platform dependent. 1667 // 1668 pMemoryManager->Ram.fbRegion[idxISORegion].rsvdSize += rsvdISOSize; 1669 pMemoryManager->Ram.fbRegion[idxSlowRegion].rsvdSize += rsvdSlowSize; 1670 pMemoryManager->Ram.fbRegion[idxFastRegion].rsvdSize += rsvdFastSize; 1671 1672 pMemoryManager->bLddmReservedMemoryCalculated = NV_TRUE; 1673 } 1674 } 1675 1676 /*! 1677 * Init channel size 1678 * 1679 * @param[in] pChannel OBJCHANNEL pointer 1680 * @param[in] numCopyBlocks Number of copies that should fit in the push buffer 1681 * 1682 * @returns NV_STATUS 1683 */ 1684 void 1685 memmgrMemUtilsSetupChannelBufferSizes_IMPL 1686 ( 1687 MemoryManager *pMemoryManager, 1688 OBJCHANNEL *pChannel, 1689 NvU32 numCopyBlocks 1690 ) 1691 { 1692 // set channel specific sizes 1693 pChannel->channelPbSize = numCopyBlocks * MEMUTILS_SIZE_PER_BLOCK_INBYTES; 1694 pChannel->channelNotifierSize = MEMUTILS_CHANNEL_NOTIFIER_SIZE; 1695 pChannel->channelNumGpFifioEntries = MEMUTILS_NUM_GPFIFIO_ENTRIES; 1696 pChannel->methodSizePerBlock = MEMUTILS_SIZE_PER_BLOCK_INBYTES; 1697 pChannel->channelSize = pChannel->channelPbSize + MEMUTILS_CHANNEL_GPFIFO_SIZE + MEMUTILS_CHANNEL_SEMAPHORE_SIZE; 1698 pChannel->semaOffset = pChannel->channelPbSize + MEMUTILS_CHANNEL_GPFIFO_SIZE; 1699 pChannel->finishPayloadOffset = pChannel->semaOffset + 4; 1700 } 1701 1702 NV_STATUS memmgrFree_IMPL 1703 ( 1704 OBJGPU *pGpu, 1705 MemoryManager *pMemoryManager, 1706 Heap *pHeap, 1707 NvHandle hClient, 1708 NvHandle hDevice, 1709 NvHandle hVASpace, 1710 NvU32 owner, 1711 MEMORY_DESCRIPTOR *pMemDesc 1712 ) 1713 { 1714 NvU64 offsetAlign; 1715 NV_STATUS status; 1716 NvU32 pmaFreeFlag = 0; 1717 1718 // IRQL TEST: must be running at equivalent of passive-level 1719 IRQL_ASSERT_AND_RETURN(!osIsRaisedIRQL()); 1720 1721 if (pMemDesc == NULL) 1722 return NV_ERR_INVALID_ARGUMENT; 1723 1724 offsetAlign = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 1725 1726 if (owner == NVOS32_BLOCK_TYPE_FREE) 1727 return NV_ERR_INVALID_ARGUMENT; 1728 1729 // Virtual heap allocs are tagged vitual and always own the memdesc 1730 if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL) 1731 { 1732 OBJVASPACE *pVAS = NULL; 1733 RsClient *pClient; 1734 1735 status = serverGetClientUnderLock(&g_resServ, hClient, &pClient); 1736 if (status != NV_OK) 1737 return status; 1738 1739 status = vaspaceGetByHandleOrDeviceDefault(pClient, hDevice, hVASpace, &pVAS); 1740 if (status != NV_OK) 1741 return status; 1742 1743 status = vaspaceFree(pVAS, offsetAlign); 1744 memdescDestroy(pMemDesc); 1745 return status; 1746 } 1747 1748 // Free up the memory allocated by PMA. 1749 if (pMemDesc->pPmaAllocInfo) 1750 { 1751 FB_ALLOC_INFO *pFbAllocInfo = NULL; 1752 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 1753 OBJGPU *pMemdescOwnerGpu = NULL; 1754 1755 // 1756 // A memdesc can be duped under a peer device. In that case, before 1757 // freeing FB make sure the GPU which owns the memdesc is available. 1758 // Otherwise, just assert, destroy the memdesc and return NV_OK to 1759 // make sure rest of the clean up happens correctly as we are on 1760 // destroy path. 1761 // Note this is just a WAR till ressrv bring in cleanup of dup objects 1762 // on GPU tear down. 1763 // RS-TODO: Nuke this check once the cleanup is implemented. 1764 // 1765 if (pGpu != pMemDesc->pGpu) 1766 { 1767 if (!gpumgrIsGpuPointerValid(pMemDesc->pGpu)) 1768 { 1769 // 1770 // This should never happen. GPU tear down should always clear 1771 // the duped memory list after resource server implements it. 1772 // For now just assert! 1773 // 1774 NV_ASSERT(0); 1775 memdescDestroy(pMemDesc); 1776 goto pma_free_exit; 1777 } 1778 } 1779 1780 pMemdescOwnerGpu = pMemDesc->pGpu; 1781 1782 // 1783 // Similar to the above WAR, if portMem alocations fail for any reason, 1784 // just assert and return NV_OK to ensure that the rest of the clean up 1785 // happens correctly. 1786 // 1787 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 1788 if (pFbAllocInfo == NULL) 1789 { 1790 NV_ASSERT(0); 1791 goto pma_free_exit; 1792 } 1793 1794 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 1795 if (pFbAllocPageFormat == NULL) { 1796 NV_ASSERT(0); 1797 goto pma_free_exit; 1798 } 1799 1800 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 1801 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 1802 pFbAllocInfo->hClient = hClient; 1803 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 1804 1805 // 1806 // Do not release any HW resources associated with this allocation 1807 // until the last reference to the allocation is freed. Passing 1808 // hwresid = 0 and format = pitch to memmgrFreeHwResources will ensure 1809 // that no comptags/zcull/zbc resources are freed. 1810 // 1811 if (pMemDesc->RefCount == 1) 1812 { 1813 pFbAllocInfo->hwResId = memdescGetHwResId(pMemDesc); 1814 pFbAllocInfo->format = memdescGetPteKind(pMemDesc); 1815 } 1816 else 1817 { 1818 pFbAllocInfo->hwResId = 0; 1819 pFbAllocInfo->format = 0; 1820 } 1821 pFbAllocInfo->offset = offsetAlign; 1822 pFbAllocInfo->size = pMemDesc->Size; 1823 1824 // Free any HW resources allocated. 1825 memmgrFreeHwResources(pMemdescOwnerGpu, 1826 GPU_GET_MEMORY_MANAGER(pMemdescOwnerGpu), pFbAllocInfo); 1827 1828 if (pMemDesc->pPmaAllocInfo != NULL) 1829 { 1830 // Disabling scrub on free for non compressible surfaces 1831 if (RMCFG_FEATURE_PLATFORM_MODS && 1832 !memmgrIsKind_HAL(GPU_GET_MEMORY_MANAGER(pMemdescOwnerGpu), 1833 FB_IS_KIND_COMPRESSIBLE, 1834 memdescGetPteKind(pMemDesc))) 1835 { 1836 pmaFreeFlag = PMA_FREE_SKIP_SCRUB; 1837 } 1838 1839 vidmemPmaFree(pMemdescOwnerGpu, pHeap, pMemDesc->pPmaAllocInfo, pmaFreeFlag); 1840 NV_PRINTF(LEVEL_INFO, "Freeing PMA allocation\n"); 1841 } 1842 1843 pma_free_exit: 1844 portMemFree(pFbAllocInfo); 1845 portMemFree(pFbAllocPageFormat); 1846 memdescDestroy(pMemDesc); 1847 1848 return NV_OK; 1849 } 1850 1851 return heapFree(pGpu, pHeap, owner, pMemDesc); 1852 } 1853 1854 NV_STATUS 1855 memmgrSetPartitionableMem_IMPL 1856 ( 1857 OBJGPU *pGpu, 1858 MemoryManager *pMemoryManager 1859 ) 1860 { 1861 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1862 NV2080_CTRL_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM_PARAMS params = {0}; 1863 Heap *pHeap = GPU_GET_HEAP(pGpu); 1864 NvU64 bottomRsvdSize = 0; 1865 NvU64 topRsvdSize = 0; 1866 NvU32 bottomRegionIdx = 0xFFFF; 1867 NvU32 topRegionIdx = 0xFFFF; 1868 NvU32 i; 1869 NvU64 size; 1870 NvU64 base; 1871 NvU64 offset; 1872 NvU64 freeMem; 1873 1874 // 1875 // Find out the first and the last region for which internal heap or 1876 // bRsvdRegion is true. In Ampere we should never have more than two 1877 // discontigous RM reserved region 1878 // To-Do - Bug 2301972 - Make sure that reserved memory is aligned to VMMU 1879 // segments 1880 // 1881 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 1882 { 1883 if (pMemoryManager->Ram.fbRegion[i].bInternalHeap || 1884 pMemoryManager->Ram.fbRegion[i].bRsvdRegion) 1885 { 1886 NvU64 rsvdSize = (pMemoryManager->Ram.fbRegion[i].limit - 1887 pMemoryManager->Ram.fbRegion[i].base + 1); 1888 1889 // Check if this is bottom reserved region 1890 if (pMemoryManager->Ram.fbRegion[i].base == 0) 1891 { 1892 bottomRegionIdx = i; 1893 bottomRsvdSize += rsvdSize; 1894 } 1895 else if (i > 0 && (pMemoryManager->Ram.fbRegion[i-1].bInternalHeap || 1896 pMemoryManager->Ram.fbRegion[i-1].bRsvdRegion) && 1897 (pMemoryManager->Ram.fbRegion[i].base == pMemoryManager->Ram.fbRegion[i - 1].limit + 1)) 1898 { 1899 // See if this is the contigous region with previous discovery 1900 if (bottomRegionIdx == (i - 1)) 1901 { 1902 // Contigous bottom region 1903 bottomRsvdSize += rsvdSize; 1904 } 1905 else 1906 { 1907 // Contigous top region 1908 topRsvdSize += rsvdSize; 1909 } 1910 } 1911 else 1912 { 1913 // 1914 // Make sure we don't have discontigous reserved regions as 1915 // they are not supported by HW also and we need to support 1916 // these by using blacklisting mechanism. 1917 // 1918 if (topRegionIdx != 0xFFFF) 1919 { 1920 NV_PRINTF(LEVEL_ERROR, 1921 "More than two discontigous rsvd regions found. " 1922 "Rsvd region base - 0x%llx, Rsvd region Size - 0x%llx\n", 1923 pMemoryManager->Ram.fbRegion[i].base, rsvdSize); 1924 NV_ASSERT(0); 1925 return NV_ERR_INVALID_STATE; 1926 } 1927 1928 topRegionIdx = i; 1929 topRsvdSize += rsvdSize; 1930 } 1931 } 1932 } 1933 1934 // 1935 // Sanity check against the biggest available memory chunk. Pick the smallest 1936 // of biggest available memory chunk or calculated total - reserved memory as 1937 // in vGPU we are still using OBJHEAP and there are some allocations which 1938 // happens at the top of the heap before we program this register 1939 // 1940 if (!memmgrIsPmaInitialized(pMemoryManager)) 1941 { 1942 NvU64 bytesTotal; 1943 const NvU64 vgpuHeapWarSize = 256 *1024 * 1024; 1944 NV_ASSERT_OK_OR_RETURN(heapInfo(pHeap, &freeMem, &bytesTotal, &base, 1945 &offset, &size)); 1946 1947 // 1948 // offset is the starting address of biggest empty block whose size is 1949 // returned and we care about the base of largest empty block 1950 // 1951 base = offset; 1952 1953 // 1954 // WAR - Bug-2383259 - TilL PMA is not enabled in vGPU-Host 1955 // we need to delay reserve some memory at the top to full fill lazy 1956 // allocations like FECS and GPCCS uCode. Leave 256MB at the top for 1957 // such lazy allocations 1958 // 1959 if (size > vgpuHeapWarSize) 1960 { 1961 size -= vgpuHeapWarSize; 1962 } 1963 } 1964 else 1965 { 1966 PMA_REGION_DESCRIPTOR *pFirstPmaRegionDesc = NULL; 1967 NvU32 numPmaRegions; 1968 NvU32 pmaConfig = PMA_QUERY_NUMA_ONLINED; 1969 1970 NV_ASSERT_OK_OR_RETURN(pmaGetRegionInfo(&pHeap->pmaObject, 1971 &numPmaRegions, &pFirstPmaRegionDesc)); 1972 1973 base = pFirstPmaRegionDesc->base; 1974 pmaGetFreeMemory(&pHeap->pmaObject, &freeMem); 1975 pmaGetTotalMemory(&pHeap->pmaObject, &size); 1976 1977 NV_ASSERT_OK(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfig)); 1978 1979 // 1980 // MIG won't be used alongside APM and hence the check below is of no use 1981 // Even if we enable the check for APM the check will fail given that after 1982 // enabling "scrub on free" using virtual CE writes, memory gets consumed by 1983 // page tables backing the scrubber channel virtual mappings and hence the 1984 // calculation below no longer holds good 1985 // In case of HCC, structures like PB, GPFIFO and USERD for scrubber and golden 1986 // channels are required to be in CPR vidmem. This changes the calculation below 1987 // We can ignore this for the non-MIG case. 1988 // 1989 // When FB memory is onlined as NUMA node, kernel can directly alloc FB memory 1990 // and hence free memory can not be expected to be same as total memory. 1991 // 1992 if ((!gpuIsCCorApmFeatureEnabled(pGpu) || IS_MIG_ENABLED(pGpu)) && 1993 !(pmaConfig & PMA_QUERY_NUMA_ONLINED)) 1994 { 1995 // 1996 // PMA should be completely free at this point, otherwise we risk 1997 // not setting the right partitionable range (pmaGetLargestFree's 1998 // offset argument is not implemented as of this writing, so we 1999 // only get the base address of the region that contains it). There 2000 // is a known allocation from the top-level scrubber channel that 2001 // is expected to be no larger than 128K. Issue a warning for any 2002 // other uses. 2003 // 2004 if ((size > RM_PAGE_SIZE_128K) && 2005 (freeMem < (size - RM_PAGE_SIZE_128K))) 2006 { 2007 NV_PRINTF(LEVEL_ERROR, 2008 "Assumption that PMA is empty (after accounting for the top-level scrubber) is not met!\n"); 2009 NV_PRINTF(LEVEL_ERROR, 2010 " free space = 0x%llx bytes, total space = 0x%llx bytes\n", 2011 freeMem, size); 2012 NV_ASSERT_OR_RETURN(freeMem >= (size - RM_PAGE_SIZE_128K), 2013 NV_ERR_INVALID_STATE); 2014 } 2015 } 2016 } 2017 2018 if (size == 0) 2019 { 2020 NV_PRINTF(LEVEL_ERROR, 2021 "No partitionable memory. MIG memory partitioning can't be enabled.\n"); 2022 return NV_OK; 2023 } 2024 2025 if (base != bottomRsvdSize) 2026 { 2027 NV_PRINTF(LEVEL_ERROR, 2028 "Partitionable memory start - 0x%llx not aligned with RM reserved " 2029 "region base-end - 0x%llx\n", base, bottomRsvdSize); 2030 return NV_ERR_INVALID_STATE; 2031 } 2032 2033 params.partitionableMemSize = size; 2034 params.bottomRsvdSize = bottomRsvdSize; 2035 params.topRsvdSize = topRsvdSize; 2036 2037 // Call physical MemorySystem to align and program the partitionable range 2038 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 2039 pRmApi->Control(pRmApi, 2040 pGpu->hInternalClient, 2041 pGpu->hInternalSubdevice, 2042 NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PARTITIONABLE_MEM, 2043 ¶ms, 2044 sizeof(params))); 2045 2046 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = 2047 rangeMake(params.partitionableStartAddr, params.partitionableEndAddr); 2048 2049 // 2050 // Make sure the created range is a valid range. 2051 // rangeIsEmpty checks lo > hi, which should be good enough to catch 2052 // inverted range case. 2053 // 2054 NV_ASSERT_OR_RETURN(!rangeIsEmpty(pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange), 2055 NV_ERR_INVALID_STATE); 2056 2057 if (!KBUS_CPU_VISIBLE_BAR12_DISABLED(pGpu)) 2058 { 2059 NV_ASSERT_OK_OR_RETURN(memmgrSetMIGPartitionableBAR1Range(pGpu, pMemoryManager)); 2060 } 2061 2062 if (IS_GSP_CLIENT(pGpu)) 2063 { 2064 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2065 2066 // 2067 // The Physical RM initializes its AMAPLIB context via 2068 // memsysSetPartitionableMem_HAL(). The GSP Client RM has a separate 2069 // AMAPLIB context that must also be initialized. 2070 // 2071 kmemsysReadMIGMemoryCfg_HAL(pGpu, pKernelMemorySystem); 2072 } 2073 2074 return NV_OK; 2075 } 2076 2077 NV_STATUS 2078 memmgrFillComprInfo_IMPL 2079 ( 2080 OBJGPU *pGpu, 2081 MemoryManager *pMemoryManager, 2082 NvU64 pageSize, 2083 NvU32 pageCount, 2084 NvU32 kind, 2085 NvU64 surfOffset, 2086 NvU32 compTagStartOffset, 2087 COMPR_INFO *pComprInfo 2088 ) 2089 { 2090 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 2091 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 2092 2093 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2094 2095 pComprInfo->kind = kind; 2096 2097 if (!memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 2098 return NV_OK; 2099 2100 // TODO: We will have to support compression on vGPU HOST for AC 2101 NV_ASSERT(compTagStartOffset != ~(NvU32)0); 2102 2103 pComprInfo->compPageShift = pMemorySystemConfig->comprPageShift; 2104 pComprInfo->compTagLineMin = compTagStartOffset; 2105 pComprInfo->compPageIndexLo = (NvU32)(surfOffset >> pComprInfo->compPageShift); 2106 pComprInfo->compPageIndexHi = (NvU32)((surfOffset + pageSize * pageCount - 1) >> pComprInfo->compPageShift); 2107 pComprInfo->compTagLineMultiplier = 1; 2108 2109 return NV_OK; 2110 } 2111 2112 NV_STATUS 2113 memmgrGetKindComprForGpu_KERNEL 2114 ( 2115 MemoryManager *pMemoryManager, 2116 MEMORY_DESCRIPTOR *pMemDesc, 2117 OBJGPU *pMappingGpu, 2118 NvU64 offset, 2119 NvU32 *pKind, 2120 COMPR_INFO *pComprInfo 2121 ) 2122 { 2123 NvU32 ctagId = FB_HWRESID_CTAGID_VAL_FERMI(memdescGetHwResId(pMemDesc)); 2124 NvU32 kind = memdescGetPteKindForGpu(pMemDesc, pMappingGpu); 2125 const MEMORY_SYSTEM_STATIC_CONFIG *pMappingMemSysConfig = 2126 kmemsysGetStaticConfig(pMappingGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pMappingGpu)); 2127 2128 // Compression is not supported on memory not backed by a GPU 2129 if (pMemDesc->pGpu != NULL && memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind) && 2130 (ctagId == 0 || ctagId == FB_HWRESID_CTAGID_VAL_FERMI(-1))) 2131 { 2132 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2133 2134 pComprInfo->kind = kind; 2135 pComprInfo->compPageShift = pMappingMemSysConfig->comprPageShift; 2136 pComprInfo->bPhysBasedComptags = NV_TRUE; 2137 pComprInfo->compTagLineMin = 1; 2138 } 2139 else 2140 { 2141 if (ctagId == FB_HWRESID_CTAGID_VAL_FERMI(0xcdcdcdcd)) 2142 { 2143 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2144 2145 pComprInfo->kind = memmgrGetUncompressedKind_HAL(pMappingGpu, pMemoryManager, kind, NV_TRUE); 2146 } 2147 else 2148 { 2149 memmgrFillComprInfoUncompressed(pMemoryManager, kind, pComprInfo); 2150 } 2151 } 2152 2153 *pKind = pComprInfo->kind; 2154 2155 return NV_OK; 2156 } 2157 2158 NV_STATUS 2159 memmgrGetKindComprFromMemDesc_IMPL 2160 ( 2161 MemoryManager *pMemoryManager, 2162 MEMORY_DESCRIPTOR *pMemDesc, 2163 NvU64 offset, 2164 NvU32 *kind, 2165 COMPR_INFO *pComprInfo 2166 ) 2167 { 2168 return memmgrGetKindComprForGpu_HAL(pMemoryManager, pMemDesc, pMemDesc->pGpu, 2169 offset, kind, pComprInfo); 2170 } 2171 2172 void 2173 memmgrSetMIGPartitionableMemoryRange_IMPL 2174 ( 2175 OBJGPU *pGpu, 2176 MemoryManager *pMemoryManager, 2177 NV_RANGE range 2178 ) 2179 { 2180 pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange = range; 2181 } 2182 2183 NV_RANGE 2184 memmgrGetMIGPartitionableMemoryRange_IMPL 2185 ( 2186 OBJGPU *pGpu, 2187 MemoryManager *pMemoryManager 2188 ) 2189 { 2190 return pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange; 2191 } 2192 2193 /* 2194 * @brief Sets total partitionable BAR1 2195 */ 2196 NV_STATUS 2197 memmgrSetMIGPartitionableBAR1Range_IMPL 2198 ( 2199 OBJGPU *pGpu, 2200 MemoryManager *pMemoryManager 2201 ) 2202 { 2203 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 2204 OBJVASPACE *pBar1VAS = kbusGetBar1VASpace_HAL(pGpu, pKernelBus); 2205 OBJEHEAP *pVASHeap; 2206 NvU64 largestFreeOffset = 0; 2207 NvU64 largestFreeSize = 0; 2208 NvU64 partitionableBar1Start; 2209 NvU64 partitionableBar1End; 2210 2211 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ZERO_FB)) 2212 return NV_OK; 2213 2214 NV_ASSERT_OR_RETURN(pBar1VAS != NULL, NV_ERR_INVALID_STATE); 2215 pVASHeap = vaspaceGetHeap(pBar1VAS); 2216 2217 // Get partitionable BAR1 range 2218 pVASHeap->eheapInfo(pVASHeap, NULL, NULL, &largestFreeOffset, &largestFreeSize, NULL, NULL); 2219 2220 // 2221 // We are not considering alignment here because VA space is reserved/allocated in chunks of pages 2222 // so largestFreeOffset should be already aligned. 2223 // 2224 partitionableBar1Start = largestFreeOffset; 2225 partitionableBar1End = largestFreeOffset + largestFreeSize - 1; 2226 NV_ASSERT_OR_RETURN(partitionableBar1Start >= vaspaceGetVaStart(pBar1VAS), NV_ERR_INVALID_STATE); 2227 NV_ASSERT_OR_RETURN(partitionableBar1End <= vaspaceGetVaLimit(pBar1VAS), NV_ERR_INVALID_STATE); 2228 2229 pMemoryManager->MIGMemoryPartitioningInfo.partitionableBar1Range = rangeMake(partitionableBar1Start, partitionableBar1End); 2230 return NV_OK; 2231 } 2232 2233 NV_RANGE 2234 memmgrGetMIGPartitionableBAR1Range_IMPL 2235 ( 2236 OBJGPU *pGpu, 2237 MemoryManager *pMemoryManager 2238 ) 2239 { 2240 return pMemoryManager->MIGMemoryPartitioningInfo.partitionableBar1Range; 2241 } 2242 2243 NV_STATUS 2244 memmgrAllocMIGGPUInstanceMemory_VF 2245 ( 2246 OBJGPU *pGpu, 2247 MemoryManager *pMemoryManager, 2248 NvU32 swizzId, 2249 NvHandle *phMemory, 2250 NV_RANGE *pAddrRange, 2251 Heap **ppMemoryPartitionHeap 2252 ) 2253 { 2254 // For vGpu we have a static memory allocation 2255 *phMemory = NV01_NULL_OBJECT; 2256 *pAddrRange = pMemoryManager->MIGMemoryPartitioningInfo.partitionableMemoryRange; 2257 *ppMemoryPartitionHeap = GPU_GET_HEAP(pGpu); 2258 2259 return NV_OK; 2260 } 2261 2262 // Function to allocate memory for a GPU instance 2263 NV_STATUS 2264 memmgrAllocMIGGPUInstanceMemory_PF 2265 ( 2266 OBJGPU *pGpu, 2267 MemoryManager *pMemoryManager, 2268 NvU32 swizzId, 2269 NvHandle *phMemory, 2270 NV_RANGE *pAddrRange, 2271 Heap **ppMemoryPartitionHeap 2272 ) 2273 { 2274 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2275 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2276 NV_STATUS rmStatus = NV_OK; 2277 NvHandle hMemory = 0; 2278 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2279 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2280 2281 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2282 NV_ASSERT_OK_OR_RETURN(kmemsysGetMIGGPUInstanceMemInfo(pGpu, pKernelMemorySystem, swizzId, pAddrRange)); 2283 2284 // 2285 // Only allocate memory for non swizzID-0 GPU instances as swizzID-0 owns full 2286 // gpu and there is no need to pre-reserve memory for that and non 2287 // coherent systems. In coherent NUMA systems, NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE 2288 // is not supported and the memory comes from the MIG partition memory 2289 // NUMA node. 2290 // 2291 if (kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2292 { 2293 if(bNumaEnabled) 2294 { 2295 NvS32 numaNodeId; 2296 NvU64 partitionBaseAddr = pAddrRange->lo; 2297 NvU64 partitionSize = rangeLength(*pAddrRange); 2298 NvU64 unalignedPartitionBaseAddr = partitionBaseAddr; 2299 NvU64 memblockSize = 0; 2300 2301 NV_ASSERT_OK_OR_RETURN(osNumaMemblockSize(&memblockSize)); 2302 // 2303 // Align the partition base and size to memblock size 2304 // Some FB memory is wasted here if it is not already aligned. 2305 // 2306 partitionBaseAddr = NV_ALIGN_UP64(unalignedPartitionBaseAddr, memblockSize); 2307 partitionSize -= (partitionBaseAddr - unalignedPartitionBaseAddr); 2308 partitionSize = NV_ALIGN_DOWN64(partitionSize, memblockSize); 2309 2310 if (kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager) == 0x0) 2311 { 2312 // Remove swizz Id 0 / baremetal GPU memory NUMA node 2313 pmaNumaOfflined(&GPU_GET_HEAP(pGpu)->pmaObject); 2314 kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, 0); 2315 } 2316 2317 // 2318 // The memory gets removed in memmgrFreeMIGGPUInstanceMemory if 2319 // there is any failure after adding the memory. 2320 // 2321 NV_ASSERT_OK_OR_RETURN(kmemsysNumaAddMemory_HAL(pGpu, 2322 pKernelMemorySystem, 2323 swizzId, 2324 partitionBaseAddr, 2325 partitionSize, 2326 &numaNodeId)); 2327 } 2328 else 2329 { 2330 // 2331 // Allocate memory using vidHeapControl 2332 // 2333 // vidHeapControl calls should happen outside GPU locks 2334 // This is a PMA requirement as memory allocation calls may invoke eviction 2335 // which UVM could get stuck behind GPU lock 2336 // See Bug 1735851-#24 2337 // 2338 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); 2339 2340 // Allocate gpfifo entries 2341 NV_MEMORY_ALLOCATION_PARAMS memAllocParams; 2342 portMemSet(&memAllocParams, 0, sizeof(NV_MEMORY_ALLOCATION_PARAMS)); 2343 memAllocParams.owner = HEAP_OWNER_RM_CLIENT_GENERIC; 2344 memAllocParams.type = NVOS32_TYPE_IMAGE; 2345 memAllocParams.size = rangeLength(*pAddrRange); 2346 memAllocParams.attr = DRF_DEF(OS32, _ATTR, _LOCATION, _VIDMEM); 2347 memAllocParams.attr |= DRF_DEF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS); 2348 memAllocParams.attr |= DRF_DEF(OS32, _ATTR, _PAGE_SIZE, _DEFAULT); 2349 memAllocParams.attr2 = DRF_DEF(OS32, _ATTR2, _PAGE_OFFLINING, _OFF); // free the offlined pages 2350 memAllocParams.flags |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 2351 memAllocParams.rangeLo = 0; 2352 memAllocParams.rangeHi = 0; 2353 memAllocParams.offset = pAddrRange->lo; // Offset needed if fixed address allocation 2354 memAllocParams.hVASpace = 0; // Physical allocation 2355 memAllocParams.internalflags = NVOS32_ALLOC_INTERNAL_FLAGS_SKIP_SCRUB; 2356 2357 rmStatus = pRmApi->Alloc(pRmApi, 2358 pMemoryManager->MIGMemoryPartitioningInfo.hClient, 2359 pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice, 2360 &hMemory, 2361 NV01_MEMORY_LOCAL_USER, 2362 &memAllocParams, 2363 sizeof(memAllocParams)); 2364 2365 // Reaquire the GPU locks 2366 if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_MEM) != NV_OK) 2367 { 2368 NV_PRINTF(LEVEL_ERROR, "failed to grab RM-Lock\n"); 2369 DBG_BREAKPOINT(); 2370 rmStatus = NV_ERR_GENERIC; 2371 goto cleanup; 2372 } 2373 2374 if (rmStatus != NV_OK) 2375 { 2376 NV_PRINTF(LEVEL_ERROR, 2377 "Unable to allocate physical memory for GPU instance.\n"); 2378 return rmStatus; 2379 } 2380 } 2381 } 2382 rmStatus = _memmgrInitMIGMemoryPartitionHeap(pGpu, pMemoryManager, swizzId, pAddrRange, ppMemoryPartitionHeap); 2383 if (rmStatus != NV_OK) 2384 { 2385 NV_PRINTF(LEVEL_ERROR, "Unable to initialize memory partition heap\n"); 2386 goto cleanup; 2387 } 2388 2389 NV_PRINTF(LEVEL_INFO, 2390 "Allocated memory partition heap for swizzId - %d with StartAddr - 0x%llx, endAddr - 0x%llx.\n", 2391 swizzId, pAddrRange->lo, pAddrRange->hi); 2392 2393 *phMemory = hMemory; 2394 return rmStatus; 2395 2396 cleanup: 2397 pRmApi->Free(pRmApi, pMemoryManager->MIGMemoryPartitioningInfo.hClient, hMemory); 2398 2399 return rmStatus; 2400 } 2401 2402 // Function to initialize heap for managing MIG partition memory 2403 static NV_STATUS 2404 _memmgrInitMIGMemoryPartitionHeap 2405 ( 2406 OBJGPU *pGpu, 2407 MemoryManager *pMemoryManager, 2408 NvU32 swizzId, 2409 NV_RANGE *pAddrRange, 2410 Heap **ppMemoryPartitionHeap 2411 ) 2412 { 2413 NV_STATUS status = NV_OK; 2414 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2415 Heap *pMemoryPartitionHeap = NULL; 2416 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2417 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2418 NvU64 partitionBaseAddr = pAddrRange->lo; 2419 NvU64 partitionSize = rangeLength(*pAddrRange); 2420 2421 // Use default heap for swizzID-0 as we don't prereserve memory for swizzID-0 2422 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2423 if (!kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2424 { 2425 *ppMemoryPartitionHeap = pMemoryManager->pHeap; 2426 return NV_OK; 2427 } 2428 else 2429 { 2430 *ppMemoryPartitionHeap = NULL; 2431 } 2432 2433 NV_ASSERT_OK_OR_GOTO( 2434 status, 2435 objCreate(ppMemoryPartitionHeap, pMemoryManager, Heap), 2436 fail); 2437 2438 pMemoryPartitionHeap = *ppMemoryPartitionHeap; 2439 2440 if (memmgrIsPmaEnabled(pMemoryManager) && 2441 memmgrIsPmaSupportedOnPlatform(pMemoryManager)) 2442 { 2443 portMemSet(&pMemoryPartitionHeap->pmaObject, 0, sizeof(pMemoryPartitionHeap->pmaObject)); 2444 NV_ASSERT_OK_OR_GOTO( 2445 status, 2446 memmgrPmaInitialize(pGpu, pMemoryManager, &pMemoryPartitionHeap->pmaObject), 2447 fail); 2448 2449 if (bNumaEnabled) 2450 { 2451 NV_ASSERT_OR_GOTO(pKernelMemorySystem->memPartitionNumaInfo[swizzId].bInUse, fail); 2452 partitionBaseAddr = pKernelMemorySystem->memPartitionNumaInfo[swizzId].offset; 2453 partitionSize = pKernelMemorySystem->memPartitionNumaInfo[swizzId].size; 2454 2455 // 2456 // The base and size passed here is the FB base and size and 2457 // not the partition's. pmaNumaOnlined requires the FB base and 2458 // size to convert between FB local address and SPA. 2459 // memmgrPmaRegisterRegions is where the partition's base and size 2460 // is reported to PMA. 2461 // 2462 NV_ASSERT_OK_OR_GOTO( 2463 status, 2464 pmaNumaOnlined(&pMemoryPartitionHeap->pmaObject, 2465 pKernelMemorySystem->memPartitionNumaInfo[swizzId].numaNodeId, 2466 pKernelMemorySystem->coherentCpuFbBase, 2467 pKernelMemorySystem->numaOnlineSize), 2468 fail); 2469 } 2470 } 2471 2472 NV_ASSERT_OK_OR_GOTO( 2473 status, 2474 heapInit(pGpu, pMemoryPartitionHeap, partitionBaseAddr, 2475 partitionSize, 2476 HEAP_TYPE_PARTITION_LOCAL, 2477 GPU_GFID_PF, 2478 NULL), 2479 fail); 2480 2481 if (memmgrIsPmaInitialized(pMemoryManager) && 2482 (pMemoryPartitionHeap->bHasFbRegions)) 2483 { 2484 NV_ASSERT_OK_OR_GOTO( 2485 status, 2486 memmgrPmaRegisterRegions(pGpu, pMemoryManager, pMemoryPartitionHeap, 2487 &pMemoryPartitionHeap->pmaObject), 2488 fail); 2489 } 2490 2491 if (!IsSLIEnabled(pGpu)) 2492 { 2493 // Do the actual blacklisting of pages from the heap 2494 if (pMemoryPartitionHeap->blackListAddresses.count != 0) 2495 { 2496 status = heapBlackListPages(pGpu, pMemoryPartitionHeap); 2497 2498 if (status != NV_OK) 2499 { 2500 // Warn and continue 2501 NV_PRINTF(LEVEL_WARNING, "Error 0x%x creating blacklist\n", 2502 status); 2503 } 2504 } 2505 } 2506 2507 return NV_OK; 2508 2509 fail: 2510 2511 if (pMemoryPartitionHeap != NULL) 2512 { 2513 objDelete(pMemoryPartitionHeap); 2514 *ppMemoryPartitionHeap = NULL; 2515 } 2516 2517 return status; 2518 } 2519 2520 // Function to free GPU instance memory 2521 NV_STATUS 2522 memmgrFreeMIGGPUInstanceMemory_IMPL 2523 ( 2524 OBJGPU *pGpu, 2525 MemoryManager *pMemoryManager, 2526 NvU32 swizzId, 2527 NvHandle hMemory, 2528 Heap **ppMemoryPartitionHeap 2529 ) 2530 { 2531 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 2532 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2533 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2534 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2535 2536 NV_ASSERT_OR_RETURN(pKernelMIGManager != NULL, NV_ERR_INVALID_STATE); 2537 2538 // Nothing to do for swizzId 0 as we neither allocate memory nor allocate new heap object 2539 if (!kmigmgrIsMemoryPartitioningNeeded_HAL(pGpu, pKernelMIGManager, swizzId)) 2540 return NV_OK; 2541 2542 objDelete(*ppMemoryPartitionHeap); 2543 *ppMemoryPartitionHeap = NULL; 2544 2545 if (bNumaEnabled) 2546 { 2547 kmemsysNumaRemoveMemory_HAL(pGpu, pKernelMemorySystem, swizzId); 2548 2549 if (kmigmgrGetSwizzIdInUseMask(pGpu, pKernelMIGManager) == 0x0) 2550 { 2551 NvS32 numaNodeId; 2552 2553 // Add back the baremetal GPU memory NUMA node. 2554 NV_ASSERT_OK_OR_RETURN(kmemsysNumaAddMemory_HAL(pGpu, 2555 pKernelMemorySystem, 2556 0, 2557 pKernelMemorySystem->numaOnlineBase, 2558 pKernelMemorySystem->numaOnlineSize, 2559 &numaNodeId)); 2560 // Baremetal NUMA node id should be same as pGpu->numaNodeId 2561 NV_ASSERT_OR_RETURN(numaNodeId == pGpu->numaNodeId, NV_ERR_INVALID_STATE); 2562 NV_ASSERT_OK_OR_RETURN(pmaNumaOnlined(&GPU_GET_HEAP(pGpu)->pmaObject, 2563 pGpu->numaNodeId, 2564 pKernelMemorySystem->coherentCpuFbBase, 2565 pKernelMemorySystem->numaOnlineSize)); 2566 } 2567 } 2568 2569 // Free allocated memory 2570 if (!bNumaEnabled && (hMemory != NV01_NULL_OBJECT)) 2571 { 2572 pRmApi->Free(pRmApi, pMemoryManager->MIGMemoryPartitioningInfo.hClient, hMemory); 2573 } 2574 return NV_OK; 2575 } 2576 2577 void memmgrComprInfoDisableCompression_IMPL 2578 ( 2579 MemoryManager *pMemoryManager, 2580 COMPR_INFO *pComprInfo 2581 ) 2582 { 2583 memmgrFillComprInfoUncompressed(pMemoryManager, pComprInfo->kind, pComprInfo); 2584 } 2585 2586 void memmgrFillComprInfoUncompressed_IMPL 2587 ( 2588 MemoryManager *pMemoryManager, 2589 NvU32 kind, 2590 COMPR_INFO *pComprInfo 2591 ) 2592 { 2593 if (memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 2594 kind = memmgrGetUncompressedKind_HAL(ENG_GET_GPU(pMemoryManager), pMemoryManager, kind, NV_FALSE); 2595 2596 portMemSet(pComprInfo, 0, sizeof(*pComprInfo)); 2597 pComprInfo->kind = kind; 2598 } 2599 2600 /*! 2601 * @brief Creates the SW state of the page level pools. 2602 * 2603 * @param pGpu 2604 * @param pMemoryManager 2605 * 2606 * @returns On success, returns NV_OK. 2607 * On failure, returns error code. 2608 */ 2609 NV_STATUS 2610 memmgrPageLevelPoolsCreate_IMPL 2611 ( 2612 OBJGPU *pGpu, 2613 MemoryManager *pMemoryManager 2614 ) 2615 { 2616 NV_STATUS status = NV_OK; 2617 2618 if (RMCFG_FEATURE_PMA && 2619 memmgrIsPmaInitialized(pMemoryManager) && 2620 memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2621 { 2622 Heap *pHeap = GPU_GET_HEAP(pGpu); 2623 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 2624 const GMMU_FMT *pFmt = NULL; 2625 2626 pFmt = kgmmuFmtGet(pKernelGmmu, GMMU_FMT_VERSION_DEFAULT, 0); 2627 NV_ASSERT_OR_RETURN(NULL != pFmt, NV_ERR_INVALID_ARGUMENT); 2628 2629 status = rmMemPoolSetup((void *)&pHeap->pmaObject, &pMemoryManager->pPageLevelReserve, 2630 (pFmt->version == GMMU_FMT_VERSION_1) ? POOL_CONFIG_GMMU_FMT_1 : POOL_CONFIG_GMMU_FMT_2); 2631 2632 NV_ASSERT(NV_OK == status); 2633 2634 // 2635 // Allocate the pool in CPR in case of Confidential Compute 2636 // When Hopper Confidential Compute is enabled, page tables 2637 // cannot be in non-CPR region 2638 // 2639 if (gpuIsCCFeatureEnabled(pGpu) && (status == NV_OK)) 2640 { 2641 rmMemPoolAllocateProtectedMemory(pMemoryManager->pPageLevelReserve, NV_TRUE); 2642 } 2643 } 2644 return status; 2645 } 2646 2647 /*! 2648 * @brief Destroys the SW state of the page level pools. 2649 * 2650 * @param pGpu 2651 * @param pMemoryManager 2652 * 2653 * @returns 2654 */ 2655 void 2656 memmgrPageLevelPoolsDestroy_IMPL 2657 ( 2658 OBJGPU *pGpu, 2659 MemoryManager *pMemoryManager 2660 ) 2661 { 2662 if (RMCFG_FEATURE_PMA && 2663 memmgrIsPmaInitialized(pMemoryManager) && 2664 memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2665 { 2666 rmMemPoolDestroy(pMemoryManager->pPageLevelReserve); 2667 pMemoryManager->pPageLevelReserve = NULL; 2668 } 2669 } 2670 2671 /*! 2672 * @brief Gets page level pool to use 2673 * 2674 * @param pGpu 2675 * @param pMemoryManager 2676 * @param[in] hClient client handle 2677 * @param[out] ppMemPoolInfo page level pool 2678 * 2679 * @returns On success, returns NV_OK. 2680 * On failure, returns error code. 2681 */ 2682 NV_STATUS 2683 memmgrPageLevelPoolsGetInfo_IMPL 2684 ( 2685 OBJGPU *pGpu, 2686 MemoryManager *pMemoryManager, 2687 NvHandle hClient, 2688 RM_POOL_ALLOC_MEM_RESERVE_INFO **ppMemPoolInfo 2689 ) 2690 { 2691 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 2692 NvBool bMemPartitioningEnabled = (pKernelMIGManager != NULL) && kmigmgrIsMIGMemPartitioningEnabled(pGpu, pKernelMIGManager); 2693 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemPool = NULL; 2694 NV_ASSERT_OR_RETURN(ppMemPoolInfo != NULL, NV_ERR_INVALID_ARGUMENT); 2695 2696 if (!memmgrIsPmaInitialized(pMemoryManager) || 2697 !memmgrAreClientPageTablesPmaManaged(pMemoryManager)) 2698 { 2699 return NV_ERR_INVALID_STATE; 2700 } 2701 2702 // If memory partitioning is enabled, then use per-partition pool allocator 2703 if (bMemPartitioningEnabled) 2704 { 2705 MIG_INSTANCE_REF ref; 2706 NV_ASSERT_OK_OR_RETURN( 2707 kmigmgrGetInstanceRefFromClient(pGpu, pKernelMIGManager, hClient, &ref)); 2708 pMemPool = ref.pKernelMIGGpuInstance->pPageTableMemPool; 2709 } 2710 else 2711 { 2712 pMemPool = pMemoryManager->pPageLevelReserve; 2713 } 2714 NV_ASSERT_OR_RETURN(pMemPool != NULL, NV_ERR_INVALID_STATE); 2715 2716 *ppMemPoolInfo = pMemPool; 2717 return NV_OK; 2718 } 2719 2720 /*! 2721 * @brief Initialize the PMA object 2722 * 2723 * @param pGpu 2724 * @param pMemoryManager 2725 * @param[in] pPma Pointer to the PMA object to init 2726 * 2727 * @returns On success, returns NV_OK. 2728 * On failure, returns error code. 2729 */ 2730 NV_STATUS 2731 memmgrPmaInitialize_IMPL 2732 ( 2733 OBJGPU *pGpu, 2734 MemoryManager *pMemoryManager, 2735 PMA *pPma 2736 ) 2737 { 2738 NvU32 pmaInitFlags = PMA_INIT_NONE; 2739 NV_STATUS status = NV_OK; 2740 NvBool bNumaEnabled = osNumaOnliningEnabled(pGpu->pOsGpuInfo); 2741 2742 NV_ASSERT(memmgrIsPmaEnabled(pMemoryManager) && 2743 memmgrIsPmaSupportedOnPlatform(pMemoryManager)); 2744 2745 if (memmgrIsPmaForcePersistence(pMemoryManager)) 2746 { 2747 pmaInitFlags |= PMA_INIT_FORCE_PERSISTENCE; 2748 } 2749 2750 if (memmgrIsScrubOnFreeEnabled(pMemoryManager)) 2751 { 2752 pmaInitFlags |= PMA_INIT_SCRUB_ON_FREE; 2753 } 2754 2755 // Disable client page table management on SLI. 2756 if (IsSLIEnabled(pGpu)) 2757 { 2758 memmgrSetClientPageTablesPmaManaged(pMemoryManager, NV_FALSE); 2759 } 2760 2761 if (bNumaEnabled) 2762 { 2763 NV_PRINTF(LEVEL_INFO, "Initializing PMA with NUMA flag.\n"); 2764 pmaInitFlags |= PMA_INIT_NUMA; 2765 } 2766 2767 if (gpuIsSelfHosted(pGpu)) 2768 { 2769 NV_PRINTF(LEVEL_INFO, "Initializing PMA with NUMA_AUTO_ONLINE flag.\n"); 2770 pmaInitFlags |= PMA_INIT_NUMA_AUTO_ONLINE; 2771 } 2772 2773 if (memmgrIsPmaAddrTree(pMemoryManager)) 2774 { 2775 pmaInitFlags |= PMA_INIT_ADDRTREE; 2776 } 2777 2778 status = pmaInitialize(pPma, pmaInitFlags); 2779 if (status != NV_OK) 2780 { 2781 NV_PRINTF(LEVEL_ERROR, "Failed to initialize PMA!\n"); 2782 return status; 2783 } 2784 2785 if (bNumaEnabled) 2786 { 2787 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 2788 2789 NvU32 numaSkipReclaimVal = NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_DEFAULT; 2790 2791 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE, &numaSkipReclaimVal) == NV_OK) 2792 { 2793 if (numaSkipReclaimVal > NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_MAX) 2794 { 2795 numaSkipReclaimVal = NV_REG_STR_RM_NUMA_ALLOC_SKIP_RECLAIM_PERCENTAGE_MAX; 2796 } 2797 } 2798 pmaNumaSetReclaimSkipThreshold(pPma, numaSkipReclaimVal); 2799 2800 // Full FB memory is added and onlined already 2801 if (pKernelMemorySystem->memPartitionNumaInfo[0].bInUse) 2802 { 2803 NV_ASSERT_OK_OR_RETURN(pmaNumaOnlined(pPma, pGpu->numaNodeId, 2804 pKernelMemorySystem->coherentCpuFbBase, 2805 pKernelMemorySystem->numaOnlineSize)); 2806 } 2807 2808 } 2809 2810 return NV_OK; 2811 } 2812 2813 NV_STATUS 2814 memmgrInitFbRegions_IMPL 2815 ( 2816 OBJGPU *pGpu, 2817 MemoryManager *pMemoryManager 2818 ) 2819 { 2820 NV_ASSERT_OR_RETURN(pMemoryManager->Ram.numFBRegions == 0, NV_ERR_INVALID_STATE); 2821 2822 // Dont setup regions if FB is broken and we aren't using L2 cache as "FB". 2823 if ((pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) && 2824 !gpuIsCacheOnlyModeEnabled(pGpu))) 2825 return NV_OK; 2826 2827 NV_ASSERT_OK_OR_RETURN(memmgrInitBaseFbRegions_HAL(pGpu, pMemoryManager)); 2828 2829 NV_ASSERT_OK_OR_RETURN(memmgrInitFbRegionsHal_HAL(pGpu, pMemoryManager)); 2830 2831 // 2832 // Build a list of regions sorted by allocation priority 2833 // (highest to lowest). Used for allocations using ObjHeap. 2834 // 2835 memmgrRegenerateFbRegionPriority(pGpu, pMemoryManager); 2836 2837 if (RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM) 2838 { 2839 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_EXTERNAL_HEAP_CONTROL)) 2840 { 2841 // KMD in WDDM mode 2842 if (pMemoryManager->bMixedDensityFbp) 2843 { 2844 // 2845 // For mixed memory on LDDM platforms, when we are using kernel-managed 2846 // heap (not TCC mode), we want to prefer allocating in slow memory to conserve 2847 // fast memory for applications. 2848 // 2849 pMemoryManager->bPreferSlowRegion = NV_TRUE; 2850 } 2851 } 2852 } 2853 2854 NV_ASSERT_OK_OR_RETURN(memmgrSetPlatformPmaSupport(pGpu, pMemoryManager)); 2855 2856 return NV_OK; 2857 } 2858 2859 /*! 2860 * @brief Register regions to the PMA object 2861 * 2862 * @param pGpu 2863 * @param pMemoryManager 2864 * @param[in] pPma Pointer to the PMA object to register with 2865 * 2866 * @returns On success, returns NV_OK. 2867 * On failure, returns error code. 2868 */ 2869 NV_STATUS 2870 memmgrPmaRegisterRegions_IMPL 2871 ( 2872 OBJGPU *pGpu, 2873 MemoryManager *pMemoryManager, 2874 Heap *pHeap, 2875 PMA *pPma 2876 ) 2877 { 2878 HEAP_TYPE_INTERNAL heapType = pHeap->heapType; 2879 PMA_REGION_DESCRIPTOR pmaRegion; 2880 NvU32 pmaRegionIdx = 0; 2881 NvU32 i; 2882 PMA_BLACKLIST_ADDRESS *pBlacklistPages = NULL; 2883 NvU32 blRegionCount = 0; 2884 NvU32 blPageIndex; 2885 NvU32 blackListCount; 2886 NvU64 base, size; 2887 NvU64 pmaTotalMemorySize = 0; 2888 NV_STATUS status = NV_OK; 2889 const MEMORY_SYSTEM_STATIC_CONFIG *pMemsysConfig = 2890 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 2891 2892 blackListCount = pHeap->blackListAddresses.count; 2893 base = pHeap->base; 2894 size = pHeap->total; 2895 2896 // 2897 // If there are blacklisted pages, prepare a staging buffer to pass the 2898 // per-region blacklisted pages to PMA 2899 // 2900 if (blackListCount > 0) 2901 { 2902 pBlacklistPages = portMemAllocNonPaged( 2903 sizeof(PMA_BLACKLIST_ADDRESS) * blackListCount); 2904 if (pBlacklistPages == NULL) 2905 { 2906 NV_PRINTF(LEVEL_ERROR, 2907 "Could not allocate memory for blackList!\n"); 2908 status = NV_ERR_NO_MEMORY; 2909 goto _pmaInitFailed; 2910 } 2911 } 2912 2913 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 2914 { 2915 // 2916 // Skip all regions that are completely outside the heap boundry 2917 // OR marked as internal(used for internal RM allocations) 2918 // OR marked as reserved(used for console, display, link training buffer etc.) 2919 // 2920 if ((pMemoryManager->Ram.fbRegion[i].limit < base || 2921 pMemoryManager->Ram.fbRegion[i].base >= (base + size)) || 2922 (pMemoryManager->Ram.fbRegion[i].bInternalHeap) || 2923 (pMemoryManager->Ram.fbRegion[i].bRsvdRegion)) 2924 { 2925 continue; 2926 } 2927 2928 NV_PRINTF(LEVEL_INFO, 2929 "PMA: Register FB region[%d] %llx..%llx EXTERNAL\n", i, 2930 pMemoryManager->Ram.fbRegion[i].base, pMemoryManager->Ram.fbRegion[i].limit); 2931 2932 pmaRegion.base = pMemoryManager->Ram.fbRegion[i].base; 2933 pmaRegion.limit = pMemoryManager->Ram.fbRegion[i].limit; 2934 2935 // Check if the base of managed memory is not based at FB region base. 2936 if (pmaRegion.base < base) 2937 { 2938 pmaRegion.base = base; 2939 } 2940 2941 // check if limit of managed memory is less than FB region limit 2942 if (pmaRegion.limit >= (base + size)) 2943 { 2944 pmaRegion.limit = base + size - 1; 2945 } 2946 2947 pmaRegion.performance = pMemoryManager->Ram.fbRegion[i].performance; 2948 pmaRegion.bSupportCompressed = pMemoryManager->Ram.fbRegion[i].bSupportCompressed; 2949 pmaRegion.bSupportISO = pMemoryManager->Ram.fbRegion[i].bSupportISO; 2950 pmaRegion.bProtected = pMemoryManager->Ram.fbRegion[i].bProtected; 2951 2952 // 2953 // Now we know the region, find if it has any blacklisted pages 2954 // TODO: Try to coalesce to unique 64K pages 2955 // 2956 blRegionCount = 0; 2957 if (pBlacklistPages != NULL) 2958 { 2959 for (blPageIndex = 0; blPageIndex < blackListCount; blPageIndex++) 2960 { 2961 if ((pHeap->blackListAddresses.data[blPageIndex].address 2962 != NV2080_CTRL_FB_OFFLINED_PAGES_INVALID_ADDRESS) && 2963 (pHeap->blackListAddresses.data[blPageIndex].address >= pmaRegion.base) && 2964 (pHeap->blackListAddresses.data[blPageIndex].address <= pmaRegion.limit)) 2965 { 2966 // Collect the region's blacklisted pages 2967 pBlacklistPages[blRegionCount].physOffset = pHeap->blackListAddresses.data[blPageIndex].address; 2968 2969 pBlacklistPages[blRegionCount].bIsDynamic = 2970 ((pHeap->blackListAddresses.data[blPageIndex].type == 2971 NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_MULTIPLE_SBE) || 2972 (pHeap->blackListAddresses.data[blPageIndex].type == 2973 NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE)); 2974 2975 blRegionCount++; 2976 } 2977 } 2978 } 2979 2980 pmaTotalMemorySize += (pmaRegion.limit - pmaRegion.base + 1); 2981 NV_PRINTF(LEVEL_INFO, 2982 "Register FB region %llx..%llx of size %llx with PMA\n", 2983 pmaRegion.base, pmaRegion.limit, 2984 pmaRegion.limit - pmaRegion.base + 1); 2985 // 2986 // Register the region for PMA management, and note if asynchronous 2987 // scrubbing is enabled. Synchronous scrubbing is done before 2988 // heap/PMA is initialized, but asynchronously scrubbed pages will 2989 // need to be unmarked once they are scrubbed. 2990 // 2991 status = pmaRegisterRegion(pPma, pmaRegionIdx, 2992 memmgrEccScrubInProgress_HAL(pGpu, pMemoryManager), 2993 &pmaRegion, blRegionCount, 2994 ((blRegionCount==0) ? NULL : pBlacklistPages)); 2995 if (status != NV_OK) 2996 { 2997 NV_PRINTF(LEVEL_ERROR, 2998 "failed to register FB region %llx..%llx with PMA\n", 2999 pmaRegion.base, pmaRegion.limit); 3000 DBG_BREAKPOINT(); 3001 goto _pmaInitFailed; 3002 } 3003 pmaRegionIdx++; 3004 } 3005 3006 if (gpuIsSelfHosted(pGpu) && osNumaOnliningEnabled(pGpu->pOsGpuInfo)) 3007 { 3008 // 3009 // NUMA onlined memory size should not exceed memory size assigned to PMA. 3010 // TODO : Currently in selfhosted and P9+GV100 systems numaOnlined size is less 3011 // than PMA Memory Size. Ideally both of them should be identical. Bug 4051320. 3012 // 3013 NvU64 numaTotalSize = 0; 3014 NvU64 numaFreeSize = 0; 3015 osGetNumaMemoryUsage(pPma->numaNodeId, &numaFreeSize, &numaTotalSize); 3016 NV_ASSERT_OR_RETURN(pmaTotalMemorySize >= numaTotalSize, NV_ERR_INVALID_STATE); 3017 } 3018 // 3019 // bug #200354346, make sure the RM reserved region(s) are 3020 // scrubbed during the region creation itself. Top Down scrubber, 3021 // skips the RM reserved region(s) because the assumption is, they 3022 // are pre-scrubbed. 3023 // 3024 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 3025 memmgrScrubInternalRegions_HAL(pGpu, pMemoryManager); 3026 3027 _pmaInitFailed: 3028 portMemFree(pBlacklistPages); 3029 3030 if ((status == NV_OK) && (pMemsysConfig->fbOverrideStartKb != 0)) 3031 { 3032 NvU64 allocSize = NV_ALIGN_UP(((NvU64)pMemsysConfig->fbOverrideStartKb << 10), PMA_GRANULARITY); 3033 NvU32 numPages = (NvU32)(allocSize >> PMA_PAGE_SHIFT); 3034 PMA_ALLOCATION_OPTIONS allocOptions = {0}; 3035 3036 allocOptions.flags = PMA_ALLOCATE_CONTIGUOUS; 3037 allocOptions.flags |= PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE; 3038 allocOptions.physBegin = 0; 3039 allocOptions.physEnd = allocSize - 1; 3040 3041 // This is intentionally thrown away 3042 NvU64 *pPages = NULL; 3043 pPages = portMemAllocNonPaged(numPages * sizeof(NvU64)); 3044 if (pPages != NULL) 3045 { 3046 // Accommodate the regkey override for FB start 3047 status = pmaAllocatePages(pPma, numPages, _PMA_64KB, &allocOptions, pPages); 3048 portMemFree(pPages); 3049 } 3050 } 3051 if (status != NV_OK) 3052 { 3053 if (memmgrIsPmaInitialized(pMemoryManager)) 3054 { 3055 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 3056 { 3057 memmgrSetPmaInitialized(pMemoryManager, NV_FALSE); 3058 } 3059 pmaDestroy(pPma); 3060 } 3061 } 3062 3063 return status; 3064 } 3065 3066 /*! 3067 * @brief Allocate internal handles for MIG partition memory allocation 3068 */ 3069 NV_STATUS 3070 memmgrAllocMIGMemoryAllocationInternalHandles_IMPL 3071 ( 3072 OBJGPU *pGpu, 3073 MemoryManager *pMemoryManager 3074 ) 3075 { 3076 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 3077 3078 NV_ASSERT_OR_RETURN(pMemoryManager->MIGMemoryPartitioningInfo.hClient == NV01_NULL_OBJECT, NV_ERR_INVALID_STATE); 3079 NV_ASSERT_OK_OR_RETURN( 3080 rmapiutilAllocClientAndDeviceHandles(pRmApi, pGpu, 3081 &pMemoryManager->MIGMemoryPartitioningInfo.hClient, 3082 &pMemoryManager->MIGMemoryPartitioningInfo.hDevice, 3083 &pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice)); 3084 3085 return NV_OK; 3086 } 3087 3088 /*! 3089 * @brief Free internal handles used to support MIG memory partitioning 3090 */ 3091 void 3092 memmgrFreeMIGMemoryAllocationInternalHandles_IMPL 3093 ( 3094 OBJGPU *pGpu, 3095 MemoryManager *pMemoryManager 3096 ) 3097 { 3098 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 3099 3100 rmapiutilFreeClientAndDeviceHandles(pRmApi, 3101 &pMemoryManager->MIGMemoryPartitioningInfo.hClient, 3102 &pMemoryManager->MIGMemoryPartitioningInfo.hDevice, 3103 &pMemoryManager->MIGMemoryPartitioningInfo.hSubdevice); 3104 } 3105 3106 /*! 3107 * @brief Gets free memory (client visible) for all valid GPU instances 3108 */ 3109 void 3110 memmgrGetFreeMemoryForAllMIGGPUInstances_IMPL 3111 ( 3112 OBJGPU *pGpu, 3113 MemoryManager *pMemoryManager, 3114 NvU64 *pBytes 3115 ) 3116 { 3117 NvU64 val = 0; 3118 Heap *pHeap = NULL; 3119 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3120 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance; 3121 3122 *pBytes = 0; 3123 3124 FOR_EACH_VALID_GPU_INSTANCE(pGpu, pKernelMIGManager, pKernelMIGGPUInstance) 3125 { 3126 NV_ASSERT(pKernelMIGGPUInstance->pMemoryPartitionHeap != NULL); 3127 pHeap = pKernelMIGGPUInstance->pMemoryPartitionHeap; 3128 3129 if (memmgrIsPmaInitialized(pMemoryManager)) 3130 pmaGetFreeMemory(&pHeap->pmaObject, &val); 3131 else 3132 heapGetFree(pHeap, &val); 3133 3134 *pBytes += val; 3135 } 3136 FOR_EACH_VALID_GPU_INSTANCE_END(); 3137 } 3138 3139 /*! 3140 * @brief Gets total memory for all valid GPU instances 3141 * 3142 * @param pGpu 3143 * @param pMemoryManager 3144 * @param[out] pBytes pointer to the total memory 3145 * 3146 */ 3147 void 3148 memmgrGetTotalMemoryForAllMIGGPUInstances_IMPL 3149 ( 3150 OBJGPU *pGpu, 3151 MemoryManager *pMemoryManager, 3152 NvU64 *pBytes 3153 ) 3154 { 3155 NvU64 val = 0; 3156 Heap *pHeap = NULL; 3157 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 3158 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance; 3159 3160 *pBytes = 0; 3161 3162 FOR_EACH_VALID_GPU_INSTANCE(pGpu, pKernelMIGManager, pKernelMIGGPUInstance) 3163 { 3164 NV_ASSERT(pKernelMIGGPUInstance->pMemoryPartitionHeap != NULL); 3165 pHeap = pKernelMIGGPUInstance->pMemoryPartitionHeap; 3166 3167 if (memmgrIsPmaInitialized(pMemoryManager)) 3168 pmaGetTotalMemory(&pHeap->pmaObject, &val); 3169 else 3170 heapGetSize(pHeap, &val); 3171 3172 *pBytes += val; 3173 } 3174 FOR_EACH_VALID_GPU_INSTANCE_END(); 3175 } 3176 3177 void 3178 memmgrGetTopLevelScrubberStatus_IMPL 3179 ( 3180 OBJGPU *pGpu, 3181 MemoryManager *pMemoryManager, 3182 NvBool *pbTopLevelScrubberEnabled, 3183 NvBool *pbTopLevelScrubberConstructed 3184 ) 3185 { 3186 NvBool bTopLevelScrubberEnabled = NV_FALSE; 3187 NvBool bTopLevelScrubberConstructed = NV_FALSE; 3188 NvU32 pmaConfigs = PMA_QUERY_SCRUB_ENABLED | PMA_QUERY_SCRUB_VALID; 3189 3190 if (memmgrIsPmaInitialized(pMemoryManager)) 3191 { 3192 Heap *pHeap = GPU_GET_HEAP(pGpu); 3193 NV_ASSERT_OK(pmaQueryConfigs(&pHeap->pmaObject, &pmaConfigs)); 3194 bTopLevelScrubberEnabled = (pmaConfigs & PMA_QUERY_SCRUB_ENABLED) != 0x0; 3195 bTopLevelScrubberConstructed = (pmaConfigs & PMA_QUERY_SCRUB_VALID) != 0x0; 3196 } 3197 3198 if (pbTopLevelScrubberEnabled != NULL) 3199 *pbTopLevelScrubberEnabled = bTopLevelScrubberEnabled; 3200 if (pbTopLevelScrubberConstructed != NULL) 3201 *pbTopLevelScrubberConstructed = bTopLevelScrubberConstructed; 3202 } 3203 3204 /** 3205 * @brief Save pre-MIG top level scrubber constructed status and teardown if constructed 3206 */ 3207 NV_STATUS 3208 memmgrSaveAndDestroyTopLevelScrubber_IMPL 3209 ( 3210 OBJGPU *pGpu, 3211 MemoryManager *pMemoryManager 3212 ) 3213 { 3214 // Save the pre-MIG top-level scrubber status for later 3215 memmgrGetTopLevelScrubberStatus(pGpu, pMemoryManager, NULL, &pMemoryManager->MIGMemoryPartitioningInfo.bNonMIGTopLevelScrubber); 3216 3217 // Destroy the top level scrubber if it exists 3218 if (pMemoryManager->MIGMemoryPartitioningInfo.bNonMIGTopLevelScrubber) 3219 { 3220 // Delete top level scrubber 3221 NV_ASSERT_OK_OR_RETURN(memmgrScrubHandlePreSchedulingDisable_HAL(pGpu, pMemoryManager)); 3222 } 3223 3224 return NV_OK; 3225 } 3226 3227 /** 3228 * @brief Init top level scrubber if previous status was constructed 3229 */ 3230 NV_STATUS 3231 memmgrInitSavedTopLevelScrubber_IMPL 3232 ( 3233 OBJGPU *pGpu, 3234 MemoryManager *pMemoryManager 3235 ) 3236 { 3237 if (!pMemoryManager->MIGMemoryPartitioningInfo.bNonMIGTopLevelScrubber) 3238 return NV_OK; 3239 3240 NV_ASSERT_OK_OR_RETURN(memmgrScrubHandlePostSchedulingEnable_HAL(pGpu, pMemoryManager)); 3241 3242 return NV_OK; 3243 } 3244 3245 /*! 3246 * @brief Return the full address range for the partition assigend for the vGPU. 3247 * 3248 * @param[in] pGpu 3249 * @param[in] pMemoryManager 3250 * @param[out] base reference to the base address of the partition 3251 * @param[out] size reference to the overall size of the partition 3252 */ 3253 static void 3254 _memmgrGetFullMIGAddrRange 3255 ( 3256 OBJGPU *pGpu, 3257 MemoryManager *pMemoryManager, 3258 NvU64 *base, 3259 NvU64 *size 3260 ) 3261 { 3262 NvU32 i; 3263 NvU64 lo, hi; 3264 3265 *base = 0; 3266 *size = 0; 3267 if (pMemoryManager->Ram.numFBRegions == 0) 3268 { 3269 return; 3270 } 3271 3272 lo = pMemoryManager->Ram.fbRegion[0].base; 3273 hi = pMemoryManager->Ram.fbRegion[0].limit; 3274 3275 for (i = 1; i < pMemoryManager->Ram.numFBRegions; i++) 3276 { 3277 if (pMemoryManager->Ram.fbRegion[i].base < lo) 3278 { 3279 lo = pMemoryManager->Ram.fbRegion[i].base; 3280 } 3281 3282 if (pMemoryManager->Ram.fbRegion[i].limit > hi) 3283 { 3284 hi = pMemoryManager->Ram.fbRegion[i].limit; 3285 } 3286 } 3287 3288 *base = lo; 3289 *size = hi - lo + 1; 3290 } 3291 3292 /*! 3293 * @brief Discover MIG partitionable memory range based on PMA status 3294 */ 3295 NV_STATUS 3296 memmgrDiscoverMIGPartitionableMemoryRange_VF 3297 ( 3298 OBJGPU *pGpu, 3299 MemoryManager *pMemoryManager, 3300 NV_RANGE *pMemoryRange 3301 ) 3302 { 3303 NvU64 size; 3304 NvU64 base; 3305 3306 // Set memory information 3307 if (!memmgrIsPmaInitialized(pMemoryManager)) 3308 { 3309 Heap *pHeap = GPU_GET_HEAP(pGpu); 3310 NvU64 freeMem; 3311 NvU64 bytesTotal; 3312 NvU64 offset; 3313 3314 NV_ASSERT_OK_OR_RETURN(heapInfo(pHeap, &freeMem, &bytesTotal, &base, 3315 &offset, &size)); 3316 3317 // 3318 // offset is the starting address of biggest empty block whose size is 3319 // returned and we care about the base of largest empty block 3320 // 3321 base = offset; 3322 } 3323 else 3324 { 3325 // 3326 // In the case of vGPU, pmaGetLargestFree only returns the user-visible 3327 // PMA region and not the reserved/internal regions that constitute the 3328 // overall partition size assigned to the vGPU. 3329 // This is misleading as pMemoryManager->partitionableMemoryRange is expected to 3330 // represent the actual partition size. 3331 // 3332 _memmgrGetFullMIGAddrRange(pGpu, pMemoryManager, &base, &size); 3333 } 3334 3335 *pMemoryRange = rangeMake(base, base + size - 1); 3336 3337 return NV_OK; 3338 } 3339 3340 NV_STATUS 3341 memmgrReserveMemoryForFsp_IMPL 3342 ( 3343 OBJGPU *pGpu, 3344 MemoryManager *pMemoryManager 3345 ) 3346 { 3347 KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu); 3348 3349 // 3350 // If we sent FSP commands to boot ACR, we need to allocate the surfaces 3351 // used by FSP and ACR as WPR/FRTS here from the reserved heap 3352 // 3353 if (pKernelFsp && (!pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_DISABLE_FRTS_VIDMEM) && 3354 (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_BOOT_COMMAND_OK)))) 3355 { 3356 3357 // For GSP-RM flow, we don't need to allocate WPR since it is handled by CPU 3358 if (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM)) 3359 { 3360 return NV_OK; 3361 } 3362 3363 } 3364 return NV_OK; 3365 } 3366 3367 NvU64 3368 memmgrGetVgpuHostRmReservedFb_KERNEL 3369 ( 3370 OBJGPU *pGpu, 3371 MemoryManager *pMemoryManager, 3372 NvU32 vgpuTypeId 3373 ) 3374 { 3375 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3376 NV2080_CTRL_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB_PARAMS params = {0}; 3377 3378 params.vgpuTypeId = vgpuTypeId; 3379 // Send to GSP to get amount of FB reserved for the host 3380 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, 3381 pGpu->hInternalClient, 3382 pGpu->hInternalSubdevice, 3383 NV2080_CTRL_CMD_INTERNAL_MEMMGR_GET_VGPU_CONFIG_HOST_RESERVED_FB, 3384 ¶ms, 3385 sizeof(params))); 3386 return params.hostReservedFb; 3387 } 3388 3389 /*! 3390 * @brief Memory Manager State post load 3391 * 3392 * @param[in] pGpu GPU pointer 3393 * @param[in/out] pMemoryManager MemoryManager pointer 3394 * @param[in] flags State transition flags 3395 * 3396 * @returns On success, returns NV_OK. 3397 * On failure, returns error code. 3398 */ 3399 NV_STATUS 3400 memmgrStatePostLoad_IMPL 3401 ( 3402 OBJGPU *pGpu, 3403 MemoryManager *pMemoryManager, 3404 NvU32 flags 3405 ) 3406 { 3407 if (memmgrIsLocalEgmSupported(pMemoryManager)) 3408 { 3409 NvU64 egmPhysAddr, egmSize; 3410 NvS32 egmNodeId; 3411 NvU32 data32; 3412 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 3413 3414 pMemoryManager->localEgmNodeId = -1; 3415 if (gpuIsSelfHosted(pGpu) && 3416 pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP) && // EGM can be enabled only in C2C path. 3417 (osGetEgmInfo(pGpu, &egmPhysAddr, &egmSize, &egmNodeId) == NV_OK) && 3418 (egmSize != 0)) 3419 { 3420 pMemoryManager->localEgmBasePhysAddr = egmPhysAddr; 3421 pMemoryManager->localEgmSize = egmSize; 3422 pMemoryManager->localEgmNodeId = egmNodeId; 3423 // 3424 // Using fixed Peer ID 7 for local EGM so that vGPU 3425 // migration doesn't fail because of peer id conflict in 3426 // the new host system. 3427 // 3428 pMemoryManager->localEgmPeerId = 7; 3429 pMemoryManager->bLocalEgmEnabled = NV_TRUE; 3430 } 3431 3432 // 3433 // regkey can override the production flow values. 3434 // Note that this could cause an issue with vGPU migration 3435 // if one host system uses regkey to override the EGM peer id 3436 // and other host system doesn't. 3437 // 3438 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_ENABLE_LOCAL_EGM_PEER_ID, &data32) == NV_OK) 3439 { 3440 pMemoryManager->bLocalEgmEnabled = NV_TRUE; 3441 pMemoryManager->localEgmPeerId = data32; 3442 } 3443 } 3444 3445 // 3446 // Reserve the peerID used for local EGM so that the peerID isn't 3447 // resused for other peer Gpus. 3448 // 3449 if (memmgrIsLocalEgmEnabled(pMemoryManager)) 3450 { 3451 if (kbusReserveP2PPeerIds_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), NVBIT(pMemoryManager->localEgmPeerId)) == NV_OK) 3452 { 3453 { 3454 NV2080_CTRL_INTERNAL_HSHUB_EGM_CONFIG_PARAMS params = { 0 }; 3455 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 3456 NV_STATUS status; 3457 3458 params.egmPeerId = pMemoryManager->localEgmPeerId; 3459 3460 // Call physical HSHUB to program the EGM PeerId settings. 3461 3462 status = pRmApi->Control(pRmApi, 3463 pGpu->hInternalClient, 3464 pGpu->hInternalSubdevice, 3465 NV2080_CTRL_CMD_INTERNAL_HSHUB_EGM_CONFIG, 3466 ¶ms, 3467 sizeof(params)); 3468 if (status != NV_OK) 3469 { 3470 NV_PRINTF(LEVEL_ERROR, "HSHUB programming failed for EGM Peer ID: %u\n", 3471 pMemoryManager->localEgmPeerId); 3472 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 3473 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3474 return status; 3475 } 3476 } 3477 } 3478 else 3479 { 3480 NV_PRINTF(LEVEL_ERROR, 3481 "Peer ID specified for local EGM already in use!\n"); 3482 pMemoryManager->bLocalEgmEnabled = NV_FALSE; 3483 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3484 return NV_ERR_INVALID_ARGUMENT; 3485 } 3486 } 3487 else 3488 { 3489 pMemoryManager->localEgmPeerId = BUS_INVALID_PEER; 3490 } 3491 return NV_OK; 3492 } 3493 3494 NV_STATUS 3495 memmgrInitCeUtils_IMPL 3496 ( 3497 MemoryManager *pMemoryManager, 3498 NvBool bFifoLite 3499 ) 3500 { 3501 NV0050_ALLOCATION_PARAMETERS ceUtilsParams = {0}; 3502 3503 NV_ASSERT_OR_RETURN(pMemoryManager->pCeUtils == NULL, NV_ERR_INVALID_STATE); 3504 3505 ceUtilsParams.flags = bFifoLite ? DRF_DEF(0050_CEUTILS, _FLAGS, _FIFO_LITE, _TRUE) : 0; 3506 return objCreate(&pMemoryManager->pCeUtils, pMemoryManager, CeUtils, ENG_GET_GPU(pMemoryManager), NULL, &ceUtilsParams); 3507 } 3508 3509 void 3510 memmgrDestroyCeUtils_IMPL 3511 ( 3512 MemoryManager *pMemoryManager 3513 ) 3514 { 3515 objDelete(pMemoryManager->pCeUtils); 3516 pMemoryManager->pCeUtils = NULL; 3517 } 3518