1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @file 26 * @brief Standard local frame buffer allocation and management routines 27 */ 28 29 #include "os/os.h" 30 #include "gpu/gpu.h" 31 #include "gpu/mem_mgr/mem_mgr.h" 32 #include "gpu/mem_mgr/heap.h" 33 #include "gpu/mem_sys/kern_mem_sys.h" 34 #include "mem_mgr/video_mem.h" 35 #include "mem_mgr/vaspace.h" 36 #include "mem_mgr/system_mem.h" 37 #include "gpu/mem_mgr/mem_utils.h" 38 #include "gpu/mem_mgr/virt_mem_allocator.h" 39 #include "gpu/mem_mgr/mem_desc.h" 40 #include "gpu_mgr/gpu_mgr.h" 41 #include "core/locks.h" 42 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 43 #include "vgpu/rpc.h" 44 #include "gpu/mmu/kern_gmmu.h" 45 #include "virtualization/hypervisor/hypervisor.h" 46 #include "gpu/device/device.h" 47 #include "kernel/gpu/intr/intr.h" 48 49 typedef enum 50 { 51 BLOCK_ADD, 52 BLOCK_REMOVE, 53 BLOCK_SIZE_CHANGED, 54 BLOCK_FREE_STATE_CHANGED, 55 } BlockAction; 56 57 // 58 // Statics 59 // 60 static NV_STATUS _heapBlockFree(OBJGPU *, Heap *, NvHandle, NvHandle, MEM_BLOCK *); 61 static void _heapSetTexturePlacement(Heap *, NvU32, NvU32, NvBool*, 62 NvU32*, NvU8*); 63 static NV_STATUS _heapGetMaxFree(Heap *, NvU64 *, NvU64 *); 64 static NV_STATUS _heapGetBankPlacement(OBJGPU *, Heap *, NvU32, 65 NvU32 *, NvU32, NvU32, NvU32 *); 66 static MEM_BLOCK *_heapFindAlignedBlockWithOwner(OBJGPU *, Heap *, NvU32, 67 NvU64/* aligned*/); 68 static NV_STATUS _heapProcessFreeBlock(OBJGPU *, MEM_BLOCK *, MEM_BLOCK **, 69 MEM_BLOCK **, Heap *, 70 MEMORY_ALLOCATION_REQUEST *, 71 NvHandle, OBJHEAP_ALLOC_DATA *, 72 FB_ALLOC_INFO *, NvU64, NvU64 *); 73 static void _heapAddBlockToNoncontigList(Heap *, MEM_BLOCK *); 74 static void _heapRemoveBlockFromNoncontigList(Heap *, MEM_BLOCK *); 75 static NV_STATUS _heapFindBlockByOffset(OBJGPU *, Heap *, NvU32, 76 MEMORY_DESCRIPTOR *, NvU64, 77 MEM_BLOCK **); 78 static NV_STATUS _heapAllocNoncontig(OBJGPU *, NvHandle, Heap *, 79 MEMORY_ALLOCATION_REQUEST *, NvHandle, 80 OBJHEAP_ALLOC_DATA *, FB_ALLOC_INFO *, 81 NvU32, NvU64, NvU64 *, MEMORY_DESCRIPTOR *, 82 HWRESOURCE_INFO **); 83 static NV_STATUS _heapUpdate(Heap *, MEM_BLOCK *, BlockAction); 84 static void _heapAdjustFree(Heap *pHeap, NvS64 blockSize, NvBool internalHeap); 85 static void _heapBlacklistChunksInFreeBlocks(OBJGPU *, Heap *); 86 87 #ifdef DEBUG 88 89 /****************************************************************************/ 90 /* */ 91 /* DEBUG support! */ 92 /* */ 93 /****************************************************************************/ 94 95 NvU32 dbgDumpHeap = 0; 96 NvU32 dbgReverseDumpHeap = 0; 97 98 static void _heapDump(Heap *); 99 static void _heapValidate(Heap *); 100 101 #define HEAP_VALIDATE(h) {_heapValidate(h);if(dbgDumpHeap)_heapDump(h);} 102 103 static void ConvertOwnerToString(NvU32 owner, char *string) 104 { 105 int i; 106 string[0] = (unsigned char)((owner >> 24)); 107 string[1] = (unsigned char)((owner >> 16) & 0xFF); 108 string[2] = (unsigned char)((owner >> 8) & 0xFF); 109 string[3] = (unsigned char)((owner ) & 0xFF); 110 string[4] = 0; 111 for (i=0 ;i<4; i++) { 112 // Assuming ASCII these should be "safe" printable characters. 113 if ((string[i] < ' ') || (string[i] > 0x7E)) { 114 string[i] = '?'; 115 } 116 } 117 } 118 119 static void _heapDump 120 ( 121 Heap *pHeap 122 ) 123 { 124 NvU64 free; 125 MEM_BLOCK *pBlock; 126 char ownerString[5]; 127 128 if (!pHeap) return; 129 130 NV_PRINTF(LEVEL_INFO, "Heap dump. Size = 0x%08llx\n", pHeap->total); 131 NV_PRINTF(LEVEL_INFO, " Free = 0x%08llx\n", pHeap->free); 132 NV_PRINTF(LEVEL_INFO, " Reserved = 0x%08llx\n", pHeap->reserved); 133 NV_PRINTF(LEVEL_INFO, 134 "=================================================================\n"); 135 NV_PRINTF(LEVEL_INFO, 136 "\t\t Begin End Size \t Type ResId Owner" 137 " \"owns\"\n"); 138 NV_PRINTF(LEVEL_INFO, "Block List %s\n", 139 dbgReverseDumpHeap ? "Reverse" : "Forward"); 140 pBlock = pHeap->pBlockList; 141 do 142 { 143 if ( dbgReverseDumpHeap ) 144 pBlock = pBlock->prev; 145 146 NV_PRINTF(LEVEL_INFO, "\t\t0x%08llx 0x%08llx 0x%08llx", pBlock->begin, 147 pBlock->end, 1 + (pBlock->end - pBlock->begin)); 148 149 if (pBlock->owner == NVOS32_BLOCK_TYPE_FREE) { 150 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\tFREE\n"); 151 } 152 else 153 { 154 ConvertOwnerToString(pBlock->owner, ownerString); 155 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, 156 "\t0x%04x 0x%08x \"%s\"\n", pBlock->u0.type, 157 pBlock->owner, ownerString); 158 } 159 160 if ( !dbgReverseDumpHeap ) 161 pBlock = pBlock->next; 162 } while (pBlock != pHeap->pBlockList); 163 164 NV_PRINTF(LEVEL_INFO, "FREE Block List %s\n", 165 dbgReverseDumpHeap ? "Reverse" : "Forward"); 166 free = 0; 167 pBlock = pHeap->pFreeBlockList; 168 if (pBlock) 169 do 170 { 171 if ( dbgReverseDumpHeap ) 172 pBlock = pBlock->u0.prevFree; 173 174 NV_PRINTF(LEVEL_INFO, "\t\t0x%08llx 0x%08llx 0x%08llx\tFREE\n", 175 pBlock->begin, pBlock->end, 176 1 + (pBlock->end - pBlock->begin)); 177 178 free += pBlock->end - pBlock->begin + 1; 179 180 if ( !dbgReverseDumpHeap ) 181 pBlock = pBlock->u1.nextFree; 182 } while (pBlock != pHeap->pFreeBlockList); 183 184 NV_PRINTF(LEVEL_INFO, "\tCalculated free count = 0x%08llx\n", free); 185 } 186 187 static void _heapValidate 188 ( 189 Heap *pHeap 190 ) 191 { 192 MEM_BLOCK *pBlock, *pBlockFree; 193 NvU64 free, used; 194 195 if (!pHeap) return; 196 197 /* 198 * Scan the blocks and check for consistency. 199 */ 200 free = 0; 201 used = 0; 202 pBlock = pHeap->pBlockList; 203 pBlockFree = pHeap->pFreeBlockList; 204 do 205 { 206 if (pBlock->owner == NVOS32_BLOCK_TYPE_FREE) 207 { 208 if (!pBlockFree) 209 { 210 NV_PRINTF(LEVEL_ERROR, 211 "Invalid free list with free blocks found.\n"); 212 _heapDump(pHeap); 213 DBG_BREAKPOINT(); 214 } 215 free += pBlock->end - pBlock->begin + 1; 216 if (pBlock != pBlockFree) 217 { 218 NV_PRINTF(LEVEL_ERROR, 219 "Free list not consistent with block list.\n"); 220 _heapDump(pHeap); 221 DBG_BREAKPOINT(); 222 } 223 pBlockFree = pBlockFree->u1.nextFree; 224 } 225 else 226 { 227 used += pBlock->end - pBlock->begin + 1; 228 } 229 if (pBlock->next != pHeap->pBlockList) 230 { 231 if (pBlock->end != pBlock->next->begin - 1) 232 { 233 NV_PRINTF(LEVEL_ERROR, 234 "Hole between blocks at offset = 0x%llx\n", 235 pBlock->end); 236 _heapDump(pHeap); 237 DBG_BREAKPOINT(); 238 } 239 } 240 else 241 { 242 if (pBlock->end != pHeap->base + pHeap->total - 1) 243 { 244 NV_PRINTF(LEVEL_ERROR, "Last block doesn't end at top.\n"); 245 _heapDump(pHeap); 246 DBG_BREAKPOINT(); 247 } 248 if (pBlock->next->begin != pHeap->base) 249 { 250 NV_PRINTF(LEVEL_ERROR, 251 "First block doesn't start at bottom.\n"); 252 _heapDump(pHeap); 253 DBG_BREAKPOINT(); 254 } 255 } 256 if (pBlock->end < pBlock->begin) 257 { 258 NV_PRINTF(LEVEL_ERROR, 259 "Validate: Invalid block begin = 0x%08llx\n", 260 pBlock->begin); 261 NV_PRINTF(LEVEL_ERROR, 262 " end = 0x%08llx\n", 263 pBlock->end); 264 _heapDump(pHeap); 265 DBG_BREAKPOINT(); 266 } 267 pBlock = pBlock->next; 268 } while (pBlock != pHeap->pBlockList); 269 if (free != pHeap->free) 270 { 271 NV_PRINTF(LEVEL_ERROR, 272 "Calculated free count (%llx) not consistent with heap free count (%llx).\n", 273 free, pHeap->free); 274 _heapDump(pHeap); 275 DBG_BREAKPOINT(); 276 } 277 if ((used + free) > pHeap->total) 278 { 279 NV_PRINTF(LEVEL_ERROR, 280 "Calculated used count (%llx) not consistent with heap size (%llx).\n", 281 used + free, pHeap->total); 282 _heapDump(pHeap); 283 DBG_BREAKPOINT(); 284 } 285 } 286 #else 287 #define HEAP_VALIDATE(h) 288 #endif // DEBUG 289 290 291 /****************************************************************************/ 292 /* */ 293 /* Heap Manager */ 294 /* */ 295 /****************************************************************************/ 296 297 static NV_STATUS heapReserveRegion 298 ( 299 MemoryManager *pMemoryManager, 300 Heap *pHeap, 301 NvU64 offset, 302 NvU64 size, 303 MEMORY_DESCRIPTOR **ppMemDesc, 304 NvBool isRmRsvdRegion, 305 NvBool bProtected 306 ) 307 { 308 NV_STATUS rmStatus = NV_OK; 309 OBJGPU *pGpu = ENG_GET_GPU(pMemoryManager); 310 NvU64 heapSize = (pHeap->base + pHeap->total); 311 FB_ALLOC_INFO *pFbAllocInfo = NULL; 312 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 313 314 MEMORY_ALLOCATION_REQUEST allocRequest = {0}; 315 NV_MEMORY_ALLOCATION_PARAMS allocData = {0}; 316 317 NvU64 align = 0; 318 NvU32 height = 1; 319 NvU32 pitch = 1; 320 NvU32 attr = DRF_DEF(OS32, _ATTR, _PAGE_SIZE, _4KB) | 321 DRF_DEF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS); 322 NvU32 attr2 = DRF_DEF(OS32, _ATTR2, _INTERNAL, _YES); 323 324 NV_ASSERT_OR_RETURN((offset < heapSize), NV_OK); 325 326 allocRequest.pUserParams = &allocData; 327 328 allocData.owner = ((isRmRsvdRegion) ? HEAP_OWNER_RM_RESERVED_REGION : HEAP_OWNER_PMA_RESERVED_REGION); 329 allocData.height = height; 330 allocData.type = ((isRmRsvdRegion) ? NVOS32_TYPE_RESERVED : NVOS32_TYPE_PMA); 331 allocData.flags = NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE; 332 allocData.attr = attr; 333 allocData.attr2 = attr2; 334 allocData.pitch = pitch; 335 allocData.alignment = align; 336 allocData.size = NV_MIN(size, (heapSize - offset)); 337 allocData.offset = offset; 338 339 if (bProtected) 340 allocData.flags |= NVOS32_ALLOC_FLAGS_PROTECTED; 341 342 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 343 NV_ASSERT_TRUE_OR_GOTO(rmStatus, pFbAllocInfo != NULL, NV_ERR_NO_MEMORY, done); 344 345 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 346 NV_ASSERT_TRUE_OR_GOTO(rmStatus, pFbAllocPageFormat != NULL, NV_ERR_NO_MEMORY, done); 347 348 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 349 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 350 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 351 352 memUtilsInitFBAllocInfo(&allocData, pFbAllocInfo, 0, 0); 353 354 NV_ASSERT_OK_OR_GOTO(rmStatus, 355 memmgrAllocResources(pGpu, pMemoryManager, &allocRequest, pFbAllocInfo), 356 done); 357 358 NV_ASSERT_OK_OR_GOTO(rmStatus, 359 vidmemAllocResources(pGpu, pMemoryManager, &allocRequest, pFbAllocInfo, pHeap), 360 done); 361 362 NV_PRINTF(LEVEL_INFO, "Reserved heap for %s %llx..%llx\n", 363 ((isRmRsvdRegion) ? "RM" : "PMA"), offset, (offset+size-1)); 364 365 *ppMemDesc = allocRequest.pMemDesc; 366 367 // Account for reserved size removed from the total address space size 368 if (isRmRsvdRegion) 369 { 370 pHeap->reserved += allocData.size; 371 } 372 373 done: 374 portMemFree(pFbAllocPageFormat); 375 portMemFree(pFbAllocInfo); 376 377 return rmStatus; 378 } 379 380 /*! 381 * @brief Initializes a heap object 382 * 383 * @param[in] pFb FB object ptr 384 * @param[in/out] pHeap HEAP object ptr 385 * @param[in] base Base for this heap 386 * @param[in] size Size of this heap 387 * @param[in] heapType Heap type (Global or PMSA) 388 * @param[in] pPtr A generic pointer which will be typecasted based on heapType 389 */ 390 NV_STATUS heapInitInternal_IMPL 391 ( 392 OBJGPU *pGpu, 393 Heap *pHeap, 394 NvU64 base, 395 NvU64 size, 396 HEAP_TYPE_INTERNAL heapType, 397 void *pPtr 398 ) 399 { 400 MEM_BLOCK *pBlock; 401 NvU32 i; 402 NV_STATUS status; 403 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 404 NvU32 typeDataSize = 0; 405 FB_REGION_DESCRIPTOR *pFbRegion; 406 MEMORY_DESCRIPTOR *pPmsaMemDesc = NULL; 407 408 // 409 // Simply create a free heap. 410 // 411 NV_PRINTF(LEVEL_INFO, 412 "Heap Manager: HEAP ABOUT TO BE CREATED. (Base: 0x%llx Size: 0x%llx)\n", 413 base, size); 414 415 pHeap->base = base; 416 pHeap->total = size; 417 pHeap->free = size; 418 pHeap->reserved = 0; 419 pHeap->heapType = heapType; 420 421 pHeap->peakInternalUsage = 0; 422 pHeap->peakExternalUsage = 0; 423 pHeap->currInternalUsage = 0; 424 pHeap->currExternalUsage = 0; 425 426 427 // Set the flags based on HEAP type 428 switch (heapType) 429 { 430 case HEAP_TYPE_RM_GLOBAL: 431 pHeap->bHasFbRegions = NV_TRUE; 432 break; 433 case HEAP_TYPE_PHYS_MEM_SUBALLOCATOR: 434 NV_ASSERT(pPtr != NULL); 435 436 pHeap->bHasFbRegions = NV_FALSE; 437 typeDataSize = sizeof(PHYS_MEM_SUBALLOCATOR_DATA); 438 pPmsaMemDesc = ((PHYS_MEM_SUBALLOCATOR_DATA *)pPtr)->pMemDesc; 439 break; 440 case HEAP_TYPE_PARTITION_LOCAL: 441 pHeap->bHasFbRegions = NV_TRUE; 442 break; 443 default: 444 return NV_ERR_INVALID_ARGUMENT; 445 } 446 447 pHeap->pHeapTypeSpecificData = NULL; 448 if ((pPtr != NULL) && (typeDataSize > 0)) 449 { 450 pHeap->pHeapTypeSpecificData = portMemAllocNonPaged(typeDataSize); 451 if (pHeap->pHeapTypeSpecificData == NULL) 452 { 453 return NV_ERR_OPERATING_SYSTEM; 454 } 455 NV_ASSERT(pHeap->pHeapTypeSpecificData != NULL); 456 portMemCopy(pHeap->pHeapTypeSpecificData, typeDataSize, pPtr, typeDataSize); 457 } 458 459 pBlock = portMemAllocNonPaged(sizeof(MEM_BLOCK)); 460 if (pBlock == NULL) 461 { 462 return NV_ERR_OPERATING_SYSTEM; 463 } 464 portMemSet(pBlock, 0, sizeof(MEM_BLOCK)); 465 466 pBlock->owner = NVOS32_BLOCK_TYPE_FREE; 467 pBlock->textureId= 0; 468 pBlock->begin = base; 469 pBlock->align = 0; 470 pBlock->alignPad = 0; 471 pBlock->end = base + size - 1; 472 pBlock->u0.prevFree = pBlock; 473 pBlock->u1.nextFree = pBlock; 474 pBlock->next = pBlock; 475 pBlock->prev = pBlock; 476 pBlock->format = 0; 477 478 pHeap->pBlockList = pBlock; 479 pHeap->pFreeBlockList = pBlock; 480 pHeap->memHandle = 0xcafe0000; 481 pHeap->numBlocks = 1; 482 pHeap->pBlockTree = NULL; 483 484 // 485 // Set the client id as invalid since there isn't one that exists 486 // Initialize the client texture data structure 487 // 488 portMemSet(pHeap->textureData, 0, 489 sizeof(TEX_INFO) * MAX_TEXTURE_CLIENT_IDS); 490 491 // 492 // Call into the hal to get bank placement policy. Note this will vary chip to chip, but let's allow the HAL to tell us 493 // the implementation details. 494 // 495 status = memmgrGetBankPlacementData_HAL(pGpu, pMemoryManager, pHeap->placementStrategy); 496 if (status != NV_OK) 497 { 498 // 499 // ooops, can't get HAL version of where to place things - let's default to something 500 // 501 NV_PRINTF(LEVEL_ERROR, 502 "Heap Manager unable to get bank placement policy from HAL.\n"); 503 NV_PRINTF(LEVEL_ERROR, 504 "Heap Manager defaulting to BAD placement policy.\n"); 505 506 pHeap->placementStrategy[BANK_PLACEMENT_IMAGE] = ((0) 507 | BANK_MEM_GROW_UP 508 | MEM_GROW_UP 509 | 0xFFFFFF00); 510 pHeap->placementStrategy[BANK_PLACEMENT_DEPTH] = ((0) 511 | BANK_MEM_GROW_DOWN 512 | MEM_GROW_DOWN 513 | 0xFFFFFF00); 514 pHeap->placementStrategy[BANK_PLACEMENT_TEX_OVERLAY_FONT] = ((0) 515 | BANK_MEM_GROW_DOWN 516 | MEM_GROW_DOWN 517 | 0xFFFFFF00); 518 pHeap->placementStrategy[BANK_PLACEMENT_OTHER] = ((0) 519 | BANK_MEM_GROW_DOWN 520 | MEM_GROW_DOWN 521 | 0xFFFFFF00); 522 status = NV_OK; 523 } 524 525 // Setup noncontig list 526 pHeap->pNoncontigFreeBlockList = NULL; 527 528 // insert first block into rb-tree 529 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_ADD)) 530 { 531 return NV_ERR_INVALID_STATE; 532 } 533 534 // 535 // If there are FB regions defined, check to see if any of them are 536 // marked reserved. Tag those regions as reserved in the heap. 537 // 538 if ((pMemoryManager->Ram.numFBRegions > 0) && (pHeap->bHasFbRegions)) 539 { 540 NvBool bConsoleFbRegionContentPreserved; 541 FB_REGION_DESCRIPTOR consoleFbRegion; 542 portMemSet(&consoleFbRegion, 0, sizeof(consoleFbRegion)); 543 544 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 545 { 546 // 547 // If a region of FB is actively being used for console display memory 548 // on this GPU, mark it reserved in-place. 549 // 550 memmgrReserveConsoleRegion_HAL(pGpu, pMemoryManager, &consoleFbRegion); 551 status = memmgrAllocateConsoleRegion_HAL(pGpu, pMemoryManager, &consoleFbRegion); 552 if (status != NV_OK) 553 { 554 NV_PRINTF(LEVEL_WARNING, "Squashing the error status after failing to allocate console region, status: %x\n", 555 status); 556 status = NV_OK; 557 } 558 } 559 560 // 561 // Define PMA-managed regions 562 // This will be moved to memmgr once we refactor SMC partitions 563 // 564 if (memmgrIsPmaEnabled(pMemoryManager) && 565 memmgrIsPmaSupportedOnPlatform(pMemoryManager) && 566 (heapType != HEAP_TYPE_PARTITION_LOCAL)) 567 { 568 memmgrSetPmaInitialized(pMemoryManager, NV_TRUE); 569 memmgrRegionSetupForPma(pGpu, pMemoryManager); 570 } 571 572 bConsoleFbRegionContentPreserved = NV_FALSE; 573 574 if (heapType != HEAP_TYPE_PARTITION_LOCAL) 575 { 576 // For GSP RM, all PMA candidate regions are given to CPU RM for its use 577 if (RMCFG_FEATURE_PLATFORM_GSP) 578 { 579 memmgrRegionSetupForPma(pGpu, pMemoryManager); 580 } 581 582 for (i = 0; i < pMemoryManager->Ram.numFBRegions; i++) 583 { 584 pFbRegion = &pMemoryManager->Ram.fbRegion[i]; 585 586 // If the region is marked reserved, reserve it in the heap 587 if (pFbRegion->bRsvdRegion || 588 ((memmgrIsPmaInitialized(pMemoryManager) || 589 RMCFG_FEATURE_PLATFORM_GSP) && 590 !pFbRegion->bInternalHeap)) 591 { 592 NvU64 fbRegionBase; 593 MEMORY_DESCRIPTOR *pMemDesc = NULL; 594 595 // Skip regions which are outside the heap boundaries 596 if (pFbRegion->base < base && pFbRegion->limit < base) 597 { 598 continue; 599 } 600 601 // TODO: Remove SRIOV check and enable on baremetal as well. 602 if (IS_VIRTUAL_WITH_SRIOV(pGpu) && (pFbRegion->base >= (base + size))) 603 { 604 continue; 605 } 606 607 // Adjust base of reserved region on heap 608 fbRegionBase = NV_MAX(base, pFbRegion->base); 609 610 NV_PRINTF(LEVEL_INFO, "Reserve at %llx of size %llx\n", 611 fbRegionBase, (pFbRegion->limit - fbRegionBase + 1)); 612 613 status = heapReserveRegion( 614 pMemoryManager, 615 pHeap, 616 fbRegionBase, 617 (pFbRegion->limit - fbRegionBase + 1), 618 &pMemDesc, 619 pFbRegion->bRsvdRegion, 620 pFbRegion->bProtected); 621 622 if (status != NV_OK || pMemDesc == NULL) 623 { 624 NV_PRINTF(LEVEL_ERROR, "failed to reserve %llx..%llx\n", 625 pFbRegion->base, pFbRegion->limit); 626 return status; 627 } 628 629 if ((pMemoryManager->Ram.ReservedConsoleDispMemSize > 0) && 630 (pFbRegion->base == consoleFbRegion.base) && (pFbRegion->limit == consoleFbRegion.limit)) 631 { 632 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_LOST_ON_SUSPEND, NV_FALSE); 633 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PRESERVE_CONTENT_ON_SUSPEND, NV_TRUE); 634 635 bConsoleFbRegionContentPreserved = NV_TRUE; 636 } 637 } 638 } 639 640 if ((pMemoryManager->Ram.ReservedConsoleDispMemSize > 0) && 641 !bConsoleFbRegionContentPreserved) 642 { 643 NV_PRINTF(LEVEL_ERROR, 644 "failed to preserve content of console display memory\n"); 645 } 646 } 647 648 #ifdef DEBUG 649 _heapDump(pHeap); 650 #endif 651 } //if ((pMemoryManager->Ram.numFBRegions > 0) && (pHeap->bHasFbRegions)) 652 653 // Hand over all the memory of partition-heap to partition-PMA 654 if ((heapType == HEAP_TYPE_PARTITION_LOCAL) && 655 (memmgrIsPmaInitialized(pMemoryManager))) 656 { 657 MEMORY_DESCRIPTOR *pMemDesc = NULL; 658 NvBool bProtected = NV_FALSE; 659 660 bProtected = gpuIsCCFeatureEnabled(pGpu); 661 status = heapReserveRegion( 662 pMemoryManager, 663 pHeap, 664 base, 665 size, 666 &pMemDesc, 667 NV_FALSE, 668 bProtected); 669 670 if (status != NV_OK || pMemDesc == NULL) 671 { 672 NV_PRINTF(LEVEL_ERROR, "failed to reserve %llx..%llx\n", base, 673 base + size - 1); 674 675 return status; 676 } 677 } 678 679 // If PHYS_MEM_SUBALLOCATOR, increase its refCount 680 if ((status == NV_OK) && (pPmsaMemDesc != NULL)) 681 { 682 memdescAddRef(pPmsaMemDesc); 683 } 684 685 return (status); 686 } 687 688 void 689 heapDestruct_IMPL 690 ( 691 Heap *pHeap 692 ) 693 { 694 MEM_BLOCK *pBlock, *pBlockFirst, *pBlockNext; 695 OBJGPU *pGpu = ENG_GET_GPU(pHeap); 696 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 697 NvBool headptr_updated; 698 MEMORY_DESCRIPTOR *pPmsaMemDesc = NULL; 699 700 NV_PRINTF(LEVEL_INFO, "Heap Manager: HEAP ABOUT TO BE DESTROYED.\n"); 701 702 #ifdef DEBUG 703 _heapDump(pHeap); 704 #endif 705 706 // Free all blacklisted pages 707 if (pHeap->blackListAddresses.count != 0) 708 { 709 heapFreeBlackListedPages(pGpu, pHeap); 710 } 711 712 // 713 // Free all allocated blocks, but preserve primary surfaces. 714 // If the head of our list changes, restart the search, since our terminating 715 // block pointer may not be in the list anymore. 716 // 717 do 718 { 719 pBlock = pBlockFirst = pHeap->pBlockList; 720 if (pBlock == NULL) 721 { 722 break; 723 } 724 725 headptr_updated = NV_FALSE; 726 727 do 728 { 729 pBlockNext = pBlock->next; 730 731 // If we are freeing the reserved region created at heapInit, free the memory descriptor too 732 if ((pBlock->allocedMemDesc) && ((pBlock->owner == HEAP_OWNER_RM_RESERVED_REGION) || 733 (pBlock->owner == HEAP_OWNER_PMA_RESERVED_REGION))) 734 { 735 memdescDestroy(pBlock->pMemDesc); 736 pBlock->pMemDesc = NULL; 737 pBlock->allocedMemDesc = NV_FALSE; 738 } 739 740 _heapBlockFree(pGpu, pHeap, NV01_NULL_OBJECT, NV01_NULL_OBJECT, pBlock); 741 742 // restart scanning the list, if the heap->pBlockList changed 743 if (pBlockFirst != pHeap->pBlockList) 744 { 745 headptr_updated = NV_TRUE; 746 break; 747 } 748 749 pBlock = pBlockNext; 750 751 } while (pBlock != pHeap->pBlockList); 752 753 } while (headptr_updated); 754 755 // 756 // Now that the console region is no longer reserved, free the console 757 // memdesc. 758 // 759 if (pHeap->heapType != HEAP_TYPE_PARTITION_LOCAL) 760 memmgrReleaseConsoleRegion(pGpu, pMemoryManager); 761 762 // 763 // Free the heap structure, if we freed everything 764 // (the first block represents the entire free space of the heap). 765 // this is only done if the "internal" interface is used. 766 // heapDestroy is an exported function now to user/display driver land, 767 // and we don't want the heap structures being freed unless we've been 768 // called from RM-land during a STATE_DESTROY 769 // 770 if ((pHeap->pBlockList != NULL) && 771 (pHeap->pBlockList->begin == pHeap->base) && 772 (pHeap->pBlockList->end == (pHeap->base + pHeap->total - 1))) 773 { 774 portMemFree(pHeap->pBlockList); 775 } 776 777 // Free the type specific data allocated 778 if (pHeap->pHeapTypeSpecificData != NULL) 779 { 780 if (pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR) 781 { 782 pPmsaMemDesc = ((PHYS_MEM_SUBALLOCATOR_DATA *)(pHeap->pHeapTypeSpecificData))->pMemDesc; 783 memdescDestroy(pPmsaMemDesc); 784 } 785 portMemFree(pHeap->pHeapTypeSpecificData); 786 pHeap->pHeapTypeSpecificData = NULL; 787 } 788 789 if ((pHeap->bHasFbRegions) && (memmgrIsPmaInitialized(pMemoryManager))) 790 { 791 if (pHeap->heapType != HEAP_TYPE_PARTITION_LOCAL) 792 memmgrSetPmaInitialized(pMemoryManager, NV_FALSE); 793 794 pmaDestroy(&pHeap->pmaObject); 795 portMemSet(&pHeap->pmaObject, 0, sizeof(pHeap->pmaObject)); 796 } 797 } 798 799 static NV_STATUS _heapGetBankPlacement 800 ( 801 OBJGPU *pGpu, 802 Heap *pHeap, 803 NvU32 owner, 804 NvU32 *flags, 805 NvU32 type, 806 NvU32 bank, 807 NvU32 *placement 808 ) 809 { 810 NvU32 bankPlacement, i; 811 812 if (type != NVOS32_TYPE_PRIMARY) 813 { 814 NvU32 bankPlacementType; 815 816 // what kind of allocation is it? 817 switch (type) 818 { 819 case NVOS32_TYPE_IMAGE: 820 case NVOS32_TYPE_NOTIFIER: 821 bankPlacementType = BANK_PLACEMENT_IMAGE; 822 break; 823 case NVOS32_TYPE_DEPTH: 824 case NVOS32_TYPE_ZCULL: 825 case NVOS32_TYPE_STENCIL: 826 bankPlacementType = BANK_PLACEMENT_DEPTH; 827 break; 828 case NVOS32_TYPE_TEXTURE: 829 case NVOS32_TYPE_VIDEO: 830 case NVOS32_TYPE_FONT: 831 bankPlacementType = BANK_PLACEMENT_TEX_OVERLAY_FONT; 832 break; 833 default: 834 bankPlacementType = BANK_PLACEMENT_OTHER; 835 } 836 837 // 838 // NV50+ doesn't care about bank placement since the fb has bank 839 // striding and we dont need to care about allocating primary surfaces 840 // in special areas to avoid bank conflicts. This strategy management 841 // should be removed in the future. 842 // 843 bankPlacement = pHeap->placementStrategy[bankPlacementType]; 844 } 845 else 846 { 847 // 848 // primary allocation, default grow direction is up, starting at bank 0 849 // Can be overridden with NVOS32_ALLOC_FLAGS_FORCE_MEM_* 850 // 851 bankPlacement = ((0) 852 | BANK_MEM_GROW_UP 853 | MEM_GROW_UP 854 | 0xFFFFFF00); 855 } 856 857 // 858 // check if bank placement force was passed in - hint is handled in the first loop below 859 // 860 if (*flags & NVOS32_ALLOC_FLAGS_BANK_FORCE) 861 { 862 // replace data in bankplacement 863 if (*flags & NVOS32_ALLOC_FLAGS_BANK_GROW_DOWN) 864 bankPlacement = bank | BANK_MEM_GROW_DOWN | 0xFFFFFF00; 865 else 866 bankPlacement = bank | BANK_MEM_GROW_UP | 0xFFFFFF00; 867 *flags &= ~(NVOS32_ALLOC_FLAGS_BANK_HINT); // remove hint flag 868 } 869 870 // 871 // Check if FORCE_MEM_GROWS_UP or FORCE_MEM_GROWS_DOWN was passed in 872 // to override the MEM_GROWS direction for this allocation. Make sure 873 // to override each of the first MEM_NUM_BANKS_TO_TRY bytes in the NvU32 874 // 875 if (*flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_UP) 876 { 877 *flags |= NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT; 878 for (i = 0; i < MEM_NUM_BANKS_TO_TRY; i++) 879 { 880 bankPlacement = (bankPlacement & ~(MEM_GROW_MASK << (i*MEM_BANK_DATA_SIZE))) | 881 (MEM_GROW_UP << (i*MEM_BANK_DATA_SIZE)); 882 } 883 } 884 if (*flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN) 885 { 886 *flags |= NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT; 887 for (i = 0; i < MEM_NUM_BANKS_TO_TRY; i++) 888 { 889 bankPlacement = (bankPlacement & ~(MEM_GROW_MASK << (i*MEM_BANK_DATA_SIZE))) | 890 (MEM_GROW_DOWN << (i*MEM_BANK_DATA_SIZE)); 891 } 892 } 893 894 // return the bank placement to use 895 *placement = bankPlacement; 896 return (NV_OK); 897 } 898 899 // 900 // Workaround for Bug 67690: 901 // NV28M-WinXP: (Lindbergh) StencilFloor OpenGL Sample Locks Up when Maximized on Secondary DualView Display 902 // 903 // Change heap placement for textures if more than two clients 904 // are detected. In the case of two or more clients, ignoreBankPlacement, textureClientIndex, 905 // and currentBankInfo are modified. IgnoreBankPlacement flag is set to true, textureClientIndex 906 // is returned with the index of the client to be used as heap->textureData[textureClientIndex] 907 // which pertains to the current client. Lastly, currentBankInfo is modified to grow in the 908 // opposite direction of the most recently allocated client. 909 // 910 static void _heapSetTexturePlacement 911 ( 912 Heap *pHeap, 913 NvU32 client, 914 NvU32 type, 915 NvBool *ignoreBankPlacement, 916 NvU32 *textureClientIndex, 917 NvU8 *currentBankInfo 918 ) 919 { 920 NvU32 index, numClients, clientFound, mostRecentIndex; 921 mostRecentIndex = 0xFFFFFFFF; 922 clientFound = NV_FALSE; 923 numClients = 0; 924 925 // 926 // let's first check to see if the client is already registered 927 // We will iterate thru to find number of clients 928 // 929 for (index = 0; index < MAX_TEXTURE_CLIENT_IDS; index++) 930 { 931 // client already registered 932 if (pHeap->textureData[index].clientId == client) 933 { 934 // give the currentBankInfo the new flags 935 *currentBankInfo = pHeap->textureData[index].placementFlags; 936 // 937 // Set the client as found so that we skip allocation 938 // of the client in the texture data structure 939 // 940 clientFound = NV_TRUE; 941 *textureClientIndex = index; 942 } 943 944 // 945 // We loop through the whole structure to determine the 946 // number of texture clients currently listed 947 // 948 if (pHeap->textureData[index].clientId != 0) 949 numClients++; 950 951 // 952 // This is used to assign new textures to the buffer 953 // A value of 0xFFFFFFFF indicates that this is the first allocation 954 // 955 if (pHeap->textureData[index].mostRecentAllocatedFlag == NV_TRUE) 956 mostRecentIndex = index; 957 } 958 959 // 960 // If more than one clinet is detected, ignore bank placement 961 // otherwise, defaults to bank placement 962 // 963 if (numClients > 1) 964 *ignoreBankPlacement = NV_TRUE; 965 966 // 967 // We fall into this if statement if no client was listed 968 // or if we have exceeded the allowable clients available 969 // 970 if (clientFound == NV_FALSE) 971 { 972 index = 0; 973 while (clientFound == NV_FALSE) 974 { 975 // the case of full texture buffer of clients, greater than 4 clients 976 if (index == MAX_TEXTURE_CLIENT_IDS) 977 { 978 index = (mostRecentIndex + 1) % MAX_TEXTURE_CLIENT_IDS; 979 980 // assign the new client and update the texture data 981 pHeap->textureData[index].clientId = client; 982 pHeap->textureData[index].mostRecentAllocatedFlag = NV_TRUE; 983 pHeap->textureData[mostRecentIndex].mostRecentAllocatedFlag = NV_FALSE; 984 pHeap->textureData[index].refCount = 0; 985 986 // 987 // Reverse the placementFlags from the one that was previously allocated 988 // 989 if (pHeap->textureData[mostRecentIndex].placementFlags & MEM_GROW_MASK) 990 *currentBankInfo = MEM_GROW_UP; 991 else 992 *currentBankInfo = MEM_GROW_DOWN; 993 994 // Assign the new value to the texture data structure 995 pHeap->textureData[index].placementFlags = *currentBankInfo; 996 clientFound = NV_TRUE; 997 *ignoreBankPlacement = NV_TRUE; 998 *textureClientIndex = index; 999 } 1000 1001 // the case in which there is still room available in the buffer 1002 if (pHeap->textureData[index].clientId == 0) 1003 { 1004 // If we fall in here, it means there is still room available 1005 pHeap->textureData[index].clientId = client; 1006 1007 // deal with the grow directivity 1008 if (mostRecentIndex == 0xFFFFFFFF) 1009 { 1010 // this is the very first client to be allocated 1011 pHeap->textureData[index].placementFlags = *currentBankInfo; 1012 if (pHeap->textureData[index].placementFlags & MEM_GROW_MASK) 1013 *currentBankInfo = MEM_GROW_DOWN; 1014 else 1015 *currentBankInfo = MEM_GROW_UP; 1016 pHeap->textureData[index].mostRecentAllocatedFlag = NV_TRUE; 1017 } 1018 else 1019 { 1020 if (pHeap->textureData[mostRecentIndex].placementFlags & MEM_GROW_MASK) 1021 *currentBankInfo = MEM_GROW_UP; 1022 else 1023 *currentBankInfo = MEM_GROW_DOWN; 1024 1025 // Set the last client allocated to the new client allocated 1026 pHeap->textureData[mostRecentIndex].mostRecentAllocatedFlag = NV_FALSE; 1027 pHeap->textureData[index].mostRecentAllocatedFlag = NV_TRUE; 1028 1029 // update the placement flags 1030 pHeap->textureData[index].placementFlags = *currentBankInfo; 1031 1032 // if this isn't the first client in the heap, then we ignore bank placement 1033 *ignoreBankPlacement = NV_TRUE; 1034 } 1035 1036 clientFound = NV_TRUE; 1037 *textureClientIndex = index; 1038 } 1039 index++; 1040 } // while (clientFound == NV_FALSE) 1041 } // if (clientFound == NV_FALSE) 1042 } 1043 1044 // 1045 // If we have two different alignment requirements for a memory 1046 // allocation, this routine calculates the LCM (least common multiple) 1047 // to satisfy both requirements. 1048 // 1049 // An alignment of 0 means "no preferred alignment". The return value 1050 // will not exceed maxAlignment = NV_U64_MAX; it returns maxAlignment if the limit 1051 // is exceeded. 1052 // 1053 // Called by heapAlloc and heapAllocHint. 1054 // 1055 1056 1057 /*! 1058 * @Is Alloc Valid For FB Region 1059 * 1060 * Check the prospective allocation to see if the candidate block supports 1061 * the requested surface type. 1062 * 1063 * NOTE: The FB region and FB heap allocation code assume that free blocks 1064 * reside in a single FB region. This is true in current implementations that 1065 * have the regions separated by a reserved block, but may not be true in future 1066 * implementations. 1067 * 1068 * @param[in] pGpu GPU object 1069 * @param[in] pHeap heap object 1070 * @param[in] pFbAllocInfo allocation request information 1071 * @param[in] pAllocData allocation candidate information 1072 * 1073 * @returns NV_TRUE if block can be allocated at the prospective address 1074 * 1075 */ 1076 static NvBool 1077 _isAllocValidForFBRegion 1078 ( 1079 OBJGPU *pGpu, 1080 Heap *pHeap, 1081 FB_ALLOC_INFO *pFbAllocInfo, 1082 OBJHEAP_ALLOC_DATA *pAllocData 1083 ) 1084 { 1085 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1086 NvBool isValid = NV_FALSE; 1087 FB_REGION_DESCRIPTOR *fbRegion; 1088 1089 // Check if any regions are defined. If not, then we are done. 1090 if (pMemoryManager->Ram.numFBRegions > 0) 1091 { 1092 fbRegion = memmgrLookupFbRegionByOffset(pGpu, pMemoryManager, pAllocData->allocLo, pAllocData->allocHi); 1093 1094 if (fbRegion != NULL) 1095 { 1096 // Because we heapAlloc the reserved region. 1097 if (pFbAllocInfo->pageFormat->type == NVOS32_TYPE_PMA && 1098 pFbAllocInfo->owner == HEAP_OWNER_PMA_RESERVED_REGION) 1099 { 1100 if (!fbRegion->bInternalHeap && !fbRegion->bRsvdRegion) 1101 { 1102 isValid = NV_TRUE; 1103 } 1104 return isValid; 1105 } 1106 // Check if the region is reserved/not usable 1107 if (fbRegion->bRsvdRegion && 1108 (pFbAllocInfo->pageFormat->type != NVOS32_TYPE_RESERVED)) 1109 { 1110 NV_PRINTF(LEVEL_INFO, 1111 "Reserved region. Rejecting placement\n"); 1112 return NV_FALSE; 1113 } 1114 1115 // 1116 // Check if the region supports compression and if we need it. 1117 // Surfaces that *require* compression can be allocated *only* in 1118 // regions that support compression. *Optionally* compressed surfaces 1119 // can be allocated anywhere though -- the selection of an uncompressed 1120 // KIND will be handled in dmaUpdateVASpace. 1121 // 1122 if (!fbRegion->bSupportCompressed) 1123 { 1124 if (DRF_VAL(OS32, _ATTR, _COMPR , pFbAllocInfo->pageFormat->attr) == NVOS32_ATTR_COMPR_REQUIRED) 1125 { 1126 NV_PRINTF(LEVEL_INFO, 1127 "Compression not supported. Rejecting placement\n"); 1128 return NV_FALSE; 1129 } 1130 } 1131 1132 // Check if the allocation type is specifically not allowed 1133 if (pFbAllocInfo->pageFormat->type < NVOS32_NUM_MEM_TYPES) 1134 { 1135 if ((!fbRegion->bSupportISO) && 1136 ((pFbAllocInfo->pageFormat->type == NVOS32_TYPE_PRIMARY) || 1137 (pFbAllocInfo->pageFormat->type == NVOS32_TYPE_CURSOR) || 1138 (pFbAllocInfo->pageFormat->type == NVOS32_TYPE_VIDEO))) 1139 { 1140 NV_PRINTF(LEVEL_INFO, 1141 "ISO surface type #%d not supported. Rejecting placement\n", 1142 pFbAllocInfo->pageFormat->type); 1143 return NV_FALSE; 1144 } 1145 } 1146 1147 if (!!fbRegion->bProtected ^ 1148 !!(pFbAllocInfo->pageFormat->flags & NVOS32_ALLOC_FLAGS_PROTECTED)) 1149 { 1150 NV_PRINTF(LEVEL_INFO, 1151 "Protection mismatch. Rejecting placement\n"); 1152 return NV_FALSE; 1153 } 1154 1155 } 1156 else if (pFbAllocInfo->pageFormat->type != NVOS32_TYPE_RESERVED) 1157 { 1158 // 1159 // Allow reserved allocs outside of valid regions, but everything else 1160 // must be allocated in a region. 1161 // 1162 NV_PRINTF(LEVEL_INFO, 1163 "pFbAllocInfo->type != NVOS32_TYPE_RESERVED\n"); 1164 return NV_FALSE; 1165 } 1166 1167 } 1168 1169 return NV_TRUE; 1170 } 1171 1172 /** 1173 * Blacklists a single page 1174 * This function will allocate the memory descriptor with a fixed memory offset 1175 * and allocate the FB physical offset. Will replace the blacklist allocation 1176 * path in the heapBlackListPages_IMPL. 1177 * 1178 * @param[in] pGpu OBJGPU pointer 1179 * @param[in] pHeap Heap pointer 1180 * @param[in] pBlacklistChunk BLACKLIST_CHUNK pointer 1181 * 1182 * @returns NV_OK on success 1183 * NV_ERR_OUT_OF_MEMORY, if the memory is already blacklisted 1184 */ 1185 1186 static NV_STATUS 1187 _heapBlacklistSingleChunk 1188 ( 1189 OBJGPU *pGpu, 1190 Heap *pHeap, 1191 BLACKLIST_CHUNK *pBlacklistChunk 1192 ) 1193 { 1194 NV_STATUS status = NV_OK; 1195 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1196 NV_ASSERT(pBlacklistChunk != NULL); 1197 1198 status = memdescCreate(&pBlacklistChunk->pMemDesc, 1199 pGpu, pBlacklistChunk->size, RM_PAGE_SIZE, 1200 NV_TRUE, ADDR_FBMEM, NV_MEMORY_UNCACHED, 1201 MEMDESC_FLAGS_FIXED_ADDRESS_ALLOCATE | 1202 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 1203 if (NV_OK != status) 1204 { 1205 NV_PRINTF(LEVEL_FATAL, 1206 "Error 0x%x creating memdesc for blacklisted chunk for address0x%llx, skipping\n", 1207 status, pBlacklistChunk->physOffset); 1208 NV_ASSERT(NV_FALSE); 1209 return status; 1210 } 1211 1212 // this is how FIXED_ADDRESS_ALLOCATE works 1213 memdescSetPte(pBlacklistChunk->pMemDesc, AT_GPU, 0, RM_PAGE_ALIGN_DOWN(pBlacklistChunk->physOffset)); 1214 1215 if (pHeap->heapType != HEAP_TYPE_PHYS_MEM_SUBALLOCATOR) 1216 { 1217 // 1218 // Allocate memory for this page. This is marked as an internal RM allocation 1219 // and will be saved/restored during suspend/resume 1220 // 1221 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_78, 1222 pBlacklistChunk->pMemDesc); 1223 if (NV_OK != status) 1224 { 1225 // no use for the memdesc if page couldn't be allocated 1226 memdescDestroy(pBlacklistChunk->pMemDesc); 1227 1228 NV_PRINTF(LEVEL_FATAL, 1229 "Error 0x%x creating page for blacklisting address: 0x%llx, skipping\n", 1230 status, pBlacklistChunk->physOffset); 1231 NV_ASSERT(NV_FALSE); 1232 return status; 1233 } 1234 } 1235 1236 // set the flags properly 1237 pBlacklistChunk->bIsValid = NV_TRUE; 1238 1239 // if dynamic blacklisteing is enabled, clear the pending retirement flag 1240 if (pMemoryManager->bEnableDynamicPageOfflining) 1241 { 1242 pBlacklistChunk->bPendingRetirement = NV_FALSE; 1243 } 1244 return status; 1245 } 1246 1247 /** 1248 * Free-s the blacklisted pages within the range [begin, begin+size-1] 1249 * This function will iterate the blacklisted chunks data structure, 1250 * and free the blacklisted pages within the range [begin, begin+size-1] 1251 * 1252 * @param[in] pGpu OBJGPU pointer 1253 * @param[in] pMemoryManager MemoryManager pointer 1254 * @param[in] pBlackList BLACKLIST pointer 1255 * @param[in] begin starting address of the range 1256 * @param[in] size Size of the region, where blacklisted pages to be free-d 1257 * 1258 * @returns NV_OK on success 1259 */ 1260 static NV_STATUS 1261 _heapFreeBlacklistPages 1262 ( 1263 OBJGPU *pGpu, 1264 MemoryManager *pMemoryManager, 1265 BLACKLIST *pBlackList, 1266 NvU64 begin, 1267 NvU64 size 1268 ) 1269 { 1270 NvU32 chunk = 0; 1271 NvU64 baseChunkAddress = 0; 1272 NvU64 endChunkAddress = 0; 1273 BLACKLIST_CHUNK *pBlacklistChunks = pBlackList->pBlacklistChunks; 1274 1275 for (chunk = 0; chunk < pBlackList->count; chunk++) 1276 { 1277 baseChunkAddress = 0; 1278 endChunkAddress = 0; 1279 // No need to process the chunk if it's not a valid chunk 1280 if (pBlacklistChunks[chunk].bIsValid != NV_TRUE || 1281 (pMemoryManager->bEnableDynamicPageOfflining && 1282 pBlacklistChunks[chunk].bPendingRetirement)) 1283 continue; 1284 1285 baseChunkAddress = pBlacklistChunks[chunk].physOffset; 1286 endChunkAddress = baseChunkAddress + pBlacklistChunks[chunk].size - 1; 1287 1288 if (baseChunkAddress >= begin && endChunkAddress <= (begin + size - 1)) 1289 { 1290 // 1291 // free the mem desc, set the excludeGlobalListFlag 1292 // invalidate the entry 1293 // 1294 NV_PRINTF(LEVEL_FATAL, 1295 "removing from blacklist... page start %llx, page end:%llx\n", 1296 baseChunkAddress, endChunkAddress); 1297 1298 memdescFree(pBlacklistChunks[chunk].pMemDesc); 1299 memdescDestroy(pBlacklistChunks[chunk].pMemDesc); 1300 1301 pBlacklistChunks[chunk].bIsValid = NV_FALSE; 1302 } 1303 } 1304 return NV_OK; 1305 } 1306 1307 /** 1308 * Blacklist pages within the range [begin, begin+size-1] 1309 * This function will iterate the blacklisted chunks data structure, 1310 * and blacklist pages within the range [begin, begin+size-1] 1311 * 1312 * @param[in] pGpu OBJGPU pointer 1313 * @param[in] pHeap Heap pointer 1314 * @param[in] pBlackList BLACKLIST pointer 1315 * @param[in] begin starting address of the range 1316 * @param[in] size Size of the region, where pages will be blacklisted 1317 * 1318 * @returns NV_OK on success 1319 * error, if _heapBlacklistSingleChunk fails 1320 */ 1321 static NV_STATUS 1322 _heapBlacklistChunks 1323 ( 1324 OBJGPU *pGpu, 1325 Heap *pHeap, 1326 BLACKLIST *pBlackList, 1327 NvU64 begin, 1328 NvU64 size 1329 ) 1330 { 1331 NvU32 chunk = 0; 1332 NvU64 baseAddress = 0; 1333 NvU64 endAddress = 0; 1334 BLACKLIST_CHUNK *pBlacklistChunks = pBlackList->pBlacklistChunks; 1335 NV_STATUS status = NV_OK; 1336 1337 1338 for (chunk = 0; chunk < pBlackList->count; chunk++) 1339 { 1340 baseAddress = 0; 1341 endAddress = 0; 1342 1343 // No need to process the chunk if it's a valid chunk 1344 if (pBlacklistChunks[chunk].bIsValid == NV_TRUE) 1345 continue; 1346 1347 baseAddress = pBlacklistChunks[chunk].physOffset; 1348 endAddress = baseAddress + pBlacklistChunks[chunk].size - 1; 1349 1350 //TODO: what if the blacklisted chunk is halfway inside the allocated region?? 1351 if (baseAddress >= begin && endAddress <= (begin + size - 1)) 1352 { 1353 NV_PRINTF(LEVEL_ERROR, 1354 "blacklisting chunk from addr: 0x%llx to 0x%llx, new begin :0x%llx, end:0x%llx\n", 1355 baseAddress, endAddress, begin, begin + size - 1); 1356 status = _heapBlacklistSingleChunk(pGpu, pHeap, &pBlacklistChunks[chunk]); 1357 NV_ASSERT(status == NV_OK); 1358 } 1359 } 1360 return status; 1361 } 1362 1363 /*! 1364 * @brief allocate memory from heap 1365 * 1366 * Allocates a memory region with requested parameters from heap. 1367 * If requested contiguous allocation is not possible, tries to allocate non-contiguous memory. 1368 * 1369 * @param[in] pGpu GPU object 1370 * @param[in] hClient client handle 1371 * @param[in] pHeap heap object 1372 * @param[in] pAllocRequest allocation request 1373 * @param[in] memHandle memory handle 1374 * @param[in/out] pAllocData heap-specific allocation data 1375 * @param[in/out] pFbAllocInfo allocation data 1376 * @param[out] pHwResource pointer to allocation HW resource info 1377 * @param[in/out] pNoncontigAllocation the requested/provided allocation is noncotig 1378 * @param[in] bNoncontigAllowed allocation can be made noncontig 1379 * @param[in] bAllocedMemdesc memdesc should be freed if a new one is created 1380 */ 1381 NV_STATUS heapAlloc_IMPL 1382 ( 1383 OBJGPU *pGpu, 1384 NvHandle hClient, 1385 Heap *pHeap, 1386 MEMORY_ALLOCATION_REQUEST *pAllocRequest, 1387 NvHandle memHandle, 1388 OBJHEAP_ALLOC_DATA *pAllocData, 1389 FB_ALLOC_INFO *pFbAllocInfo, 1390 HWRESOURCE_INFO **pHwResource, 1391 NvBool *pNoncontigAllocation, 1392 NvBool bNoncontigAllowed, 1393 NvBool bAllocedMemdesc 1394 ) 1395 { 1396 NV_MEMORY_ALLOCATION_PARAMS *pVidHeapAlloc = pAllocRequest->pUserParams; 1397 MEMORY_DESCRIPTOR *pMemDesc = pAllocRequest->pMemDesc; 1398 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1399 NvU32 textureClientIndex = 0xFFFFFFFF; 1400 NvU64 desiredOffset = pFbAllocInfo->offset; 1401 NvU64 adjustedSize = pFbAllocInfo->size - pFbAllocInfo->alignPad; 1402 NvU32 bankPlacement = 0; 1403 NvBool ignoreBankPlacement = NV_FALSE; 1404 NvU8 currentBankInfo; 1405 MEM_BLOCK *pBlockFirstFree; 1406 MEM_BLOCK *pBlockFree; 1407 MEM_BLOCK *pBlockNew = NULL; 1408 MEM_BLOCK *pBlockSplit = NULL; 1409 NvU64 allocatedOffset = 0; 1410 NvBool bTurnBlacklistOff = NV_FALSE; 1411 NvBool bDone = NV_FALSE; 1412 NV_STATUS status = NV_OK; 1413 NvU32 i; 1414 1415 NV_ASSERT_OR_RETURN( 1416 (memmgrAllocGetAddrSpace(GPU_GET_MEMORY_MANAGER(pGpu), pVidHeapAlloc->flags, pVidHeapAlloc->attr) 1417 == ADDR_FBMEM) && 1418 (pAllocRequest->pPmaAllocInfo[gpumgrGetSubDeviceInstanceFromGpu(pGpu)] == NULL), 1419 NV_ERR_INVALID_ARGUMENT); 1420 1421 if (pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) 1422 desiredOffset -= pFbAllocInfo->alignPad; 1423 1424 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT) && 1425 gpuCheckPageRetirementSupport_HAL(pGpu) && 1426 FLD_TEST_DRF(OS32, _ATTR2, _BLACKLIST, _OFF, pVidHeapAlloc->attr2)) 1427 { 1428 NV_PRINTF(LEVEL_INFO, 1429 "Trying to turn blacklisting pages off for this allocation of size: %llx\n", 1430 pVidHeapAlloc->size); 1431 if (!hypervisorIsVgxHyper()) 1432 _heapFreeBlacklistPages(pGpu, pMemoryManager, &pHeap->blackList, desiredOffset, pVidHeapAlloc->size); 1433 else 1434 _heapFreeBlacklistPages(pGpu, pMemoryManager, &pHeap->blackList, pHeap->base, pHeap->total); 1435 bTurnBlacklistOff = NV_TRUE; 1436 // Now continue with the heap allocation. 1437 } 1438 1439 // 1440 // Check for range-limited request. 1441 // Range of [0,0] is a special case that means to use the entire heap. 1442 // 1443 // A range-limited request allows caller to say: I really want memory 1444 // which only falls completely within a particular range. Returns 1445 // error if can't allocate within that range. 1446 // 1447 // Used on Windows by OpenGL. On Windows during a modeswitch, the 1448 // display driver frees all vidmem surfaces. Unfortunately, OpenGL 1449 // writes to some vidmem surface with the CPU from user mode. If these 1450 // surfaces are freed during the modeswitch, then the user mode OpenGL 1451 // app might scribble on someone else's surface if that video memory is 1452 // reused before OpenGL notices the modeswitch. Because modeswitches 1453 // are asynchronous to the OpenGL client, it does not notice the 1454 // modeswitches right away. 1455 // 1456 // A solution is for OpenGL to restrict vidmem surfaces that have 1457 // this problem to a range of memory where it is safe *not* to free 1458 // the surface during a modeswitch. 1459 // 1460 // virtual allocation are checked in dmaAllocVA() 1461 if (pVidHeapAlloc->rangeLo == 0 && pVidHeapAlloc->rangeHi == 0) 1462 { 1463 pVidHeapAlloc->rangeHi = pHeap->base + pHeap->total - 1; 1464 } 1465 if (pVidHeapAlloc->rangeHi > pHeap->base + pHeap->total - 1) 1466 { 1467 pVidHeapAlloc->rangeHi = pHeap->base + pHeap->total - 1; 1468 } 1469 1470 if ((pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) == 0) 1471 { 1472 // Only want to override in one direction at a time 1473 if (pMemoryManager->overrideInitHeapMin == 0) 1474 { 1475 pVidHeapAlloc->rangeHi = NV_MIN(pVidHeapAlloc->rangeHi, pMemoryManager->overrideHeapMax); 1476 } 1477 else 1478 { 1479 pVidHeapAlloc->rangeLo = NV_MAX(pVidHeapAlloc->rangeLo, pMemoryManager->overrideInitHeapMin); 1480 } 1481 } 1482 1483 // 1484 // Check for valid range. 1485 // 1486 if (pVidHeapAlloc->rangeLo > pVidHeapAlloc->rangeHi) 1487 { 1488 status = NV_ERR_INVALID_ARGUMENT; 1489 goto return_early; 1490 } 1491 1492 // 1493 // The bank placement loop does not know how to limit allocations to be 1494 // within a range. 1495 // 1496 if (((pVidHeapAlloc->rangeLo > 0) || (pVidHeapAlloc->rangeHi < pHeap->base + pHeap->total - 1))) 1497 { 1498 pVidHeapAlloc->flags |= NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT; 1499 } 1500 1501 // 1502 // Set up bank placement data - should have been preselected in heapCreate 1503 // 1504 status = _heapGetBankPlacement(pGpu, pHeap, pVidHeapAlloc->owner, 1505 &pVidHeapAlloc->flags, 1506 pVidHeapAlloc->type, 1507 0, 1508 &bankPlacement); 1509 if (status != NV_OK) 1510 { 1511 NV_PRINTF(LEVEL_ERROR, 1512 "_heapGetBankPlacement failed for current allocation\n"); 1513 goto return_early; 1514 } 1515 1516 // 1517 // Find the best bank to start looking in for this pVidHeapAlloc->type, but only if we're 1518 // not ignoring bank placement rules. Save the current bank info. 1519 // 1520 currentBankInfo = (NvU8)bankPlacement; // this is always non zero from above 1521 1522 // 1523 // Check for fixed address request. 1524 // This allows caller to say: I really want this memory at a particular 1525 // offset. Returns error if can't get that offset. 1526 // Used initially by Mac display driver twinview code. 1527 // On the Mac it is a very bad thing to *ever* move the primary 1528 // during a modeset since a lot of sw caches the value and never 1529 // checks again. 1530 // 1531 if (pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) 1532 { 1533 1534 // is our desired offset suitably aligned? 1535 if (desiredOffset % pAllocData->alignment) 1536 { 1537 NV_PRINTF(LEVEL_ERROR, 1538 "offset 0x%llx not aligned to 0x%llx\n", 1539 desiredOffset, pAllocData->alignment); 1540 goto failed; 1541 } 1542 1543 pBlockFree = pHeap->pFreeBlockList; 1544 1545 if (pBlockFree == NULL) 1546 { 1547 NV_PRINTF(LEVEL_ERROR, "no free blocks\n"); 1548 goto failed; 1549 } 1550 1551 do { 1552 // 1553 // Allocate from the bottom of the memory block. 1554 // 1555 pBlockFree = pBlockFree->u1.nextFree; 1556 1557 // Does this block contain our desired range? 1558 if ((desiredOffset >= pBlockFree->begin) && 1559 (desiredOffset + pAllocData->allocSize - 1) <= pBlockFree->end) 1560 { 1561 // we have a match, now remove it from the pool 1562 pAllocData->allocLo = desiredOffset; 1563 pAllocData->allocHi = desiredOffset + pAllocData->allocSize - 1; 1564 pAllocData->allocAl = pAllocData->allocLo; 1565 1566 // Check that the candidate block can support the allocation type 1567 if (_isAllocValidForFBRegion(pGpu, pHeap, pFbAllocInfo, pAllocData)) 1568 goto got_one; 1569 } 1570 1571 } while (pBlockFree != pHeap->pFreeBlockList); 1572 1573 // return error if can't get that particular address 1574 NV_PRINTF(LEVEL_ERROR, 1575 "failed NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE @%llx (%lld bytes)\n", 1576 desiredOffset, pAllocData->allocSize); 1577 goto failed; 1578 } 1579 1580 // 1581 // Check if NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT was passed in with 1582 // the pVidHeapAlloc->type to ignore placing this allocation in a particular bank. 1583 // This means we default to the second loop where we choose first fit. 1584 // 1585 if (pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT) 1586 ignoreBankPlacement = NV_TRUE; 1587 1588 // 1589 // Bug 67690: Treat textures differently for more than one client (eg. opengl), 1590 // [IN]: client, pVidHeapAlloc->type, ignoreBankPlacement 1591 // [OUT]: heap, ignoreBankPlacement, textureClientIndex 1592 // 1593 // Bug 69385: Treat textures differently only if pVidHeapAlloc->flags are also set to zero. 1594 // NV30GL-WinXP: Unable to run 3DMark2001SE @ 1600x1200x32bpp. 1595 // 1596 if ((pVidHeapAlloc->type == NVOS32_TYPE_TEXTURE) && (!pVidHeapAlloc->flags)) 1597 _heapSetTexturePlacement(pHeap, hClient, pVidHeapAlloc->type, &ignoreBankPlacement, &textureClientIndex, ¤tBankInfo); 1598 1599 if (!ignoreBankPlacement) 1600 { 1601 currentBankInfo = (NvU8)bankPlacement & BANK_MEM_GROW_MASK; 1602 1603 if (pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_BANK_HINT) 1604 { 1605 if (pVidHeapAlloc->flags & NVOS32_ALLOC_FLAGS_BANK_GROW_DOWN) 1606 currentBankInfo = MEM_GROW_DOWN; 1607 else 1608 currentBankInfo = MEM_GROW_UP; 1609 pVidHeapAlloc->flags &= ~(NVOS32_ALLOC_FLAGS_BANK_HINT); // hint flag only lasts for 1 loop 1610 } 1611 else 1612 { 1613 // Convert bank grow up/down to mem grow up/down 1614 currentBankInfo = (currentBankInfo & BANK_MEM_GROW_DOWN ? MEM_GROW_DOWN : MEM_GROW_UP); 1615 } 1616 } // if (!ignoreBankPlacement) 1617 1618 pBlockFirstFree = pHeap->pFreeBlockList; 1619 if (!pBlockFirstFree) 1620 { 1621 NV_PRINTF(LEVEL_ERROR, "no free blocks\n"); 1622 goto failed; 1623 } 1624 1625 if (*pNoncontigAllocation) 1626 { 1627 NV_PRINTF(LEVEL_INFO, "non-contig vidmem requested\n"); 1628 goto non_contig_alloc; 1629 } 1630 1631 // 1632 // Loop through all available regions. 1633 // Note we don't check for bRsvdRegion here because when blacklisting 1634 // those regions we need them to succeed. 1635 // 1636 bDone = NV_FALSE; 1637 i = 0; 1638 while (!bDone) 1639 { 1640 NvU64 saveRangeLo = pVidHeapAlloc->rangeLo; 1641 NvU64 saveRangeHi = pVidHeapAlloc->rangeHi; 1642 1643 if (!memmgrAreFbRegionsSupported(pMemoryManager) || 1644 gpuIsCacheOnlyModeEnabled(pGpu)) 1645 { 1646 bDone = NV_TRUE; 1647 } 1648 else 1649 { 1650 NV_ASSERT( pMemoryManager->Ram.numFBRegionPriority > 0 ); 1651 1652 if (FLD_TEST_DRF(OS32, _ATTR2, _PRIORITY, _LOW, pFbAllocInfo->pageFormat->attr2) || 1653 (pMemoryManager->bPreferSlowRegion && 1654 !FLD_TEST_DRF(OS32, _ATTR2, _PRIORITY, _HIGH, pFbAllocInfo->pageFormat->attr2))) 1655 { 1656 NV_ASSERT( pMemoryManager->Ram.fbRegionPriority[pMemoryManager->Ram.numFBRegionPriority-1-i] < pMemoryManager->Ram.numFBRegions ); 1657 NV_ASSERT( !pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[pMemoryManager->Ram.numFBRegionPriority-1-i]].bRsvdRegion ); 1658 // 1659 // We prefer slow memory, or we want _LOW priority 1660 // ==>> Try allocations in increasing order of performance, 1661 // slowest first 1662 // 1663 pVidHeapAlloc->rangeLo = NV_MAX(pVidHeapAlloc->rangeLo, pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[pMemoryManager->Ram.numFBRegionPriority-1-i]].base); 1664 pVidHeapAlloc->rangeHi = NV_MIN(pVidHeapAlloc->rangeHi, pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[pMemoryManager->Ram.numFBRegionPriority-1-i]].limit); 1665 } 1666 else 1667 { 1668 NV_ASSERT( pMemoryManager->Ram.fbRegionPriority[i] < pMemoryManager->Ram.numFBRegions ); 1669 NV_ASSERT( !pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[i]].bRsvdRegion ); 1670 // 1671 // We don't explicitly want slow memory or we don't prefer 1672 // allocations in the slow memory 1673 // ==>> Try allocations in decreasing order of performance, 1674 // fastest first 1675 // 1676 pVidHeapAlloc->rangeLo = NV_MAX(pVidHeapAlloc->rangeLo, pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[i]].base); 1677 pVidHeapAlloc->rangeHi = NV_MIN(pVidHeapAlloc->rangeHi, pMemoryManager->Ram.fbRegion[pMemoryManager->Ram.fbRegionPriority[i]].limit); 1678 } 1679 i++; 1680 1681 bDone = !(i < pMemoryManager->Ram.numFBRegionPriority); 1682 } 1683 1684 // 1685 // When scanning upwards, start at the bottom - 1 so the following loop looks symetrical. 1686 // 1687 if ( ! (currentBankInfo & MEM_GROW_DOWN)) 1688 pBlockFirstFree = pBlockFirstFree->u0.prevFree; 1689 pBlockFree = pBlockFirstFree; 1690 1691 do 1692 { 1693 NvU64 blockLo; 1694 NvU64 blockHi; 1695 1696 if (currentBankInfo & MEM_GROW_DOWN) 1697 pBlockFree = pBlockFree->u0.prevFree; 1698 else 1699 pBlockFree = pBlockFree->u1.nextFree; 1700 1701 // 1702 // Is this block completely in requested range? 1703 // 1704 // We *should* check that pBlockFree is wholely resident in the range, but the 1705 // old check didn't and checking it causes some tests to fail. 1706 // So check that at least *some* of the block resides within the requested range. 1707 // 1708 if ((pBlockFree->end >= pVidHeapAlloc->rangeLo) && (pBlockFree->begin <= pVidHeapAlloc->rangeHi)) 1709 { 1710 // 1711 // Find the intersection of the free block and the specified range. 1712 // 1713 blockLo = (pVidHeapAlloc->rangeLo > pBlockFree->begin) ? pVidHeapAlloc->rangeLo : pBlockFree->begin; 1714 blockHi = (pVidHeapAlloc->rangeHi < pBlockFree->end) ? pVidHeapAlloc->rangeHi : pBlockFree->end; 1715 1716 if (currentBankInfo & MEM_GROW_DOWN) 1717 { 1718 // 1719 // Allocate from the top of the memory block. 1720 // 1721 pAllocData->allocLo = (blockHi - pAllocData->allocSize + 1) / pAllocData->alignment * pAllocData->alignment; 1722 pAllocData->allocAl = pAllocData->allocLo; 1723 pAllocData->allocHi = pAllocData->allocAl + pAllocData->allocSize - 1; 1724 } 1725 else 1726 { 1727 // 1728 // Allocate from the bottom of the memory block. 1729 // 1730 pAllocData->allocAl = (blockLo + (pAllocData->alignment - 1)) / pAllocData->alignment * pAllocData->alignment; 1731 pAllocData->allocLo = pAllocData->allocAl; 1732 pAllocData->allocHi = pAllocData->allocAl + pAllocData->allocSize - 1; 1733 } 1734 1735 // 1736 // Does the desired range fall completely within this block? 1737 // Also make sure it does not wrap-around. 1738 // Also make sure it is within the desired range. 1739 // 1740 if ((pAllocData->allocLo >= pBlockFree->begin) && (pAllocData->allocHi <= pBlockFree->end)) 1741 { 1742 if (pAllocData->allocLo <= pAllocData->allocHi) 1743 { 1744 if ((pAllocData->allocLo >= pVidHeapAlloc->rangeLo) && (pAllocData->allocHi <= pVidHeapAlloc->rangeHi)) 1745 { 1746 // Check that the candidate block can support the allocation type 1747 if (_isAllocValidForFBRegion(pGpu, pHeap, pFbAllocInfo, pAllocData)) 1748 { 1749 pVidHeapAlloc->rangeLo = saveRangeLo; 1750 pVidHeapAlloc->rangeHi = saveRangeHi; 1751 goto got_one; 1752 } 1753 } 1754 } 1755 } 1756 } 1757 1758 } while (pBlockFree != pBlockFirstFree); 1759 1760 pVidHeapAlloc->rangeLo = saveRangeLo; 1761 pVidHeapAlloc->rangeHi = saveRangeHi; 1762 } 1763 1764 non_contig_alloc: 1765 if (!bNoncontigAllowed) 1766 goto failed; 1767 1768 if (!*pNoncontigAllocation) 1769 { 1770 NV_PRINTF(LEVEL_INFO, 1771 "Contig vidmem allocation failed, running noncontig allocator\n"); 1772 1773 // Create a new noncontig memdescriptor 1774 memdescDestroy(pAllocRequest->pMemDesc); 1775 1776 status = memdescCreate(&pAllocRequest->pMemDesc, pGpu, adjustedSize, 1777 0, NV_FALSE, ADDR_FBMEM, NV_MEMORY_UNCACHED, 1778 MEMDESC_FLAGS_NONE); 1779 1780 if (status != NV_OK) 1781 { 1782 NV_PRINTF(LEVEL_ERROR, "cannot alloc memDesc!\n"); 1783 pMemDesc = pAllocRequest->pMemDesc = NULL; 1784 goto failed; 1785 } 1786 1787 pMemDesc = pAllocRequest->pMemDesc; 1788 pMemDesc->pHeap = pHeap; 1789 1790 memdescSetPteKind(pMemDesc, pFbAllocInfo->format); 1791 memdescSetHwResId(pMemDesc, pFbAllocInfo->hwResId); 1792 } 1793 1794 // Try the noncontig allocator 1795 if (NV_OK == _heapAllocNoncontig(pGpu, 1796 hClient, 1797 pHeap, 1798 pAllocRequest, 1799 memHandle, 1800 pAllocData, 1801 pFbAllocInfo, 1802 textureClientIndex, 1803 pFbAllocInfo->alignPad, 1804 &allocatedOffset, 1805 pMemDesc, 1806 pHwResource)) 1807 { 1808 *pNoncontigAllocation = NV_TRUE; 1809 1810 // 1811 // The noncontig allocator calls _heapProcessFreeBlock() 1812 // by itself, so we goto done: straight 1813 // 1814 status = NV_OK; 1815 goto return_early; 1816 } 1817 1818 NV_PRINTF(LEVEL_INFO, 1819 "failed to allocate block. Heap total=0x%llx free=0x%llx\n", 1820 pHeap->total, pHeap->free); 1821 // Out of memory. 1822 goto failed; 1823 1824 // 1825 // We have a match. Now link it in, trimming or splitting 1826 // any slop from the enclosing block as needed. 1827 // 1828 1829 got_one: 1830 if (NV_OK != _heapProcessFreeBlock(pGpu, pBlockFree, 1831 &pBlockNew, &pBlockSplit, 1832 pHeap, pAllocRequest, 1833 memHandle, 1834 pAllocData, pFbAllocInfo, 1835 pFbAllocInfo->alignPad, 1836 &allocatedOffset) || 1837 NV_OK != _heapUpdate(pHeap, pBlockNew, BLOCK_FREE_STATE_CHANGED)) 1838 failed: 1839 { 1840 1841 NV_PRINTF(LEVEL_INFO, 1842 "failed to allocate block. Heap total=0x%llx free=0x%llx\n", 1843 pHeap->total, pHeap->free); 1844 1845 portMemFree(pBlockNew); 1846 pBlockNew = NULL; 1847 portMemFree(pBlockSplit); 1848 status = NV_ERR_NO_MEMORY; 1849 goto return_early; 1850 } 1851 1852 // 1853 // If a client calls us with pVidHeapAlloc->type == NVOS32_TYPE_TEXTURE, but where flags 1854 // are non-zero, we won't call _heapSetTexturePlacement and initialize 1855 // textureClientIndex to a proper value (default is 0xFFFFFFFF). In that 1856 // case, we won't track this texture allocation. Bug 79586. 1857 // 1858 if (pVidHeapAlloc->type == NVOS32_TYPE_TEXTURE && 1859 textureClientIndex != 0xFFFFFFFF) 1860 { 1861 pBlockNew->textureId = hClient; 1862 pHeap->textureData[textureClientIndex].refCount++; 1863 } 1864 else 1865 { 1866 pBlockNew->textureId = 0; 1867 } 1868 1869 pFbAllocInfo->offset = allocatedOffset; 1870 1871 // TODO : This must be inside *all* blocks of a noncontig allocation 1872 if (!*pNoncontigAllocation) 1873 { 1874 pBlockNew->pitch = pFbAllocInfo->pitch; 1875 pBlockNew->height = pFbAllocInfo->height; 1876 pBlockNew->width = pFbAllocInfo->width; 1877 } 1878 1879 *pHwResource = &pBlockNew->hwResource; 1880 1881 // Remember memory descriptor 1882 memdescDescribe(pMemDesc, ADDR_FBMEM, allocatedOffset, adjustedSize); 1883 pBlockNew->pMemDesc = pMemDesc; 1884 pBlockNew->allocedMemDesc = bAllocedMemdesc; 1885 1886 status = NV_OK; 1887 1888 return_early: 1889 HEAP_VALIDATE(pHeap); 1890 1891 if (bTurnBlacklistOff) 1892 { 1893 if (!hypervisorIsVgxHyper()) 1894 _heapBlacklistChunks(pGpu, pHeap, &pHeap->blackList, desiredOffset, pVidHeapAlloc->size); 1895 else 1896 _heapBlacklistChunksInFreeBlocks(pGpu, pHeap); 1897 } 1898 1899 return status; 1900 } 1901 1902 static void _heapBlacklistChunksInFreeBlocks 1903 ( 1904 OBJGPU *pGpu, 1905 Heap *pHeap 1906 ) 1907 { 1908 MEM_BLOCK *pBlockFirstFree, *pBlockFree; 1909 NvU64 blockLo; 1910 NvU64 blockHi; 1911 NvU64 size; 1912 1913 pBlockFirstFree = pHeap->pFreeBlockList; 1914 1915 if (pBlockFirstFree) 1916 { 1917 pBlockFirstFree = pBlockFirstFree->u0.prevFree; 1918 pBlockFree = pBlockFirstFree; 1919 do 1920 { 1921 pBlockFree = pBlockFree->u1.nextFree; 1922 blockLo = pBlockFree->begin; 1923 blockHi = pBlockFree->end; 1924 size = blockHi - blockLo + 1; 1925 1926 _heapBlacklistChunks(pGpu, pHeap, &pHeap->blackList, blockLo, size); 1927 1928 } while (pBlockFree != pBlockFirstFree); 1929 } 1930 } 1931 1932 static NV_STATUS _heapBlockFree 1933 ( 1934 OBJGPU *pGpu, 1935 Heap *pHeap, 1936 NvHandle hClient, 1937 NvHandle hDevice, 1938 MEM_BLOCK *pBlock 1939 ) 1940 { 1941 MEM_BLOCK *pBlockTmp; 1942 NvU32 i; 1943 OBJOS *pOS = GPU_GET_OS(pGpu); 1944 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1945 NvBool bBlocksMerged = NV_FALSE; 1946 1947 // 1948 // Check for valid owner. 1949 // 1950 if (pBlock->owner == NVOS32_BLOCK_TYPE_FREE) 1951 return NV_ERR_INVALID_STATE; 1952 1953 pBlock->owner = NVOS32_BLOCK_TYPE_FREE; 1954 1955 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_FREE_STATE_CHANGED)) 1956 { 1957 return NV_ERR_INVALID_STATE; 1958 } 1959 1960 // 1961 // Update free count. 1962 // 1963 _heapAdjustFree(pHeap, pBlock->end - pBlock->begin + 1, 1964 FLD_TEST_DRF(OS32, _ATTR2, _INTERNAL, _YES, pBlock->hwResource.attr2)); 1965 1966 // 1967 // Release any HW resources that might've been in use 1968 // 1969 { 1970 FB_ALLOC_INFO *pFbAllocInfo = NULL; 1971 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 1972 1973 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 1974 if (pFbAllocInfo == NULL) 1975 { 1976 NV_ASSERT(0); 1977 return NV_ERR_NO_MEMORY; 1978 } 1979 1980 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 1981 if (pFbAllocPageFormat == NULL) { 1982 NV_ASSERT(0); 1983 portMemFree(pFbAllocInfo); 1984 return NV_ERR_NO_MEMORY; 1985 } 1986 1987 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 1988 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 1989 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 1990 1991 pFbAllocInfo->pageFormat->type = pBlock->u0.type; 1992 pFbAllocInfo->hwResId = pBlock->hwResource.hwResId; 1993 pFbAllocInfo->height = 0; 1994 pFbAllocInfo->pitch = 0; 1995 pFbAllocInfo->size = pBlock->end - pBlock->begin + 1; 1996 pFbAllocInfo->align = pBlock->align; 1997 pFbAllocInfo->alignPad = pBlock->alignPad; 1998 pFbAllocInfo->offset = pBlock->begin; 1999 pFbAllocInfo->format = pBlock->format; 2000 pFbAllocInfo->comprCovg = pBlock->hwResource.comprCovg; 2001 pFbAllocInfo->zcullCovg = 0; 2002 pFbAllocInfo->pageFormat->attr = pBlock->hwResource.attr; 2003 pFbAllocInfo->pageFormat->attr2 = pBlock->hwResource.attr2; 2004 pFbAllocInfo->ctagOffset = pBlock->hwResource.ctagOffset; 2005 pFbAllocInfo->hClient = hClient; 2006 pFbAllocInfo->hDevice = hDevice; 2007 2008 memmgrFreeHwResources(pGpu, pMemoryManager, pFbAllocInfo); 2009 2010 if (FLD_TEST_DRF(OS32, _ATTR2, _INTERNAL, _YES, pFbAllocInfo->pageFormat->attr2)) 2011 { 2012 pOS->osInternalReserveFreeCallback(pFbAllocInfo->offset, pGpu->gpuId); 2013 } 2014 2015 // Clear the HW resource associations since this block can be reused or merged. 2016 portMemSet(&pBlock->hwResource, 0, sizeof(pBlock->hwResource)); 2017 2018 portMemFree(pFbAllocPageFormat); 2019 portMemFree(pFbAllocInfo); 2020 } 2021 2022 if ((pBlock->u0.type == NVOS32_TYPE_TEXTURE) && (pBlock->textureId != 0)) 2023 { 2024 for (i = 0; i < MAX_TEXTURE_CLIENT_IDS; i++) 2025 { 2026 // 2027 // 1. Find the client within the textureData structure 2028 // 2. Once found, set the value to 0 2029 // 3. Then decrement its refCount 2030 // 4. If refCount goes to zero, reset the textureData structure 2031 // that pertains to that index. 2032 // 2033 if (pHeap->textureData[i].clientId == pBlock->textureId) 2034 { 2035 pBlock->textureId = 0; 2036 pHeap->textureData[i].refCount--; 2037 if (pHeap->textureData[i].refCount == 0) 2038 portMemSet(&pHeap->textureData[i], 0, 2039 sizeof(TEX_INFO)); 2040 break; 2041 } 2042 } 2043 } 2044 2045 // Account for freeing any reserved RM region 2046 if ((pBlock->u0.type == NVOS32_TYPE_RESERVED) && (pBlock->owner == HEAP_OWNER_RM_RESERVED_REGION)) 2047 { 2048 NV_ASSERT(pHeap->reserved >= pBlock->end - pBlock->begin + 1); 2049 pHeap->reserved -= pBlock->end - pBlock->begin + 1; 2050 } 2051 2052 // 2053 // 2054 // Can this merge with any surrounding free blocks? 2055 // 2056 if ((pBlock->prev->owner == NVOS32_BLOCK_TYPE_FREE) && (pBlock != pHeap->pBlockList)) 2057 { 2058 // 2059 // Remove block to be freed and previous one since nodes will be 2060 // combined into single one. 2061 // 2062 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_REMOVE)) 2063 { 2064 return NV_ERR_INVALID_STATE; 2065 } 2066 if (NV_OK != _heapUpdate(pHeap, pBlock->prev, BLOCK_REMOVE)) 2067 { 2068 return NV_ERR_INVALID_STATE; 2069 } 2070 2071 // 2072 // Merge with previous block. 2073 // 2074 pBlock->prev->next = pBlock->next; 2075 pBlock->next->prev = pBlock->prev; 2076 pBlock->prev->end = pBlock->end; 2077 pBlockTmp = pBlock; 2078 pBlock = pBlock->prev; 2079 pHeap->numBlocks--; 2080 portMemFree(pBlockTmp); 2081 2082 // re-insert updated free block into rb-tree 2083 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_SIZE_CHANGED)) 2084 { 2085 return NV_ERR_INVALID_STATE; 2086 } 2087 2088 bBlocksMerged = NV_TRUE; 2089 } 2090 2091 if ((pBlock->next->owner == NVOS32_BLOCK_TYPE_FREE) && (pBlock->next != pHeap->pBlockList)) 2092 { 2093 // 2094 // Remove block to be freed and next one since nodes will be 2095 // combined into single one. 2096 // 2097 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_REMOVE)) 2098 { 2099 return NV_ERR_INVALID_STATE; 2100 } 2101 if (NV_OK != _heapUpdate(pHeap, pBlock->next, BLOCK_REMOVE)) 2102 { 2103 return NV_ERR_INVALID_STATE; 2104 } 2105 2106 // 2107 // Merge with next block. 2108 // 2109 pBlock->prev->next = pBlock->next; 2110 pBlock->next->prev = pBlock->prev; 2111 pBlock->next->begin = pBlock->begin; 2112 2113 if (pHeap->pBlockList == pBlock) 2114 pHeap->pBlockList = pBlock->next; 2115 2116 if (bBlocksMerged) 2117 { 2118 if (pHeap->pFreeBlockList == pBlock) 2119 pHeap->pFreeBlockList = pBlock->u1.nextFree; 2120 2121 pBlock->u1.nextFree->u0.prevFree = pBlock->u0.prevFree; 2122 pBlock->u0.prevFree->u1.nextFree = pBlock->u1.nextFree; 2123 } 2124 2125 pBlockTmp = pBlock; 2126 pBlock = pBlock->next; 2127 pHeap->numBlocks--; 2128 portMemFree(pBlockTmp); 2129 2130 // re-insert updated free block into rb-tree 2131 if (NV_OK != _heapUpdate(pHeap, pBlock, BLOCK_SIZE_CHANGED)) 2132 { 2133 return NV_ERR_INVALID_STATE; 2134 } 2135 2136 bBlocksMerged = NV_TRUE; 2137 } 2138 2139 if (!bBlocksMerged) 2140 { 2141 // 2142 // Nothing was merged. Add to free list. 2143 // 2144 pBlockTmp = pHeap->pFreeBlockList; 2145 if (!pBlockTmp) 2146 { 2147 pHeap->pFreeBlockList = pBlock; 2148 pBlock->u1.nextFree = pBlock; 2149 pBlock->u0.prevFree = pBlock; 2150 } 2151 else 2152 { 2153 if (pBlockTmp->begin > pBlock->begin) 2154 // 2155 // Insert into beginning of free list. 2156 // 2157 pHeap->pFreeBlockList = pBlock; 2158 else if (pBlockTmp->u0.prevFree->begin > pBlock->begin) 2159 // 2160 // Insert into free list. 2161 // 2162 do 2163 { 2164 pBlockTmp = pBlockTmp->u1.nextFree; 2165 } while (pBlockTmp->begin < pBlock->begin); 2166 /* 2167 else 2168 * Insert at end of list. 2169 */ 2170 pBlock->u1.nextFree = pBlockTmp; 2171 pBlock->u0.prevFree = pBlockTmp->u0.prevFree; 2172 pBlock->u0.prevFree->u1.nextFree = pBlock; 2173 pBlockTmp->u0.prevFree = pBlock; 2174 } 2175 } 2176 2177 pBlock->mhandle = 0x0; 2178 pBlock->align = pBlock->begin; 2179 pBlock->alignPad = 0; 2180 pBlock->format = 0; 2181 2182 HEAP_VALIDATE(pHeap); 2183 return (NV_OK); 2184 } 2185 2186 NV_STATUS heapReference_IMPL 2187 ( 2188 OBJGPU *pGpu, 2189 Heap *pHeap, 2190 NvU32 owner, 2191 MEMORY_DESCRIPTOR *pMemDesc 2192 ) 2193 { 2194 NvU64 offsetAlign = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 2195 MEM_BLOCK *pBlock; 2196 2197 // Bail out in case allocation is in PMA owned FB region. 2198 if (pMemDesc->pPmaAllocInfo) 2199 { 2200 if (0 != pMemDesc->pPmaAllocInfo->refCount) 2201 { 2202 pMemDesc->pPmaAllocInfo->refCount++; 2203 if (IsSLIEnabled(pGpu) && 2204 (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)) 2205 { // 2206 memdescAddRef(pMemDesc); // Otherwise we have a fake parent descriptor removed with existing submem descriptors. 2207 // In SLI only (not fully understood yet!). In non SLI, that memAddref() causes a memleak. 2208 // 2209 } 2210 } 2211 return NV_OK; 2212 } 2213 2214 if (owner == NVOS32_BLOCK_TYPE_FREE) 2215 return NV_ERR_INVALID_STATE; 2216 2217 pBlock = _heapFindAlignedBlockWithOwner(pGpu, pHeap, owner, offsetAlign); 2218 2219 if (!pBlock) 2220 return NV_ERR_INVALID_OFFSET; 2221 2222 if (pBlock->refCount == HEAP_MAX_REF_COUNT) 2223 { 2224 NV_PRINTF(LEVEL_ERROR, 2225 "heapReference: reference count %x will exceed maximum 0x%x:\n", 2226 pBlock->refCount, HEAP_MAX_REF_COUNT); 2227 return NV_ERR_GENERIC; 2228 } 2229 2230 pBlock->refCount++; 2231 if (IsSLIEnabled(pGpu) && 2232 (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)) 2233 { // 2234 memdescAddRef(pMemDesc); // Otherwise we have a fake parent descriptor removed with existing submem descriptors. 2235 // In SLI only (not fully understood yet!). In non SLI, that memAddref() causes a memleak. 2236 // 2237 } 2238 return NV_OK; 2239 } 2240 2241 static NV_STATUS 2242 _heapFindBlockByOffset 2243 ( 2244 OBJGPU *pGpu, 2245 Heap *pHeap, 2246 NvU32 owner, 2247 MEMORY_DESCRIPTOR *pMemDesc, 2248 NvU64 offset, 2249 MEM_BLOCK **ppBlock 2250 ) 2251 { 2252 NV_STATUS status; 2253 2254 // IRQL TEST: must be running at equivalent of passive-level 2255 IRQL_ASSERT_AND_RETURN(!osIsRaisedIRQL()); 2256 2257 *ppBlock = _heapFindAlignedBlockWithOwner(pGpu, pHeap, owner, 2258 offset); 2259 2260 if (!*ppBlock) 2261 { 2262 // Try finding block based solely on offset. This is primarily needed 2263 // to successfully locate a block that was allocated multiple times via 2264 // NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE in heapAlloc: there can 2265 // be multiple owners, so that _heapFindAlignedBlockWithOwner may fail. 2266 if ((status = heapGetBlock(pHeap, offset, ppBlock)) != NV_OK 2267 || !*ppBlock) 2268 return NV_ERR_INVALID_OFFSET; 2269 } 2270 2271 return NV_OK; 2272 } 2273 2274 NV_STATUS 2275 heapFree_IMPL 2276 ( 2277 OBJGPU *pGpu, 2278 Heap *pHeap, 2279 NvHandle hClient, 2280 NvHandle hDevice, 2281 NvU32 owner, 2282 MEMORY_DESCRIPTOR *pMemDesc 2283 ) 2284 { 2285 NV_STATUS status; 2286 MEM_BLOCK *pBlock; 2287 MEM_BLOCK *pNextBlock; 2288 NvU64 offsetAlign = memdescGetPhysAddr(pMemDesc, AT_GPU, 0); 2289 NvU64 allocBegin = 0; 2290 NvU64 allocEnd = 0; 2291 NvBool bTurnBlacklistOff = NV_FALSE; 2292 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 2293 2294 NV_ASSERT_OR_RETURN(pMemDesc->pHeap == pHeap, NV_ERR_INVALID_ARGUMENT); 2295 2296 if (memdescGetContiguity(pMemDesc, AT_GPU)) 2297 { 2298 status = _heapFindBlockByOffset(pGpu, pHeap, 2299 owner, pMemDesc, offsetAlign, 2300 &pBlock); 2301 if (NV_OK != status) 2302 { 2303 return status; 2304 } 2305 2306 if (pBlock->allocedMemDesc) 2307 { 2308 if (pMemDesc != pBlock->pMemDesc) 2309 { 2310 NV_ASSERT(pMemDesc == pBlock->pMemDesc); 2311 return NV_ERR_INVALID_ARGUMENT; 2312 } 2313 2314 // Clear only if the memdesc is about to be freed by memdescDestroy() 2315 if (pMemDesc->RefCount == 1) 2316 { 2317 pBlock->pMemDesc = NULL; 2318 } 2319 2320 memdescFree(pMemDesc); 2321 memdescDestroy(pMemDesc); 2322 } 2323 2324 if (--pBlock->refCount != 0) 2325 return NV_OK; 2326 2327 2328 if(pGpu->getProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT) && 2329 gpuCheckPageRetirementSupport_HAL(pGpu)) 2330 { 2331 if (FLD_TEST_DRF(OS32, _ATTR2, _BLACKLIST, _OFF, pBlock->hwResource.attr2)) 2332 { 2333 bTurnBlacklistOff = NV_TRUE; 2334 allocBegin = pBlock->begin; 2335 allocEnd = pBlock->end; 2336 } 2337 } 2338 2339 // 2340 // Since _heapBlockFree() unconditionally releases HW resources 2341 // such as compression tags, some memory descriptor fields 2342 // are rendered stale. These fields need to be reset to safer 2343 // default values (e.g. invalid HW resource ID, pitch PTE 2344 // kind, etc.) - they may be referenced again before the memory 2345 // descriptor itself is freed. 2346 // 2347 if (pBlock->allocedMemDesc && (pBlock->pMemDesc != NULL)) 2348 { 2349 memdescSetHwResId(pMemDesc, 0); 2350 // XXX We cannot reset the PTE kind here since it cause corruption 2351 // in RAGE. See bug 949059 2352 // 2353 // This is an ugly hack to help OGL recover from modeswitch. 2354 // A cleaner fix would be to change the way memory is managed in OGL, 2355 // but it doesn't worth the effort to fix that on XP, since the OS is 2356 // close to end of life. The OGL linux team have plan to change their 2357 // memory management in the future, so later this hack may not be 2358 // required anymore 2359 // pMemDesc->PteKind = 0; 2360 } 2361 2362 if ((status = _heapBlockFree(pGpu, pHeap, hClient, hDevice, pBlock)) != NV_OK) 2363 { 2364 NV_ASSERT(0); 2365 } 2366 2367 // 2368 // since the mem desc is freed, now we can reallocate the blacklisted pages 2369 // in the [allocBegin, allocEnd] 2370 // 2371 if (bTurnBlacklistOff) 2372 status = _heapBlacklistChunks(pGpu, pHeap, &pHeap->blackList, allocBegin, allocEnd-allocBegin+1); 2373 2374 if (pMemoryManager->bEnableDynamicPageOfflining) 2375 { 2376 NvU32 i = 0; 2377 BLACKLIST *pBlacklist = &pHeap->blackList; 2378 BLACKLIST_CHUNK *pBlacklistChunks = pBlacklist->pBlacklistChunks; 2379 2380 for (i = 0; i < pBlacklist->count; i++) 2381 { 2382 if (pBlacklistChunks[i].bPendingRetirement && 2383 (pBlacklistChunks[i].physOffset >= allocBegin && 2384 pBlacklistChunks[i].physOffset <= allocEnd)) 2385 { 2386 status = _heapBlacklistSingleChunk(pGpu, pHeap, &pBlacklist->pBlacklistChunks[i]); 2387 if (NV_OK != status) 2388 { 2389 NV_PRINTF(LEVEL_ERROR, "heapBlacklistSingleChunk, status: %x!\n", status); 2390 return status; 2391 } 2392 } 2393 } 2394 } 2395 return status; 2396 } 2397 else 2398 { 2399 NvBool bBlacklistFailed = NV_FALSE; 2400 // 2401 // Use the pMemDesc->PteArray[0] to find the first block 2402 // The remaining blocks can be found from each block's 2403 // noncontigAllocListNext pointer 2404 // 2405 status = _heapFindBlockByOffset(pGpu, pHeap, 2406 owner, pMemDesc, 2407 memdescGetPte(pMemDesc, AT_GPU, 0), &pBlock); 2408 2409 if (NV_OK != status) 2410 { 2411 return status; 2412 } 2413 2414 while (pBlock != NULL) 2415 { 2416 // _heapBlockFree() clears pBlock, so save the next pointer 2417 pNextBlock = pBlock->noncontigAllocListNext; 2418 2419 if (--pBlock->refCount != 0) 2420 { 2421 // Remove this block from the noncontig allocation list 2422 pBlock->noncontigAllocListNext = NULL; 2423 pBlock = pNextBlock; 2424 continue; 2425 } 2426 2427 if (NV_OK != (status = _heapBlockFree(pGpu, pHeap, hClient, hDevice, pBlock))) 2428 return status; 2429 2430 // check if we need to dynamically blacklist the page 2431 if (pMemoryManager->bEnableDynamicPageOfflining) 2432 { 2433 NvU32 i = 0; 2434 BLACKLIST *pBlacklist = &pHeap->blackList; 2435 BLACKLIST_CHUNK *pBlacklistChunks = pBlacklist->pBlacklistChunks; 2436 for (i = 0; i < pBlacklist->count; i++) 2437 { 2438 if (pBlacklistChunks[i].bPendingRetirement && 2439 (pBlacklistChunks[i].physOffset >= pBlock->begin && 2440 pBlacklistChunks[i].physOffset <= pBlock->end)) 2441 { 2442 status = _heapBlacklistSingleChunk(pGpu, pHeap, &pBlacklist->pBlacklistChunks[i]); 2443 if (NV_OK != status) 2444 { 2445 NV_PRINTF(LEVEL_ERROR, "heapBlacklistSingleChunk, status: %x!\n", status); 2446 bBlacklistFailed = NV_TRUE; 2447 } 2448 } 2449 } 2450 } 2451 pBlock = pNextBlock; 2452 } 2453 2454 memdescFree(pMemDesc); 2455 memdescDestroy(pMemDesc); 2456 2457 if (bBlacklistFailed) 2458 { 2459 return NV_ERR_INVALID_STATE; 2460 } 2461 else 2462 { 2463 return status; 2464 } 2465 } 2466 } 2467 2468 NV_STATUS heapGetBlock_IMPL 2469 ( 2470 Heap *pHeap, 2471 NvU64 offset, 2472 MEM_BLOCK **ppMemBlock 2473 ) 2474 { 2475 NODE *pNode; 2476 2477 if (btreeSearch(offset, &pNode, pHeap->pBlockTree) != NV_OK) 2478 { 2479 if (ppMemBlock) 2480 { 2481 *ppMemBlock = NULL; 2482 } 2483 return NV_ERR_GENERIC; 2484 } 2485 2486 if (ppMemBlock) 2487 { 2488 *ppMemBlock = (MEM_BLOCK *)pNode->Data; 2489 } 2490 2491 return NV_OK; 2492 } 2493 2494 static MEM_BLOCK *_heapFindAlignedBlockWithOwner 2495 ( 2496 OBJGPU *pGpu, 2497 Heap *pHeap, 2498 NvU32 owner, 2499 NvU64 offset // aligned 2500 ) 2501 { 2502 MEM_BLOCK *pBlock; 2503 NODE *pNode; 2504 2505 HEAP_VALIDATE(pHeap); 2506 2507 if (btreeSearch(offset, &pNode, pHeap->pBlockTree) != NV_OK) 2508 { 2509 return NULL; 2510 } 2511 2512 pBlock = (MEM_BLOCK *)pNode->Data; 2513 if (pBlock->owner != owner) 2514 { 2515 return NULL; 2516 } 2517 2518 return pBlock; 2519 } 2520 2521 NV_STATUS heapGetSize_IMPL 2522 ( 2523 Heap *pHeap, 2524 NvU64 *size 2525 ) 2526 { 2527 *size = pHeap->total; 2528 HEAP_VALIDATE(pHeap); 2529 return (NV_OK); 2530 } 2531 2532 NV_STATUS heapGetUsableSize_IMPL 2533 ( 2534 Heap *pHeap, 2535 NvU64 *usableSize 2536 ) 2537 { 2538 *usableSize = pHeap->total - pHeap->reserved; 2539 HEAP_VALIDATE(pHeap); 2540 return (NV_OK); 2541 } 2542 2543 NV_STATUS heapGetFree_IMPL 2544 ( 2545 Heap *pHeap, 2546 NvU64 *free 2547 ) 2548 { 2549 *free = pHeap->free; 2550 HEAP_VALIDATE(pHeap); 2551 return (NV_OK); 2552 } 2553 2554 NV_STATUS heapGetBase_IMPL 2555 ( 2556 Heap *pHeap, 2557 NvU64 *base 2558 ) 2559 { 2560 *base = pHeap->base; 2561 HEAP_VALIDATE(pHeap); 2562 return (NV_OK); 2563 } 2564 2565 static NV_STATUS _heapGetMaxFree 2566 ( 2567 Heap *pHeap, 2568 NvU64 *maxOffset, 2569 NvU64 *maxFree 2570 ) 2571 { 2572 MEM_BLOCK *pBlockFirstFree, *pBlockFree; 2573 NvU64 freeBlockSize; 2574 2575 *maxFree = 0; 2576 2577 pBlockFirstFree = pHeap->pFreeBlockList; 2578 if (!pBlockFirstFree) 2579 // There are no free blocks. Max free is already set to 0 2580 return (NV_OK); 2581 2582 // Walk the free block list. 2583 pBlockFree = pBlockFirstFree; 2584 do { 2585 freeBlockSize = pBlockFree->end - pBlockFree->begin + 1; 2586 if (freeBlockSize > *maxFree) 2587 { 2588 *maxOffset = pBlockFree->begin; 2589 *maxFree = freeBlockSize; 2590 } 2591 pBlockFree = pBlockFree->u1.nextFree; 2592 } while (pBlockFree != pBlockFirstFree); 2593 2594 return (NV_OK); 2595 } 2596 2597 NV_STATUS heapInfo_IMPL 2598 ( 2599 Heap *pHeap, 2600 NvU64 *bytesFree, 2601 NvU64 *bytesTotal, 2602 NvU64 *base, 2603 NvU64 *largestOffset, // largest free blocks offset 2604 NvU64 *largestFree // largest free blocks size 2605 ) 2606 { 2607 NV_STATUS status; 2608 2609 *bytesFree = pHeap->free; 2610 *bytesTotal = pHeap->total - pHeap->reserved; 2611 *base = pHeap->base; 2612 status = _heapGetMaxFree(pHeap, largestOffset, largestFree); 2613 HEAP_VALIDATE(pHeap); 2614 2615 return status; 2616 } 2617 2618 NV_STATUS heapInfoTypeAllocBlocks_IMPL 2619 ( 2620 Heap *pHeap, 2621 NvU32 type, 2622 NvU64 *bytesTotal 2623 ) 2624 { 2625 MEM_BLOCK *pBlock; 2626 NvU64 total; 2627 2628 if (type >= NVOS32_NUM_MEM_TYPES) return (NV_ERR_GENERIC); 2629 2630 pBlock = pHeap->pBlockList; 2631 total = 0; 2632 2633 if (type == NVOS32_TYPE_OWNER_RM) 2634 { 2635 // 2636 // Scan for all the blocks whose owner is within 2637 // HEAP_OWNER_RM_SCRATCH_BEGIN and HEAP_OWNER_RM_SCRATCH_END 2638 // this is strictly speaking not 'type' search. Also note that this 2639 // includes reserved space in any,.like in case of 3FB mixed density mode. 2640 // 2641 do 2642 { 2643 if ( (pBlock->owner > HEAP_OWNER_RM_SCRATCH_BEGIN) && 2644 (pBlock->owner < HEAP_OWNER_RM_SCRATCH_END) ) 2645 { 2646 total += (pBlock->end - pBlock->begin + 1); 2647 } 2648 pBlock = pBlock->next; 2649 } while (pBlock != pHeap->pBlockList); 2650 } 2651 else 2652 { 2653 // 2654 // Scan for all the blocks belonging to this type. 2655 // 2656 do 2657 { 2658 if (pBlock->u0.type == type) 2659 total += (pBlock->end - pBlock->begin + 1); 2660 pBlock = pBlock->next; 2661 } while (pBlock != pHeap->pBlockList); 2662 } 2663 2664 *bytesTotal = total; 2665 2666 HEAP_VALIDATE(pHeap); 2667 return NV_OK; 2668 } 2669 2670 NV_STATUS heapGetBlockHandle_IMPL( 2671 Heap *pHeap, 2672 NvU32 owner, 2673 NvU32 type, 2674 NvU64 offset, 2675 NvBool bSkipCheck, // NV_TRUE if skip alignment/type check 2676 NvHandle *puHandle 2677 ) 2678 { 2679 MEM_BLOCK *pBlock; 2680 NV_STATUS status; 2681 2682 if (offset > (pHeap->base + pHeap->total - 1)) return (NV_ERR_GENERIC); 2683 2684 status = heapGetBlock(pHeap, offset, &pBlock); 2685 if (status != NV_OK) 2686 { 2687 return status; 2688 } 2689 2690 if (!((pBlock->owner == owner) && 2691 (((pBlock->u0.type == type) && (pBlock->align == offset)) || bSkipCheck))) 2692 { 2693 return NV_ERR_GENERIC; 2694 } 2695 2696 *puHandle = pBlock->mhandle; 2697 return NV_OK; 2698 } 2699 2700 // 2701 // Returns the number of blocks (free or allocated) currently in the heap 2702 // 2703 NvU32 heapGetNumBlocks_IMPL 2704 ( 2705 Heap *pHeap 2706 ) 2707 { 2708 return pHeap->numBlocks; 2709 } 2710 2711 // 2712 // Copies over block information for each block in the heap into the provided buffer 2713 // 2714 NV_STATUS heapGetBlockInfo_IMPL 2715 ( 2716 Heap *pHeap, 2717 NvU32 size, 2718 NVOS32_HEAP_DUMP_BLOCK *pBlockBuffer 2719 ) 2720 { 2721 MEM_BLOCK *pBlock; 2722 NvU32 heapSize, i; 2723 NV_STATUS rmStatus = NV_OK; 2724 2725 // ensure buffer is the same size 2726 heapSize = heapGetNumBlocks(pHeap); 2727 NV_ASSERT_OR_RETURN(heapSize == size, NV_ERR_INVALID_ARGUMENT); 2728 2729 pBlock = pHeap->pBlockList; 2730 for (i=0; i<heapSize; i++) 2731 { 2732 pBlockBuffer->begin = pBlock->begin; 2733 pBlockBuffer->align = pBlock->align; 2734 pBlockBuffer->end = pBlock->end; 2735 pBlockBuffer->owner = pBlock->owner; 2736 pBlockBuffer->format = pBlock->format; 2737 pBlock = pBlock->next; 2738 pBlockBuffer++; 2739 } 2740 2741 return rmStatus; 2742 } 2743 2744 NV_STATUS heapAllocHint_IMPL 2745 ( 2746 OBJGPU *pGpu, 2747 Heap *pHeap, 2748 NvHandle hClient, 2749 NvHandle hDevice, 2750 HEAP_ALLOC_HINT_PARAMS *pAllocHint 2751 ) 2752 { 2753 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 2754 NvU64 alignment; 2755 NV_STATUS status; 2756 NvBool ignoreBankPlacement; 2757 NvU32 textureClientIndex = 0xFFFFFFFF; 2758 NvU32 bankPlacement = 0; 2759 NvU8 currentBankInfo = 0; 2760 FB_ALLOC_INFO *pFbAllocInfo = NULL; 2761 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 2762 NvU64 pageSize = 0; 2763 NvU32 flags; 2764 NvU32 owner; 2765 2766 // Check for valid size. 2767 NV_ASSERT_OR_RETURN((pAllocHint->pSize != NULL), NV_ERR_INVALID_ARGUMENT); 2768 2769 // Ensure a valid allocation type was passed in 2770 NV_ASSERT_OR_RETURN((pAllocHint->type < NVOS32_NUM_MEM_TYPES), NV_ERR_INVALID_ARGUMENT); 2771 2772 // As we will dereference these two later, we should not allow NULL value. 2773 NV_ASSERT_OR_RETURN(((pAllocHint->pHeight != NULL) && (pAllocHint->pAttr != NULL)), NV_ERR_INVALID_ARGUMENT); 2774 2775 owner = 0x0; 2776 status = _heapGetBankPlacement(pGpu, pHeap, owner, 2777 &pAllocHint->flags, pAllocHint->type, 2778 0x0, &bankPlacement); 2779 if (status != NV_OK) 2780 { 2781 NV_PRINTF(LEVEL_ERROR, 2782 "_heapGetBankPlacement failed for current allocation\n"); 2783 goto exit; 2784 } 2785 2786 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 2787 if (pFbAllocInfo == NULL) 2788 { 2789 NV_ASSERT(0); 2790 status = NV_ERR_NO_MEMORY; 2791 goto exit; 2792 } 2793 2794 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 2795 if (pFbAllocPageFormat == NULL) { 2796 NV_ASSERT(0); 2797 status = NV_ERR_NO_MEMORY; 2798 goto exit; 2799 } 2800 2801 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 2802 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 2803 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 2804 2805 pFbAllocInfo->pageFormat->type = pAllocHint->type; 2806 pFbAllocInfo->hwResId = 0; 2807 pFbAllocInfo->pad = 0; 2808 pFbAllocInfo->height = *pAllocHint->pHeight; 2809 pFbAllocInfo->width = *pAllocHint->pWidth; 2810 pFbAllocInfo->pitch = (pAllocHint->pPitch) ? (*pAllocHint->pPitch) : 0; 2811 pFbAllocInfo->size = *pAllocHint->pSize; 2812 pFbAllocInfo->pageFormat->kind = 0; 2813 pFbAllocInfo->offset = ~0; 2814 pFbAllocInfo->hClient = hClient; 2815 pFbAllocInfo->hDevice = hDevice; 2816 pFbAllocInfo->pageFormat->flags = pAllocHint->flags; 2817 pFbAllocInfo->pageFormat->attr = *pAllocHint->pAttr; 2818 pFbAllocInfo->retAttr = *pAllocHint->pAttr; 2819 pFbAllocInfo->pageFormat->attr2 = *pAllocHint->pAttr2; 2820 pFbAllocInfo->retAttr2 = *pAllocHint->pAttr2; 2821 pFbAllocInfo->format = pAllocHint->format; 2822 2823 if ((pAllocHint->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_HINT) || 2824 (pAllocHint->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE)) 2825 pFbAllocInfo->align = *pAllocHint->pAlignment; 2826 else 2827 pFbAllocInfo->align = RM_PAGE_SIZE; 2828 2829 // Fetch RM page size 2830 pageSize = memmgrDeterminePageSize(pMemoryManager, pFbAllocInfo->hClient, pFbAllocInfo->size, 2831 pFbAllocInfo->format, pFbAllocInfo->pageFormat->flags, 2832 &pFbAllocInfo->retAttr, &pFbAllocInfo->retAttr2); 2833 if (pageSize == 0) 2834 { 2835 status = NV_ERR_INVALID_STATE; 2836 NV_PRINTF(LEVEL_ERROR, "memmgrDeterminePageSize failed, status: 0x%x\n", status); 2837 goto exit; 2838 } 2839 2840 // Fetch memory alignment 2841 status = memmgrAllocDetermineAlignment_HAL(pGpu, pMemoryManager, &pFbAllocInfo->size, &pFbAllocInfo->align, 2842 pFbAllocInfo->alignPad, pFbAllocInfo->pageFormat->flags, 2843 pFbAllocInfo->retAttr, pFbAllocInfo->retAttr2, 0); 2844 if (status != NV_OK) 2845 { 2846 NV_PRINTF(LEVEL_ERROR, "memmgrAllocDetermineAlignment failed, status: 0x%x\n", status); 2847 goto exit; 2848 } 2849 2850 // 2851 // Call into HAL to reserve any hardware resources for 2852 // the specified memory type. 2853 // If the alignment was changed due to a HW limitation, and the 2854 // flag NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE is set, bad_argument 2855 // will be passed back from the HAL 2856 // 2857 flags = pFbAllocInfo->pageFormat->flags; 2858 pFbAllocInfo->pageFormat->flags |= NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC; 2859 status = memmgrAllocHwResources(pGpu, pMemoryManager, pFbAllocInfo); 2860 pFbAllocInfo->pageFormat->flags = flags; 2861 *pAllocHint->pAttr = pFbAllocInfo->retAttr; 2862 *pAllocHint->pAttr2 = pFbAllocInfo->retAttr2; 2863 *pAllocHint->pKind = pFbAllocInfo->pageFormat->kind; 2864 2865 // Save retAttr as Possible Attributes that have passed error checking and 2866 // clear retAttr because we have not allocated them yet 2867 pFbAllocInfo->possAttr = pFbAllocInfo->retAttr; 2868 // pFbAllocInfo->possAttr2 = pFbAllocInfo->retAttr2; 2869 pFbAllocInfo->retAttr = 0x0; 2870 pFbAllocInfo->retAttr2 = 0x0; 2871 if (status != NV_OK) 2872 { 2873 // 2874 // probably means we passed in a bogus type or no tiling resources available 2875 // when tiled memory attribute was set to REQUIRED 2876 // 2877 NV_PRINTF(LEVEL_ERROR, "memmgrAllocHwResources failed, status: 0x%x\n", 2878 status); 2879 goto exit; 2880 } 2881 2882 // 2883 // Refresh search parameters. 2884 // 2885 if ((DRF_VAL(OS32, _ATTR, _FORMAT, *pAllocHint->pAttr) != NVOS32_ATTR_FORMAT_BLOCK_LINEAR)) 2886 { 2887 *pAllocHint->pHeight = pFbAllocInfo->height; 2888 if (pAllocHint->pPitch) 2889 *pAllocHint->pPitch = pFbAllocInfo->pitch; 2890 } 2891 2892 // 2893 // The heap allocator has assumed required alignments are powers of 2 2894 // (aligning FB offsets has been done using bit masks). 2895 // 2896 // 2897 *pAllocHint->pAlignment = pFbAllocInfo->align + 1; // convert mask to size 2898 alignment = pFbAllocInfo->align + 1; 2899 2900 // 2901 // Allow caller to request host page alignment to make it easier 2902 // to move things around with host os VM subsystem 2903 // 2904 2905 if (pAllocHint->flags & NVOS32_ALLOC_FLAGS_FORCE_ALIGN_HOST_PAGE) 2906 { 2907 OBJSYS *pSys = SYS_GET_INSTANCE(); 2908 NvU64 hostPageSize = pSys->cpuInfo.hostPageSize; 2909 2910 // hostPageSize *should* always be set, but.... 2911 if (hostPageSize == 0) 2912 hostPageSize = RM_PAGE_SIZE; 2913 2914 alignment = memUtilsLeastCommonAlignment(alignment, hostPageSize); 2915 } 2916 2917 if (memmgrAllocGetAddrSpace(pMemoryManager, pAllocHint->flags, *pAllocHint->pAttr) == ADDR_FBMEM) 2918 { 2919 if (alignment >= pHeap->total) 2920 { 2921 status = NV_ERR_INVALID_ARGUMENT; 2922 NV_PRINTF(LEVEL_ERROR, "heapAllocHint failed due to alignmend >= pHeap->total\n"); 2923 goto exit; 2924 } 2925 } 2926 2927 // 2928 // Check if NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT was passed in with 2929 // the type to ignore placing this allocation in a particular bank. 2930 // This means we default to the second loop where we choose first fit. 2931 // 2932 ignoreBankPlacement = NV_FALSE; 2933 if (pAllocHint->flags & NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT) 2934 ignoreBankPlacement = NV_TRUE; 2935 2936 if ((pAllocHint->type == NVOS32_TYPE_TEXTURE) && (!pAllocHint->flags)) 2937 _heapSetTexturePlacement(pHeap, pAllocHint->client, pAllocHint->type, &ignoreBankPlacement, &textureClientIndex, ¤tBankInfo); 2938 2939 pAllocHint->bankPlacement = bankPlacement; 2940 pAllocHint->ignoreBankPlacement = ignoreBankPlacement; 2941 2942 *pAllocHint->pHeight = pFbAllocInfo->height; 2943 pAllocHint->pad = pFbAllocInfo->pad; 2944 2945 *pAllocHint->pSize = pFbAllocInfo->size; // returned to caller 2946 2947 pAllocHint->alignAdjust = 0; 2948 2949 exit: 2950 portMemFree(pFbAllocPageFormat); 2951 portMemFree(pFbAllocInfo); 2952 2953 return status; 2954 } 2955 2956 NV_STATUS heapHwAlloc_IMPL 2957 ( 2958 OBJGPU *pGpu, 2959 Heap *pHeap, 2960 NvHandle hClient, 2961 NvHandle hDevice, 2962 NvHandle hMemory, 2963 MEMORY_HW_RESOURCES_ALLOCATION_REQUEST *pHwAlloc, 2964 NvU32 *pAttr, 2965 NvU32 *pAttr2 2966 ) 2967 { 2968 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 2969 NV_STATUS status = NV_OK; 2970 FB_ALLOC_INFO *pFbAllocInfo = NULL; 2971 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 2972 NvU64 pageSize = 0; 2973 NV_MEMORY_HW_RESOURCES_ALLOCATION_PARAMS *pUserParams = pHwAlloc->pUserParams; 2974 2975 // Ensure a valid allocation type was passed in 2976 if (pUserParams->type > NVOS32_NUM_MEM_TYPES - 1) 2977 return NV_ERR_GENERIC; 2978 2979 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 2980 if (NULL == pFbAllocInfo) 2981 { 2982 NV_PRINTF(LEVEL_ERROR, "No memory for Resource %p\n", 2983 pHwAlloc->pHandle); 2984 status = NV_ERR_GENERIC; 2985 goto failed; 2986 } 2987 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 2988 if (NULL == pFbAllocPageFormat) 2989 { 2990 NV_PRINTF(LEVEL_ERROR, "No memory for Resource %p\n", 2991 pHwAlloc->pHandle); 2992 status = NV_ERR_GENERIC; 2993 goto failed; 2994 } 2995 2996 portMemSet(pFbAllocInfo, 0x0, sizeof(FB_ALLOC_INFO)); 2997 portMemSet(pFbAllocPageFormat, 0x0, sizeof(FB_ALLOC_PAGE_FORMAT)); 2998 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 2999 pFbAllocInfo->pageFormat->type = pUserParams->type; 3000 pFbAllocInfo->hwResId = 0; 3001 pFbAllocInfo->pad = 0; 3002 pFbAllocInfo->height = pUserParams->height; 3003 pFbAllocInfo->width = pUserParams->width; 3004 pFbAllocInfo->pitch = pUserParams->pitch; 3005 pFbAllocInfo->size = pUserParams->size; 3006 pFbAllocInfo->origSize = pUserParams->size; 3007 pFbAllocInfo->pageFormat->kind = pUserParams->kind; 3008 pFbAllocInfo->offset = memmgrGetInvalidOffset_HAL(pGpu, pMemoryManager); 3009 pFbAllocInfo->hClient = hClient; 3010 pFbAllocInfo->hDevice = hDevice; 3011 pFbAllocInfo->pageFormat->flags = pUserParams->flags; 3012 pFbAllocInfo->pageFormat->attr = pUserParams->attr; 3013 pFbAllocInfo->pageFormat->attr2 = pUserParams->attr2; 3014 pFbAllocInfo->retAttr = pUserParams->attr; 3015 pFbAllocInfo->retAttr2 = pUserParams->attr2; 3016 pFbAllocInfo->comprCovg = pUserParams->comprCovg; 3017 pFbAllocInfo->zcullCovg = 0; 3018 pFbAllocInfo->internalflags = 0; 3019 3020 if ((pUserParams->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_HINT) || 3021 (pUserParams->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE)) 3022 pFbAllocInfo->align = pUserParams->alignment; 3023 else 3024 pFbAllocInfo->align = RM_PAGE_SIZE; 3025 3026 // Fetch RM page size 3027 pageSize = memmgrDeterminePageSize(pMemoryManager, pFbAllocInfo->hClient, pFbAllocInfo->size, 3028 pFbAllocInfo->format, pFbAllocInfo->pageFormat->flags, 3029 &pFbAllocInfo->retAttr, &pFbAllocInfo->retAttr2); 3030 if (pageSize == 0) 3031 { 3032 status = NV_ERR_INVALID_STATE; 3033 NV_PRINTF(LEVEL_ERROR, "memmgrDeterminePageSize failed\n"); 3034 } 3035 3036 // Fetch memory alignment 3037 status = memmgrAllocDetermineAlignment_HAL(pGpu, pMemoryManager, &pFbAllocInfo->size, &pFbAllocInfo->align, 3038 pFbAllocInfo->alignPad, pFbAllocInfo->pageFormat->flags, 3039 pFbAllocInfo->retAttr, pFbAllocInfo->retAttr2, 0); 3040 if (status != NV_OK) 3041 { 3042 NV_PRINTF(LEVEL_ERROR, "memmgrAllocDetermineAlignment failed\n"); 3043 } 3044 3045 // 3046 // vGPU: 3047 // 3048 // Since vGPU does all real hardware management in the 3049 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3050 // do an RPC to the host to do the hardware update. 3051 // 3052 if ((status == NV_OK) && IS_VIRTUAL(pGpu)) 3053 { 3054 { 3055 NV_RM_RPC_MANAGE_HW_RESOURCE_ALLOC(pGpu, 3056 hClient, 3057 hDevice, 3058 hMemory, 3059 pFbAllocInfo, 3060 status); 3061 pHwAlloc->hwResource.isVgpuHostAllocated = NV_TRUE; 3062 } 3063 3064 pUserParams->uncompressedKind = pFbAllocInfo->uncompressedKind; 3065 pUserParams->compPageShift = pFbAllocInfo->compPageShift; 3066 pUserParams->compressedKind = pFbAllocInfo->compressedKind; 3067 pUserParams->compTagLineMin = pFbAllocInfo->compTagLineMin; 3068 pUserParams->compPageIndexLo = pFbAllocInfo->compPageIndexLo; 3069 pUserParams->compPageIndexHi = pFbAllocInfo->compPageIndexHi; 3070 pUserParams->compTagLineMultiplier = pFbAllocInfo->compTagLineMultiplier; 3071 } 3072 else 3073 { 3074 // 3075 // Call into HAL to reserve any hardware resources for 3076 // the specified memory type. 3077 // If the alignment was changed due to a HW limitation, and the 3078 // flag NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE is set, bad_argument 3079 // will be passed back from the HAL 3080 // 3081 status = memmgrAllocHwResources(pGpu, pMemoryManager, pFbAllocInfo); 3082 } 3083 3084 // Is status bad or did we request attributes and they failed 3085 if ((status != NV_OK) || ((pUserParams->attr) && (0x0 == pFbAllocInfo->retAttr))) 3086 { 3087 // 3088 // probably means we passed in a bogus type or no tiling resources available 3089 // when tiled memory attribute was set to REQUIRED 3090 // 3091 if (status != NV_OK) 3092 { 3093 NV_PRINTF(LEVEL_ERROR, 3094 "nvHalFbAlloc failure status = 0x%x Requested Attr 0x%x!\n", 3095 status, pUserParams->attr); 3096 } 3097 else 3098 { 3099 NV_PRINTF(LEVEL_WARNING, 3100 "nvHalFbAlloc Out of Resources Requested=%x Returned=%x !\n", 3101 pUserParams->attr, pFbAllocInfo->retAttr); 3102 } 3103 goto failed; 3104 } 3105 3106 // 3107 // Refresh search parameters. 3108 // 3109 pUserParams->pitch = pFbAllocInfo->pitch; 3110 3111 pUserParams->height = pFbAllocInfo->height; 3112 pHwAlloc->pad = NvU64_LO32(pFbAllocInfo->pad); 3113 pUserParams->kind = pFbAllocInfo->pageFormat->kind; 3114 pHwAlloc->hwResId = pFbAllocInfo->hwResId; 3115 3116 pUserParams->size = pFbAllocInfo->size; // returned to caller 3117 3118 pHwAlloc->hwResource.attr = pFbAllocInfo->retAttr; 3119 pHwAlloc->hwResource.attr2 = pFbAllocInfo->retAttr2; 3120 pHwAlloc->hwResource.comprCovg = pFbAllocInfo->comprCovg; 3121 pHwAlloc->hwResource.ctagOffset = pFbAllocInfo->ctagOffset; 3122 pHwAlloc->hwResource.hwResId = pFbAllocInfo->hwResId; 3123 3124 *pAttr = pFbAllocInfo->retAttr; 3125 *pAttr2 = pFbAllocInfo->retAttr2; 3126 3127 failed: 3128 portMemFree(pFbAllocPageFormat); 3129 portMemFree(pFbAllocInfo); 3130 3131 return status; 3132 } 3133 3134 void heapHwFree_IMPL 3135 ( 3136 OBJGPU *pGpu, 3137 Heap *pHeap, 3138 Memory *pMemory, 3139 NvU32 flags 3140 ) 3141 { 3142 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 3143 FB_ALLOC_INFO *pFbAllocInfo = NULL; 3144 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 3145 3146 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 3147 if (pFbAllocInfo == NULL) 3148 { 3149 NV_ASSERT(0); 3150 goto exit; 3151 } 3152 3153 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 3154 if (pFbAllocPageFormat == NULL) { 3155 NV_ASSERT(0); 3156 goto exit; 3157 } 3158 3159 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 3160 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 3161 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 3162 3163 pFbAllocInfo->pageFormat->type = pMemory->Type; 3164 pFbAllocInfo->pageFormat->attr = pMemory->pHwResource->attr; 3165 pFbAllocInfo->pageFormat->attr2 = pMemory->pHwResource->attr2; 3166 pFbAllocInfo->hwResId = pMemory->pHwResource->hwResId; 3167 pFbAllocInfo->size = pMemory->Length; 3168 pFbAllocInfo->format = memdescGetPteKind(pMemory->pMemDesc); 3169 pFbAllocInfo->offset = ~0; 3170 pFbAllocInfo->hClient = RES_GET_CLIENT_HANDLE(pMemory); 3171 pFbAllocInfo->hDevice = RES_GET_HANDLE(pMemory->pDevice); 3172 3173 // 3174 // vGPU: 3175 // 3176 // Since vGPU does all real hardware management in the 3177 // host, if we are in guest OS (where IS_VIRTUAL(pGpu) is true), 3178 // do an RPC to the host to do the hardware update. 3179 // 3180 3181 if (IS_VIRTUAL(pGpu)) 3182 { 3183 { 3184 NV_STATUS rmStatus = NV_OK; 3185 3186 NV_RM_RPC_MANAGE_HW_RESOURCE_FREE(pGpu, 3187 RES_GET_CLIENT_HANDLE(pMemory), 3188 RES_GET_HANDLE(pMemory->pDevice), 3189 RES_GET_HANDLE(pMemory), 3190 flags, 3191 rmStatus); 3192 } 3193 } 3194 else 3195 { 3196 memmgrFreeHwResources(pGpu, pMemoryManager, pFbAllocInfo); 3197 } 3198 3199 exit: 3200 portMemFree(pFbAllocPageFormat); 3201 portMemFree(pFbAllocInfo); 3202 } 3203 3204 NV_STATUS heapFreeBlockCount_IMPL(OBJGPU *pGpu, Heap *pHeap, NvU32 *pCount) 3205 { 3206 MEM_BLOCK *pMemBlock; 3207 3208 pMemBlock = pHeap->pFreeBlockList; 3209 *pCount = 0; 3210 3211 if (pMemBlock == NULL) 3212 { 3213 return NV_OK; 3214 } 3215 3216 do 3217 { 3218 (*pCount)++; 3219 pMemBlock = pMemBlock->u1.nextFree; 3220 } while (pMemBlock != pHeap->pFreeBlockList); 3221 3222 return NV_OK; 3223 } 3224 3225 NV_STATUS heapFreeBlockInfo_IMPL(OBJGPU *pGpu, Heap *pHeap, NvU32 Count, void *pVoidInfo) 3226 { 3227 NVOS32_BLOCKINFO *pBlockInfo = pVoidInfo; 3228 NvU32 actualCount; 3229 MEM_BLOCK *pMemBlock; 3230 NV_STATUS rmStatus = NV_ERR_GENERIC; 3231 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 3232 NvU64 maxCpuOffset; 3233 3234 heapFreeBlockCount(pGpu, pHeap, &actualCount); 3235 3236 if ((actualCount == Count) && (NULL != pBlockInfo)) 3237 { 3238 if (actualCount == 0) 3239 { 3240 return NV_OK; 3241 } 3242 3243 maxCpuOffset = (pMemoryManager->Ram.mapRamSizeMb*0x100000) - 1; 3244 pMemBlock = pHeap->pFreeBlockList; 3245 actualCount = 0; 3246 do 3247 { 3248 pBlockInfo->startOffset = pMemBlock->begin; 3249 pBlockInfo->size = pMemBlock->end - pMemBlock->begin + 1; 3250 pBlockInfo->flags = 0x0; 3251 if (pBlockInfo->startOffset < maxCpuOffset) 3252 { 3253 pBlockInfo->flags |= NVOS32_FLAGS_BLOCKINFO_VISIBILITY_CPU; 3254 } 3255 pMemBlock = pMemBlock->u1.nextFree; 3256 pBlockInfo++; 3257 actualCount++; 3258 } while ((pMemBlock != pHeap->pFreeBlockList) && (actualCount < Count)); 3259 3260 rmStatus = NV_OK; 3261 } 3262 3263 return rmStatus; 3264 } 3265 3266 /*! 3267 * @brief: Adjust heap free accounting 3268 * 3269 * @param[in] pHeap Heap pointer 3270 * @param[in] blockSize +: Size of block being freed 3271 * -: Size of block being allocated 3272 * @param[in] internalHeap NV_TRUE if the allocation is 'INTERNAL' 3273 * 3274 * @return void 3275 */ 3276 3277 static void 3278 _heapAdjustFree 3279 ( 3280 Heap *pHeap, 3281 NvS64 blockSize, 3282 NvBool internalHeap 3283 ) 3284 { 3285 pHeap->free += blockSize; 3286 3287 NV_ASSERT(pHeap->free <= pHeap->total); 3288 if(pHeap->free > pHeap->total) 3289 { 3290 DBG_BREAKPOINT(); 3291 } 3292 3293 // Collect data on internal/external heap usage 3294 if (internalHeap) 3295 { 3296 pHeap->currInternalUsage -= blockSize; 3297 pHeap->peakInternalUsage = NV_MAX(pHeap->peakInternalUsage, pHeap->currInternalUsage); 3298 } 3299 else 3300 { 3301 pHeap->currExternalUsage -= blockSize; 3302 pHeap->peakExternalUsage = NV_MAX(pHeap->peakExternalUsage, pHeap->currExternalUsage); 3303 } 3304 } 3305 3306 static NV_STATUS 3307 _heapProcessFreeBlock 3308 ( 3309 OBJGPU *pGpu, 3310 MEM_BLOCK *pBlockFree, 3311 MEM_BLOCK **ppBlockNew, 3312 MEM_BLOCK **ppBlockSplit, 3313 Heap *pHeap, 3314 MEMORY_ALLOCATION_REQUEST *pAllocRequest, 3315 NvHandle memHandle, 3316 OBJHEAP_ALLOC_DATA *pAllocData, 3317 FB_ALLOC_INFO *pFbAllocInfo, 3318 NvU64 alignPad, 3319 NvU64 *offset 3320 ) 3321 { 3322 NV_MEMORY_ALLOCATION_PARAMS *pVidHeapAlloc = pAllocRequest->pUserParams; 3323 MEM_BLOCK *pBlockNew = NULL, *pBlockSplit = NULL; 3324 OBJOS *pOS = GPU_GET_OS(pGpu); 3325 NV_STATUS status = NV_OK; 3326 3327 if ((pAllocData->allocLo == pBlockFree->begin) && 3328 (pAllocData->allocHi == pBlockFree->end)) 3329 { 3330 // 3331 // Wow, exact match so replace free block. 3332 // Remove from free list. 3333 // 3334 pBlockFree->u1.nextFree->u0.prevFree = pBlockFree->u0.prevFree; 3335 pBlockFree->u0.prevFree->u1.nextFree = pBlockFree->u1.nextFree; 3336 3337 if (pHeap->pFreeBlockList == pBlockFree) 3338 { 3339 // 3340 // This could be the last free block. 3341 // 3342 if (pBlockFree->u1.nextFree == pBlockFree) 3343 pHeap->pFreeBlockList = NULL; 3344 else 3345 pHeap->pFreeBlockList = pBlockFree->u1.nextFree; 3346 } 3347 3348 // 3349 // Set pVidHeapAlloc->owner/pVidHeapAlloc->type values here. 3350 // Don't move because some fields are unions. 3351 // 3352 pBlockFree->owner = pVidHeapAlloc->owner; 3353 pBlockFree->mhandle = memHandle; 3354 pBlockFree->refCount = 1; 3355 pBlockFree->u0.type = pVidHeapAlloc->type; 3356 pBlockFree->align = pAllocData->allocAl; 3357 pBlockFree->alignPad = alignPad; 3358 pBlockFree->format = pFbAllocInfo->format; 3359 3360 // tail end code below assumes 'blockNew' is the new block 3361 pBlockNew = pBlockFree; 3362 } 3363 else if ((pAllocData->allocLo >= pBlockFree->begin) && 3364 (pAllocData->allocHi <= pBlockFree->end)) 3365 { 3366 // 3367 // Found a fit. 3368 // It isn't exact, so we'll have to do a split 3369 // 3370 pBlockNew = portMemAllocNonPaged(sizeof(MEM_BLOCK)); 3371 if (pBlockNew == NULL) 3372 { 3373 // Exit with failure and free any local allocations 3374 NV_ASSERT(0); 3375 status = NV_ERR_NO_MEMORY; 3376 goto _heapProcessFreeBlock_error; 3377 } 3378 3379 portMemSet(pBlockNew, 0, sizeof(MEM_BLOCK)); 3380 3381 pBlockNew->owner = pVidHeapAlloc->owner; 3382 pBlockNew->mhandle = memHandle; 3383 pBlockNew->refCount = 1; 3384 pBlockNew->u0.type = pVidHeapAlloc->type; 3385 pBlockNew->begin = pAllocData->allocLo; 3386 pBlockNew->align = pAllocData->allocAl; 3387 pBlockNew->alignPad = alignPad; 3388 pBlockNew->end = pAllocData->allocHi; 3389 pBlockNew->format = pFbAllocInfo->format; 3390 3391 if (gpuIsCacheOnlyModeEnabled(pGpu)) 3392 { 3393 // 3394 // In L2 Cache only mode, set the beginning of the new allocation 3395 // block to aligned (allocAl) offset rather then the start of 3396 // the free block (allocLo). And that the end of the new block is 3397 // is calculated as (allocSize - 1) from the beginning. 3398 // This insures that we don't "over allocate" for the surface in the 3399 // case where start of the free block is not properly aligned for both 3400 // the grow down and grow up cases. 3401 // Only applying this in L2 cache mode for now, as we don't want to "waste" 3402 // L2 cache space, though wonder if there are any implications to doing 3403 // it this way in normal operation. 3404 // 3405 pBlockNew->begin = pAllocData->allocAl; 3406 pBlockNew->end = pBlockNew->begin + pAllocData->allocSize - 1; 3407 } 3408 3409 if ((pBlockFree->begin < pBlockNew->begin) && 3410 (pBlockFree->end > pBlockNew->end)) 3411 { 3412 // Split free block in two. 3413 pBlockSplit = portMemAllocNonPaged(sizeof(MEM_BLOCK)); 3414 if (pBlockSplit == NULL) 3415 { 3416 // Exit with failure and free any local allocations 3417 status = NV_ERR_NO_MEMORY; 3418 goto _heapProcessFreeBlock_error; 3419 } 3420 3421 portMemSet(pBlockSplit, 0, sizeof(MEM_BLOCK)); 3422 3423 // remove free block from rb-tree since node's range will be changed 3424 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_REMOVE))) 3425 { 3426 // Exit with failure and free any local allocations 3427 goto _heapProcessFreeBlock_error; 3428 } 3429 3430 pBlockSplit->owner = NVOS32_BLOCK_TYPE_FREE; 3431 pBlockSplit->format= 0; 3432 pBlockSplit->begin = pBlockNew->end + 1; 3433 pBlockSplit->align = pBlockSplit->begin; 3434 pBlockSplit->alignPad = 0; 3435 pBlockSplit->end = pBlockFree->end; 3436 pBlockFree->end = pBlockNew->begin - 1; 3437 // 3438 // Insert free split block into free list. 3439 // 3440 pBlockSplit->u1.nextFree = pBlockFree->u1.nextFree; 3441 pBlockSplit->u0.prevFree = pBlockFree; 3442 pBlockSplit->u1.nextFree->u0.prevFree = pBlockSplit; 3443 pBlockFree->u1.nextFree = pBlockSplit; 3444 // 3445 // Insert new and split blocks into block list. 3446 // 3447 pBlockNew->next = pBlockSplit; 3448 pBlockNew->prev = pBlockFree; 3449 pBlockSplit->next = pBlockFree->next; 3450 pBlockSplit->prev = pBlockNew; 3451 pBlockFree->next = pBlockNew; 3452 pBlockSplit->next->prev = pBlockSplit; 3453 3454 // update numBlocks count 3455 pHeap->numBlocks++; 3456 3457 // re-insert updated free block into rb-tree 3458 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_SIZE_CHANGED))) 3459 { 3460 // 3461 // Exit and report success. The new block was allocated, but the 3462 // noncontig info is now out-of-sync with reality. 3463 // 3464 NV_PRINTF(LEVEL_ERROR, 3465 "_heapUpdate failed to _SIZE_CHANGE block\n"); 3466 goto _heapProcessFreeBlock_exit; 3467 } 3468 3469 // insert new and split blocks into rb-tree 3470 if (NV_OK != (status = _heapUpdate(pHeap, pBlockNew, BLOCK_ADD))) 3471 { 3472 // 3473 // Exit and report success. The new block was allocated, but the 3474 // noncontig info is now out-of-sync with reality. 3475 // 3476 NV_PRINTF(LEVEL_ERROR, "_heapUpdate failed to _ADD block\n"); 3477 goto _heapProcessFreeBlock_exit; 3478 } 3479 3480 if (NV_OK != (status = _heapUpdate(pHeap, pBlockSplit, BLOCK_ADD))) 3481 { 3482 // 3483 // Exit and report success. The new block was allocated, but the 3484 // noncontig info is now out-of-sync with reality. 3485 // 3486 NV_PRINTF(LEVEL_ERROR, "_heapUpdate failed to _ADD block\n"); 3487 goto _heapProcessFreeBlock_exit; 3488 } 3489 } 3490 else if (pBlockFree->end == pBlockNew->end) 3491 { 3492 // remove free block from rb-tree since node's range will be changed 3493 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_REMOVE))) 3494 { 3495 // Exit with failure and free any local allocations 3496 goto _heapProcessFreeBlock_error; 3497 } 3498 3499 // 3500 // New block inserted after free block. 3501 // 3502 pBlockFree->end = pBlockNew->begin - 1; 3503 pBlockNew->next = pBlockFree->next; 3504 pBlockNew->prev = pBlockFree; 3505 pBlockFree->next->prev = pBlockNew; 3506 pBlockFree->next = pBlockNew; 3507 3508 // re-insert updated free block into rb-tree 3509 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_SIZE_CHANGED))) 3510 { 3511 // 3512 // Exit and report success. The new block was allocated, but the 3513 // noncontig info is now out-of-sync with reality. 3514 // 3515 NV_PRINTF(LEVEL_ERROR, 3516 "_heapUpdate failed to _SIZE_CHANGE block\n"); 3517 goto _heapProcessFreeBlock_exit; 3518 } 3519 3520 // insert new block into rb-tree 3521 if (NV_OK != (status = _heapUpdate(pHeap, pBlockNew, BLOCK_ADD))) 3522 { 3523 // 3524 // Exit and report success. The new block was allocated, but the 3525 // noncontig info is now out-of-sync with reality. 3526 // 3527 NV_PRINTF(LEVEL_ERROR, "_heapUpdate failed to _ADD block\n"); 3528 goto _heapProcessFreeBlock_exit; 3529 } 3530 } 3531 else if (pBlockFree->begin == pBlockNew->begin) 3532 { 3533 // remove free block from rb-tree since node's range will be changed 3534 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_REMOVE))) 3535 { 3536 // Exit with failure and free any local allocations 3537 goto _heapProcessFreeBlock_error; 3538 } 3539 3540 // 3541 // New block inserted before free block. 3542 // 3543 pBlockFree->begin = pBlockNew->end + 1; 3544 pBlockFree->align = pBlockFree->begin; 3545 pBlockNew->next = pBlockFree; 3546 pBlockNew->prev = pBlockFree->prev; 3547 pBlockFree->prev->next = pBlockNew; 3548 pBlockFree->prev = pBlockNew; 3549 if (pHeap->pBlockList == pBlockFree) 3550 pHeap->pBlockList = pBlockNew; 3551 3552 // re-insert updated free block into rb-tree 3553 if (NV_OK != (status = _heapUpdate(pHeap, pBlockFree, BLOCK_SIZE_CHANGED))) 3554 { 3555 // 3556 // Exit and report success. The new block was allocated, but the 3557 // noncontig info is now out-of-sync with reality. 3558 // 3559 NV_PRINTF(LEVEL_ERROR, 3560 "_heapUpdate failed to _SIZE_CHANGE block\n"); 3561 goto _heapProcessFreeBlock_exit; 3562 } 3563 3564 // insert new block into rb-tree 3565 if (NV_OK != (status = _heapUpdate(pHeap, pBlockNew, BLOCK_ADD))) 3566 { 3567 // 3568 // Exit and report success. The new block was allocated, but the 3569 // noncontig info is now out-of-sync with reality. 3570 // 3571 NV_PRINTF(LEVEL_ERROR, "_heapUpdate failed to _ADD block\n"); 3572 goto _heapProcessFreeBlock_exit; 3573 } 3574 } 3575 else 3576 { 3577 status = NV_ERR_NO_MEMORY; 3578 // Exit with failure and free any local allocations 3579 goto _heapProcessFreeBlock_error; 3580 } 3581 3582 pHeap->numBlocks++; 3583 } 3584 3585 if (NULL == pBlockNew) 3586 status = NV_ERR_NO_MEMORY; 3587 3588 _heapProcessFreeBlock_error: 3589 if (status != NV_OK) 3590 { 3591 NV_PRINTF(LEVEL_ERROR, "failed to allocate block\n"); 3592 3593 portMemFree(pBlockNew); 3594 portMemFree(pBlockSplit); 3595 3596 *ppBlockNew = NULL; 3597 *ppBlockSplit = NULL; 3598 3599 return status; 3600 } 3601 3602 _heapProcessFreeBlock_exit: 3603 *ppBlockNew = pBlockNew; 3604 *ppBlockSplit = pBlockSplit; 3605 3606 // alignPad == 0 for all but >= NV5x 3607 *offset = pBlockNew->align + pBlockNew->alignPad; 3608 3609 // Reduce free amount by allocated block size. 3610 _heapAdjustFree(pHeap, -((NvS64) (pBlockNew->end - pBlockNew->begin + 1)), 3611 FLD_TEST_DRF(OS32, _ATTR2, _INTERNAL, _YES, pFbAllocInfo->pageFormat->attr2)); 3612 3613 if (FLD_TEST_DRF(OS32, _ATTR2, _INTERNAL, _YES, pFbAllocInfo->pageFormat->attr2)) 3614 { 3615 pOS->osInternalReserveAllocCallback(*offset, pFbAllocInfo->size, pGpu->gpuId); 3616 } 3617 3618 return NV_OK; 3619 } 3620 3621 static void 3622 _heapAddBlockToNoncontigList 3623 ( 3624 Heap *pHeap, 3625 MEM_BLOCK *pBlock 3626 ) 3627 { 3628 if (NULL == pHeap->pNoncontigFreeBlockList) 3629 { 3630 pHeap->pNoncontigFreeBlockList = pBlock; 3631 pBlock->nextFreeNoncontig = NULL; 3632 pBlock->prevFreeNoncontig = NULL; 3633 } 3634 else 3635 { 3636 MEM_BLOCK *pNextBlock = pHeap->pNoncontigFreeBlockList; 3637 NvU64 size, nextSize = 0; 3638 size = pBlock->end - pBlock->begin + 1; 3639 3640 NV_ASSERT(pBlock->prevFreeNoncontig == NULL && 3641 pBlock->nextFreeNoncontig == NULL); 3642 3643 // The noncontig block list is arranged in the descending order of size 3644 while (NULL != pNextBlock) 3645 { 3646 nextSize = pNextBlock->end - pNextBlock->begin + 1; 3647 3648 if (size > nextSize) 3649 { 3650 // Insert pBlock in front of pNextBlock 3651 pBlock->prevFreeNoncontig = pNextBlock->prevFreeNoncontig; 3652 pBlock->nextFreeNoncontig = pNextBlock; 3653 pNextBlock->prevFreeNoncontig = pBlock; 3654 3655 if (pHeap->pNoncontigFreeBlockList == pNextBlock) 3656 { 3657 // We inserted at the head of the list 3658 pHeap->pNoncontigFreeBlockList = pBlock; 3659 } 3660 else 3661 { 3662 pBlock->prevFreeNoncontig->nextFreeNoncontig = pBlock; 3663 } 3664 3665 break; 3666 } 3667 3668 if (NULL == pNextBlock->nextFreeNoncontig) 3669 { 3670 // We reached the end of the list, insert here 3671 pNextBlock->nextFreeNoncontig = pBlock; 3672 pBlock->prevFreeNoncontig = pNextBlock; 3673 pBlock->nextFreeNoncontig = NULL; 3674 3675 break; 3676 } 3677 3678 pNextBlock = pNextBlock->nextFreeNoncontig; 3679 } 3680 } 3681 } 3682 3683 static void 3684 _heapRemoveBlockFromNoncontigList 3685 ( 3686 Heap *pHeap, 3687 MEM_BLOCK *pBlock 3688 ) 3689 { 3690 // 3691 // Unless pBlock is at the head of the list (and is the only element in the 3692 // list), both prev and nextFreeNoncontig cannot be NULL at the same time. 3693 // That would imply a bug in the noncontig list building code. 3694 // 3695 NV_ASSERT(pBlock == pHeap->pNoncontigFreeBlockList || 3696 pBlock->prevFreeNoncontig != NULL || 3697 pBlock->nextFreeNoncontig != NULL); 3698 3699 // Removing first block? 3700 if (pHeap->pNoncontigFreeBlockList == pBlock) 3701 { 3702 pHeap->pNoncontigFreeBlockList = pBlock->nextFreeNoncontig; 3703 } 3704 else 3705 { 3706 if (NULL != pBlock->prevFreeNoncontig) 3707 { 3708 pBlock->prevFreeNoncontig->nextFreeNoncontig 3709 = pBlock->nextFreeNoncontig; 3710 } 3711 } 3712 3713 // Removing last block? 3714 if (NULL != pBlock->nextFreeNoncontig) 3715 { 3716 pBlock->nextFreeNoncontig->prevFreeNoncontig 3717 = pBlock->prevFreeNoncontig; 3718 } 3719 3720 pBlock->nextFreeNoncontig = pBlock->prevFreeNoncontig = NULL; 3721 } 3722 3723 // 3724 // The allocation is done using two loops. The first loop traverses the heap's 3725 // free list to build a list of blocks that can satisfy the allocation. If we 3726 // don't find enough blocks, we can exit quickly without needing to unwind, 3727 // which can happen quite frequently in low memory or heavy fragmentation 3728 // conditions. 3729 // 3730 // The second loop does the actual allocations. It calls _heapProcessFreeBlock() 3731 // to cut down a free block into the required size, which can fail, albeit 3732 // rarely. We need to unwind at that point. The two loops keep the unwinding 3733 // as infrequent as possible. 3734 // 3735 static NV_STATUS 3736 _heapAllocNoncontig 3737 ( 3738 OBJGPU *pGpu, 3739 NvHandle hClient, 3740 Heap *pHeap, 3741 MEMORY_ALLOCATION_REQUEST *pAllocRequest, 3742 NvHandle memHandle, 3743 OBJHEAP_ALLOC_DATA *pAllocData, 3744 FB_ALLOC_INFO *pFbAllocInfo, 3745 NvU32 textureClientIndex, 3746 NvU64 alignPad, 3747 NvU64 *offset, 3748 MEMORY_DESCRIPTOR *pMemDesc, 3749 HWRESOURCE_INFO **ppHwResource 3750 ) 3751 { 3752 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 3753 NV_MEMORY_ALLOCATION_PARAMS *pVidHeapAlloc = pAllocRequest->pUserParams; 3754 NvBool bFirstBlock = NV_TRUE; 3755 NvU32 pteIndexOffset = 0, i = 0; 3756 NvU32 blockId = 0; 3757 NV_STATUS status = NV_OK; 3758 NvU64 pageSize = 0; 3759 NvS64 numPagesLeft; 3760 MEM_BLOCK *pCurrBlock; 3761 MEM_BLOCK *pNextBlock; 3762 MEM_BLOCK *pSavedAllocList = NULL; 3763 MEM_BLOCK *pLastBlock = NULL; 3764 MEM_BLOCK *pBlockNew, *pBlockSplit; 3765 NvU32 k, shuffleStride = 1; 3766 NvU64 addr, j, numPages; 3767 RM_ATTR_PAGE_SIZE pageSizeAttr = dmaNvos32ToPageSizeAttr(pFbAllocInfo->retAttr, pFbAllocInfo->retAttr2); 3768 3769 switch (pageSizeAttr) 3770 { 3771 case RM_ATTR_PAGE_SIZE_DEFAULT: 3772 case RM_ATTR_PAGE_SIZE_INVALID: 3773 NV_PRINTF(LEVEL_ERROR, "Invalid page size attribute!\n"); 3774 return NV_ERR_INVALID_ARGUMENT; 3775 case RM_ATTR_PAGE_SIZE_4KB: 3776 pageSize = RM_PAGE_SIZE; 3777 break; 3778 case RM_ATTR_PAGE_SIZE_BIG: 3779 { 3780 pageSize = kgmmuGetMaxBigPageSize_HAL(pKernelGmmu); 3781 break; 3782 } 3783 case RM_ATTR_PAGE_SIZE_HUGE: 3784 { 3785 NV_ASSERT_OR_RETURN(kgmmuIsHugePageSupported(pKernelGmmu), 3786 NV_ERR_INVALID_ARGUMENT); 3787 pageSize = RM_PAGE_SIZE_HUGE; 3788 break; 3789 } 3790 case RM_ATTR_PAGE_SIZE_512MB: 3791 { 3792 NV_ASSERT_OR_RETURN(kgmmuIsPageSize512mbSupported(pKernelGmmu), 3793 NV_ERR_INVALID_ARGUMENT); 3794 pageSize = RM_PAGE_SIZE_512M; 3795 break; 3796 } 3797 } 3798 3799 // 3800 // pAllocData->allocSize already incorporates pFbAllocInfo->size, 3801 // which in turn is up aligned to pFbAllocInfo->align and alignPad, 3802 // so nothing else needs to be added here. 3803 // 3804 numPagesLeft = RM_ALIGN_UP(pAllocData->allocSize, pageSize) / pageSize; 3805 NV_PRINTF(LEVEL_INFO, 3806 "pageSize: 0x%llx, numPagesLeft: 0x%llx, allocSize: 0x%llx\n", 3807 pageSize / 1024, numPagesLeft, pAllocData->allocSize); 3808 3809 for (pCurrBlock = pHeap->pNoncontigFreeBlockList; 3810 numPagesLeft > 0 && NULL != pCurrBlock; 3811 pCurrBlock = pNextBlock) 3812 { 3813 NvU64 blockBegin = 0; 3814 NvU64 blockEnd = 0; 3815 NvU64 blockAligned; 3816 NvU64 blockSizeInPages, blockSize; 3817 NvU64 alignPad; 3818 NvU64 pteAddress; 3819 NvU64 offset; 3820 3821 // Get the next free block pointer before lists get re-linked 3822 pNextBlock = pCurrBlock->nextFreeNoncontig; 3823 3824 // Selecting blocks: Is this block completely out of range? 3825 if ((pCurrBlock->end < pVidHeapAlloc->rangeLo) || 3826 (pCurrBlock->begin > pVidHeapAlloc->rangeHi)) 3827 { 3828 continue; 3829 } 3830 3831 // Find the intersection of the block and the specified range. 3832 blockBegin = ((pVidHeapAlloc->rangeLo >= pCurrBlock->begin) ? 3833 pVidHeapAlloc->rangeLo : pCurrBlock->begin); 3834 blockEnd = ((pVidHeapAlloc->rangeHi <= pCurrBlock->end) ? 3835 pVidHeapAlloc->rangeHi : pCurrBlock->end); 3836 3837 // Check if the derived block is usable 3838 if ((blockBegin >= blockEnd) || 3839 (blockEnd-blockBegin+1 < pageSize)) 3840 { 3841 // Skip if the usable size is invalid or insufficient. 3842 continue; 3843 } 3844 3845 // 3846 // Checks above should protect against underflow, but we might still 3847 // end up with a post-aligned block that is unusable. 3848 // "end" should be RM_PAGE_SIZE-1 aligned. 3849 // 3850 blockBegin = RM_ALIGN_UP(blockBegin, pageSize); 3851 blockEnd = RM_ALIGN_DOWN(blockEnd+1, pageSize)-1; 3852 3853 if (blockBegin >= blockEnd) 3854 { 3855 // 3856 // When blockSize < page_size and blockBegin and/or blockEnd are 3857 // not page aligned initially, the above alignment can cause 3858 // blockBegin to become > blockEnd. 3859 // 3860 continue; 3861 } 3862 3863 // The first block has to handle pAllocData->alignment 3864 if (bFirstBlock) 3865 { 3866 // Align the starting address of the block to 3867 // pAllocData->alignment. 3868 blockAligned = (blockBegin + 3869 pAllocData->alignment - 1) / pAllocData->alignment 3870 * pAllocData->alignment; 3871 3872 // 3873 // Check that we'll still be within this block when 3874 // alignPad is added. 3875 // 3876 if (blockAligned + pFbAllocInfo->alignPad > blockEnd) 3877 { 3878 continue; 3879 } 3880 3881 // Then make sure this is page aligned. 3882 blockBegin = RM_ALIGN_DOWN(blockAligned, pageSize); 3883 3884 // 3885 // blockBegin is now the page aligned starting address of a 3886 // block that holds an address aligned to 3887 // pAllocData->alignment, and can take padding from 3888 // alignPad. 3889 // 3890 } 3891 else 3892 { 3893 blockAligned = blockBegin; 3894 } 3895 3896 blockSizeInPages = (blockEnd - blockBegin + 1) / pageSize; 3897 3898 // A usable block has to supply at least one page 3899 if (blockSizeInPages < 1) 3900 { 3901 continue; 3902 } 3903 3904 // blockEnd may need to be corrected for the last page 3905 if (((NvU64)numPagesLeft < blockSizeInPages)) 3906 { 3907 blockEnd = blockBegin + pageSize * numPagesLeft - 1; 3908 blockSizeInPages = numPagesLeft; 3909 } 3910 3911 blockSize = blockEnd - blockBegin + 1; 3912 3913 numPagesLeft -= blockSizeInPages; 3914 3915 NV_PRINTF(LEVEL_INFO, 3916 "\tblockId: %d, blockBegin: 0x%llx, blockEnd: 0x%llx, blockSize: " 3917 "0x%llx, blockSizeInPages: 0x%llx, numPagesLeft: 0x%llx\n", 3918 blockId, blockBegin, blockEnd, blockSize, blockSizeInPages, 3919 numPagesLeft >= 0 ? numPagesLeft : 0); 3920 3921 blockId++; 3922 3923 // 3924 // Set pAllocData values before the call to 3925 // _heapProcessFreeBlock() 3926 // 3927 pAllocData->allocLo = blockBegin; 3928 pAllocData->allocHi = blockEnd; 3929 pAllocData->allocAl = blockAligned; 3930 pAllocData->allocSize = blockSize; 3931 3932 if (bFirstBlock) 3933 { 3934 alignPad = pFbAllocInfo->alignPad; 3935 } 3936 else 3937 { 3938 alignPad = 0; 3939 } 3940 3941 // 3942 // Cut this new block down to size. pBlockNew will be the block to use 3943 // when this returns. 3944 // 3945 if (NV_OK != (status = _heapProcessFreeBlock(pGpu, pCurrBlock, 3946 &pBlockNew, &pBlockSplit, pHeap, pAllocRequest, 3947 memHandle, pAllocData, pFbAllocInfo, 3948 alignPad, &offset))) 3949 { 3950 NV_PRINTF(LEVEL_ERROR, 3951 "ERROR: Could not process free block, error: 0x%x\n", 3952 status); 3953 goto unwind_and_exit; 3954 } 3955 3956 // Never fails 3957 (void)_heapUpdate(pHeap, pBlockNew, BLOCK_FREE_STATE_CHANGED); 3958 3959 // 3960 // Save the allocation off in case we need to unwind 3961 // This also ensures that all blocks that make up the noncontig 3962 // allocation are strung together in a list, which is useful when 3963 // freeing them. 3964 // 3965 if (pSavedAllocList == NULL) 3966 { 3967 // First block 3968 pSavedAllocList = pLastBlock = pBlockNew; 3969 pSavedAllocList->noncontigAllocListNext = NULL; 3970 } 3971 else 3972 { 3973 pLastBlock->noncontigAllocListNext = pBlockNew; 3974 pLastBlock = pBlockNew; 3975 pLastBlock->noncontigAllocListNext = NULL; 3976 } 3977 3978 pteAddress = RM_PAGE_ALIGN_DOWN(pBlockNew->begin); 3979 3980 numPages = NV_MIN(blockSizeInPages, ((pMemDesc->PageCount - pteIndexOffset) * RM_PAGE_SIZE) / pageSize); 3981 3982 if (pHeap->getProperty(pHeap, PDB_PROP_HEAP_PAGE_SHUFFLE)) 3983 { 3984 i = pHeap->shuffleStrideIndex; 3985 shuffleStride = pHeap->shuffleStrides[i]; 3986 3987 // Select a stride greater the the number of pages 3988 while(numPages < shuffleStride && i > 0) 3989 { 3990 i--; 3991 shuffleStride = pHeap->shuffleStrides[i]; 3992 } 3993 3994 pHeap->shuffleStrideIndex = (pHeap->shuffleStrideIndex + 1) % SHUFFLE_STRIDE_MAX; 3995 } 3996 3997 // 3998 // Shuffling logic. 3999 // We scatter the contiguous pages at multiple of stride length. 4000 // For 5 pages with stride length 2, we have the following shuffling. 4001 // Before: 0, 1, 2, 3, 4 4002 // After : 0, 2, 4, 1, 3 4003 // 4004 for (i = 0; i < shuffleStride; i++) 4005 { 4006 for(j = i; j < numPages; j = j + shuffleStride) 4007 { 4008 addr = pteAddress + j * pageSize; 4009 for (k = 0; k < pageSize/RM_PAGE_SIZE; k++) 4010 { 4011 // 4012 // The memDesc has everything in terms of 4k pages. 4013 // If allocationSize % pageSize != 0, there will not be enough PTEs in 4014 // the memdesc for completely specifying the final block, but that's 4015 // ok. The mapping code will be mapping in the whole pageSize final 4016 // block anyway, and the heapBlockFree() code will free the whole 4017 // block. 4018 // 4019 memdescSetPte(pMemDesc, AT_GPU, pteIndexOffset, addr); 4020 pteIndexOffset++; 4021 addr += RM_PAGE_SIZE; 4022 } 4023 } 4024 } 4025 4026 // 4027 // If a client calls us with pVidHeapAlloc->type == 4028 // NVOS32_TYPE_TEXTURE, but where flags are non-zero, we won't 4029 // call objHeapSetTexturePlacement and initialize 4030 // textureClientIndex to a proper value (default is 0xFFFFFFFF). 4031 // In that case, we won't track this texture allocation. Bug 4032 // 79586. 4033 // 4034 if (pVidHeapAlloc->type == NVOS32_TYPE_TEXTURE && 4035 textureClientIndex != 0xFFFFFFFF) 4036 { 4037 pBlockNew->textureId = hClient; 4038 if (bFirstBlock) 4039 pHeap->textureData[textureClientIndex].refCount++; 4040 } 4041 else 4042 { 4043 pBlockNew->textureId = 0; 4044 } 4045 4046 if (bFirstBlock) 4047 { 4048 pFbAllocInfo->offset = offset; 4049 *ppHwResource = &pBlockNew->hwResource; 4050 } 4051 4052 pBlockNew->pMemDesc = pMemDesc; 4053 pBlockNew->allocedMemDesc = bFirstBlock; // avoid multiple frees 4054 4055 bFirstBlock = NV_FALSE; 4056 } 4057 4058 // Did we find enough pages? 4059 if (numPagesLeft > 0) 4060 { 4061 NV_PRINTF(LEVEL_INFO, 4062 "Could not satisfy request: allocSize: 0x%llx\n", 4063 pAllocData->allocSize); 4064 4065 status = NV_ERR_NO_MEMORY; 4066 4067 unwind_and_exit: 4068 4069 while (pSavedAllocList != NULL) 4070 { 4071 NV_STATUS unwindStatus; 4072 4073 pCurrBlock = pSavedAllocList->noncontigAllocListNext; 4074 4075 unwindStatus = _heapBlockFree(pGpu, pHeap, hClient, pFbAllocInfo->hDevice, pSavedAllocList); 4076 4077 if (unwindStatus != NV_OK) 4078 { 4079 NV_PRINTF(LEVEL_ERROR, 4080 "ERROR: Could not free block, error 0x%x!\n", 4081 unwindStatus); 4082 } 4083 4084 pSavedAllocList = pCurrBlock; 4085 } 4086 } 4087 return status; 4088 } 4089 4090 // 4091 // Explanation of BlockAction values: 4092 // - BLOCK_ADD, 4093 // A new block is added to the heap 4094 // o The block's node structure needs to be inited. 4095 // o The block is added to the rb-tree. 4096 // o The block is added to the noncontig freelist. 4097 // - BLOCK_REMOVE 4098 // A block is removed from the heap for good 4099 // o The block is removed from the rb-tree. 4100 // o The block is removed from the noncontig freelist. 4101 // - BLOCK_SIZE_CHANGED 4102 // A block's size has changed 4103 // o The rb-tree needs to be updated. 4104 // o The noncontig freelist needs to be updated. 4105 // - BLOCK_FREE_STATE_CHANGED 4106 // if pBlock->owner != NVOS32_BLOCK_TYPE_FREE 4107 // A block is allocated to a client 4108 // o The block is removed from the noncontig freelist. 4109 // else 4110 // A block is freed by the client 4111 // o The block is added to the noncontig freelist. 4112 // 4113 static NV_STATUS 4114 _heapUpdate 4115 ( 4116 Heap *pHeap, 4117 MEM_BLOCK *pBlock, 4118 BlockAction action 4119 ) 4120 { 4121 // A new block is to be added, init its node structure. 4122 if (BLOCK_ADD == action) 4123 { 4124 portMemSet((void *)&pBlock->node, 0, sizeof(NODE)); 4125 pBlock->node.Data = (void *)pBlock; 4126 } 4127 4128 // Both new and updated blocks need to be re-inserted into the rb tree. 4129 if ((BLOCK_SIZE_CHANGED == action) || 4130 (BLOCK_ADD == action)) 4131 { 4132 pBlock->node.keyStart = pBlock->begin; 4133 pBlock->node.keyEnd = pBlock->end; 4134 4135 if (btreeInsert(&pBlock->node, &pHeap->pBlockTree) != NV_OK) 4136 { 4137 NV_ASSERT_FAILED("btreeInsert failed to ADD/SIZE_CHANGE block"); 4138 return NV_ERR_INVALID_STATE; 4139 } 4140 } 4141 4142 // 4143 // Updated, new and freed blocks need to be added back to the noncontig 4144 // freelist. 4145 // 4146 if ((BLOCK_SIZE_CHANGED == action) || 4147 (BLOCK_ADD == action) || 4148 (BLOCK_FREE_STATE_CHANGED == action && 4149 pBlock->owner == NVOS32_BLOCK_TYPE_FREE)) 4150 { 4151 _heapAddBlockToNoncontigList(pHeap, pBlock); 4152 } 4153 4154 // Remove the block from the heap 4155 if (BLOCK_REMOVE == action) 4156 { 4157 if (btreeUnlink(&pBlock->node, &pHeap->pBlockTree) != NV_OK) 4158 { 4159 NV_ASSERT_FAILED("btreeUnlink failed to REMOVE block"); 4160 return NV_ERR_INVALID_STATE; 4161 } 4162 } 4163 4164 // An allocated block is only removed from the noncontig freelist. 4165 if ((BLOCK_REMOVE == action) || 4166 ((BLOCK_FREE_STATE_CHANGED == action && 4167 pBlock->owner != NVOS32_BLOCK_TYPE_FREE))) 4168 { 4169 _heapRemoveBlockFromNoncontigList(pHeap, pBlock); 4170 } 4171 4172 return NV_OK; 4173 } 4174 4175 static NvU32 4176 _heapGetPageBlackListGranularity(void) 4177 { 4178 return RM_PAGE_SIZE; 4179 } 4180 4181 // 4182 // This function blacklists pages from the heap. 4183 // The addresses of the pages to blacklist are available from 4184 // pHeap->blackListAddresses. 4185 // 4186 NV_STATUS 4187 heapBlackListPages_IMPL 4188 ( 4189 OBJGPU *pGpu, 4190 Heap *pHeap 4191 ) 4192 { 4193 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 4194 PMA *pPma = &pHeap->pmaObject; 4195 NvU32 i = 0, j = 0; 4196 NV_STATUS status = NV_OK; 4197 BLACKLIST *pBlackList = &pHeap->blackList; 4198 BLACKLIST_ADDRESSES *pAddresses = &pHeap->blackListAddresses; 4199 NvU32 count = pHeap->blackListAddresses.count; 4200 NvU32 staticBlacklistSize, dynamicBlacklistSize; 4201 NvU32 dynamicRmBlackListedCount; 4202 NvU32 staticRmBlackListedCount; 4203 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 4204 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 4205 4206 if (NULL == pAddresses) 4207 { 4208 return NV_ERR_INVALID_ARGUMENT; 4209 } 4210 4211 if (pBlackList->count != 0) 4212 { 4213 NV_PRINTF(LEVEL_ERROR, "Error: BlackList already exists!\n"); 4214 return NV_ERR_INVALID_STATE; 4215 } 4216 4217 // 4218 // We may not be able to allocate all pages requested, but alloc enough 4219 // space anyway 4220 // 4221 pBlackList->pBlacklistChunks = portMemAllocNonPaged(sizeof(BLACKLIST_CHUNK) * pMemorySystemConfig->maximumBlacklistPages); 4222 if (NULL == pBlackList->pBlacklistChunks) 4223 { 4224 NV_PRINTF(LEVEL_ERROR, "Could not allocate memory for blackList!\n"); 4225 return NV_ERR_NO_MEMORY; 4226 } 4227 4228 portMemSet(pBlackList->pBlacklistChunks, 0, sizeof(BLACKLIST_CHUNK) * pMemorySystemConfig->maximumBlacklistPages); 4229 4230 dynamicRmBlackListedCount = 0; 4231 staticRmBlackListedCount = 0; 4232 for (i = 0, j = 0; i < count; i++) 4233 { 4234 if (NV2080_CTRL_FB_OFFLINED_PAGES_INVALID_ADDRESS == pAddresses->data[i].address) 4235 { 4236 continue; 4237 } 4238 4239 // 4240 // If PMA is enabled, only blacklist pages in the internal heap. 4241 // PMA blacklisting is handled in pmaRegisterRegion. 4242 // 4243 if (memmgrIsPmaInitialized(pMemoryManager)) 4244 { 4245 if (heapIsPmaManaged(pGpu, pHeap, pAddresses->data[i].address, pAddresses->data[i].address)) 4246 { 4247 // Skipping non-internal address 4248 continue; 4249 } 4250 } 4251 4252 if ((pAddresses->data[i].type == NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_MULTIPLE_SBE) || 4253 (pAddresses->data[i].type == NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE)) 4254 { 4255 dynamicRmBlackListedCount++; 4256 } 4257 else 4258 { 4259 staticRmBlackListedCount++; 4260 } 4261 4262 // Create a memdesc 4263 status = memdescCreate(&pBlackList->pBlacklistChunks[j].pMemDesc, 4264 pGpu, 4265 RM_PAGE_SIZE, 4266 RM_PAGE_SIZE, 4267 NV_TRUE, 4268 ADDR_FBMEM, 4269 NV_MEMORY_UNCACHED, 4270 MEMDESC_FLAGS_FIXED_ADDRESS_ALLOCATE | 4271 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 4272 if (NV_OK != status) 4273 { 4274 portMemSet(&pBlackList->pBlacklistChunks[j], 0, sizeof(BLACKLIST_CHUNK)); 4275 NV_PRINTF(LEVEL_ERROR, 4276 "Error 0x%x creating blacklisted page memdesc for address 0x%llx, skipping\n", 4277 status, pAddresses->data[i].address); 4278 continue; 4279 } 4280 4281 if (pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR) 4282 pBlackList->pBlacklistChunks[j].pMemDesc->pHeap = pHeap; 4283 4284 // This is how _FIXED_ADDRESS_ALLOCATE works 4285 memdescSetPte(pBlackList->pBlacklistChunks[j].pMemDesc, 4286 AT_GPU, 0, RM_PAGE_ALIGN_DOWN(pAddresses->data[i].address)); 4287 4288 if (pHeap->heapType != HEAP_TYPE_PHYS_MEM_SUBALLOCATOR) 4289 { 4290 // 4291 // Allocate memory for this page. This is marked as an internal RM 4292 // allocation and WILL be saved/restored during suspend/resume. 4293 // 4294 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_79, 4295 pBlackList->pBlacklistChunks[j].pMemDesc); 4296 if (NV_OK != status) 4297 { 4298 // No use for the memdesc if the page couldn't be allocated 4299 memdescDestroy(pBlackList->pBlacklistChunks[j].pMemDesc); 4300 4301 portMemSet(&pBlackList->pBlacklistChunks[j], 0, sizeof(BLACKLIST_CHUNK)); 4302 4303 NV_PRINTF(LEVEL_ERROR, 4304 "Error 0x%x blacklisting page at address 0x%llx, skipping\n", 4305 status, pAddresses->data[i].address); 4306 continue; 4307 } 4308 } 4309 4310 // Page blacklisting is successful, add entries to the BLACKLIST 4311 pBlackList->pBlacklistChunks[j].physOffset = pAddresses->data[i].address; 4312 pBlackList->pBlacklistChunks[j].size = RM_PAGE_SIZE; 4313 pBlackList->pBlacklistChunks[j].bIsValid = NV_TRUE; 4314 4315 // If the page was successfully blacklisted, move to the next entry 4316 j++; 4317 } 4318 4319 pBlackList->count = j; 4320 4321 pmaGetBlacklistSize(pPma, &dynamicBlacklistSize, &staticBlacklistSize); 4322 dynamicBlacklistSize = dynamicBlacklistSize >> 10; 4323 staticBlacklistSize = staticBlacklistSize >> 10; 4324 4325 dynamicBlacklistSize += (dynamicRmBlackListedCount * _heapGetPageBlackListGranularity()) >> 10; 4326 staticBlacklistSize += (staticRmBlackListedCount * _heapGetPageBlackListGranularity()) >> 10; 4327 4328 pHeap->dynamicBlacklistSize = dynamicBlacklistSize; 4329 pHeap->staticBlacklistSize = staticBlacklistSize; 4330 4331 if (0 == pBlackList->count) 4332 { 4333 // No address was blacklisted 4334 portMemFree(pBlackList->pBlacklistChunks); 4335 pBlackList->pBlacklistChunks = NULL; 4336 } 4337 4338 return NV_OK; 4339 } 4340 4341 // 4342 // This function frees all blacklisted pages. 4343 // The pHeap->blackList structure holds a list of memdescs, one for each 4344 // blacklisted page. 4345 // 4346 NV_STATUS 4347 heapFreeBlackListedPages_IMPL 4348 ( 4349 OBJGPU *pGpu, 4350 Heap *pHeap 4351 ) 4352 { 4353 NvU32 i; 4354 BLACKLIST *pBlackList = &pHeap->blackList; 4355 4356 // Also free the blacklistAddresses data here 4357 if (pHeap->blackListAddresses.data) 4358 { 4359 portMemFree(pHeap->blackListAddresses.data); 4360 pHeap->blackListAddresses.count = 0; 4361 pHeap->blackListAddresses.data = NULL; 4362 } 4363 4364 if (0 == pBlackList->count) 4365 { 4366 return NV_OK; 4367 } 4368 4369 if (NULL == pBlackList->pBlacklistChunks) 4370 { 4371 return NV_ERR_INVALID_STATE; 4372 } 4373 4374 for (i = 0; i < pBlackList->count; i++) 4375 { 4376 if (pBlackList->pBlacklistChunks[i].bIsValid) 4377 { 4378 // Free the blacklisted page 4379 memdescFree(pBlackList->pBlacklistChunks[i].pMemDesc); 4380 4381 // Free the memdesc 4382 memdescDestroy(pBlackList->pBlacklistChunks[i].pMemDesc); 4383 } 4384 } 4385 4386 portMemFree(pBlackList->pBlacklistChunks); 4387 4388 pBlackList->count = 0; 4389 pBlackList->pBlacklistChunks = NULL; 4390 4391 return NV_OK; 4392 } 4393 4394 NV_STATUS 4395 heapStorePendingBlackList_IMPL 4396 ( 4397 OBJGPU *pGpu, 4398 Heap *pHeap, 4399 NvU64 pageAddressesWithEccOn, 4400 NvU64 pageAddressWithEccOff 4401 ) 4402 { 4403 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 4404 NV_STATUS status = NV_OK; 4405 NvU64 physicalAddress; 4406 NvU64 pageNumber; 4407 BLACKLIST *pBlacklist = &pHeap->blackList; 4408 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 4409 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 4410 4411 if (pMemorySystemConfig->bEnabledEccFBPA) 4412 { 4413 physicalAddress = pageAddressesWithEccOn; 4414 } 4415 else 4416 { 4417 physicalAddress = pageAddressWithEccOff; 4418 } 4419 4420 pageNumber = (physicalAddress >> RM_PAGE_SHIFT); 4421 4422 // This code is called only when DBE happens, so marking it as type DBE 4423 status = heapAddPageToBlackList(pGpu, pHeap, 4424 DRF_VAL64(_HEAP, _PAGE_OFFLINE, _PAGE_NUMBER, pageNumber), 4425 NV2080_CTRL_FB_OFFLINED_PAGES_SOURCE_DPR_DBE); 4426 if (NV_OK != status) 4427 { 4428 // No more space in the blacklist 4429 NV_PRINTF(LEVEL_ERROR, "No more space in blacklist, status: %x!\n", status); 4430 return status; 4431 } 4432 4433 if (memmgrIsPmaInitialized(pMemoryManager)) 4434 { 4435 if (heapIsPmaManaged(pGpu, pHeap, physicalAddress, physicalAddress)) 4436 { 4437 NV_PRINTF(LEVEL_INFO, "Calling PMA helper function to blacklist page offset: %llx\n", physicalAddress); 4438 status = pmaAddToBlacklistTracking(&pHeap->pmaObject, physicalAddress); 4439 return status; 4440 } 4441 else 4442 { 4443 // blacklisting needs to be done like CBC error recovery 4444 return NV_ERR_RESET_REQUIRED; 4445 } 4446 } 4447 else 4448 { 4449 if (pMemoryManager->bEnableDynamicPageOfflining) 4450 { 4451 // adding a new entry to heap managed blacklist 4452 if (pBlacklist->count == pMemorySystemConfig->maximumBlacklistPages) 4453 { 4454 NV_PRINTF(LEVEL_ERROR, "We have blacklisted maximum number of pages possible. returning error \n"); 4455 return NV_ERR_INSUFFICIENT_RESOURCES; 4456 } 4457 portMemSet(&pBlacklist->pBlacklistChunks[pBlacklist->count], 0 , sizeof(BLACKLIST_CHUNK)); 4458 pBlacklist->pBlacklistChunks[pBlacklist->count].physOffset = physicalAddress; 4459 pBlacklist->pBlacklistChunks[pBlacklist->count].size = RM_PAGE_SIZE; 4460 pBlacklist->pBlacklistChunks[pBlacklist->count].bPendingRetirement = NV_TRUE; 4461 pBlacklist->count++; 4462 } 4463 } 4464 return status; 4465 } 4466 4467 // 4468 // This function copies the addresses of pages to be blacklisted from 4469 // pPageNumbers into Heap's internal blackListAddresses structure. 4470 // 4471 NV_STATUS 4472 heapStoreBlackList_IMPL 4473 ( 4474 OBJGPU *pGpu, 4475 Heap *pHeap, 4476 NvU64 *pPageNumbersWithEccOn, 4477 NvU64 *pPageNumbersWithECcOff, 4478 NvU32 maxInputPages 4479 ) 4480 { 4481 NvU32 i; 4482 NvU64 *pPageNumbers; 4483 NV_STATUS status = NV_OK; 4484 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 4485 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 4486 4487 if (pMemorySystemConfig->bEnabledEccFBPA) 4488 { 4489 pPageNumbers = pPageNumbersWithEccOn; 4490 } 4491 else 4492 { 4493 pPageNumbers = pPageNumbersWithECcOff; 4494 } 4495 4496 for (i = 0; i < maxInputPages; i++) 4497 { 4498 // 4499 // Bug: 2999257 4500 // currently pre-Hopper we have 37b FB PA, whose PFN will be 25b 4501 // From Hopper+ we have 52b PA, whose PFN will be 40b PA and hence 4502 // the macro NV_INFOROM_BLACKLIST_PAGE_NUMBER width of 28b will not be 4503 // sufficient to capture the entire address, this needs to be fixed. 4504 // 4505 status = heapAddPageToBlackList(pGpu, pHeap, 4506 DRF_VAL64(_HEAP, _PAGE_OFFLINE, _PAGE_NUMBER, pPageNumbers[i]), 4507 (NvU32)DRF_VAL64(_HEAP, _PAGE_OFFLINE, _TYPE, pPageNumbers[i])); 4508 if (NV_OK != status) 4509 { 4510 // No more space in the blacklist 4511 NV_PRINTF(LEVEL_ERROR, "No more space in blacklist!\n"); 4512 return status; 4513 } 4514 } 4515 4516 return status; 4517 } 4518 4519 NV_STATUS 4520 heapAddPageToBlackList_IMPL 4521 ( 4522 OBJGPU *pGpu, 4523 Heap *pHeap, 4524 NvU64 pageNumber, 4525 NvU32 type 4526 ) 4527 { 4528 NvU32 index = pHeap->blackListAddresses.count; 4529 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 4530 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 4531 4532 if (index == pMemorySystemConfig->maximumBlacklistPages) 4533 { 4534 return NV_ERR_INSUFFICIENT_RESOURCES; 4535 } 4536 4537 if (pHeap->blackListAddresses.data == NULL) 4538 { 4539 NvU64 listSize = sizeof(BLACKLIST_ADDRESS) * pMemorySystemConfig->maximumBlacklistPages; 4540 4541 pHeap->blackListAddresses.data = portMemAllocNonPaged(listSize); 4542 if (pHeap->blackListAddresses.data == NULL) 4543 { 4544 return NV_ERR_NO_MEMORY; 4545 } 4546 4547 portMemSet(pHeap->blackListAddresses.data, 0, listSize); 4548 } 4549 4550 pHeap->blackListAddresses.data[index].address = (pageNumber << RM_PAGE_SHIFT); 4551 pHeap->blackListAddresses.data[index].type = type; 4552 4553 pHeap->blackListAddresses.count++; 4554 4555 NV_PRINTF(LEVEL_INFO, "Added 0x%0llx (blacklist count: %u)\n", 4556 pHeap->blackListAddresses.data[index].address, 4557 pHeap->blackListAddresses.count); 4558 4559 return NV_OK; 4560 } 4561 4562 /*! 4563 * @brief: Identify if an FB range is PMA-managed 4564 * 4565 * @param[in] pGpu OBJGPU pointer 4566 * @param[in] pHeap Heap pointer 4567 * @param[in] offset FB block offset 4568 * @param[in] limit FB block limit 4569 * 4570 * @return NV_TRUE offset is PMA-managed 4571 * NV_FALSE offset is not managed by PMA 4572 */ 4573 NvBool 4574 heapIsPmaManaged_IMPL 4575 ( 4576 OBJGPU *pGpu, 4577 Heap *pHeap, 4578 NvU64 offset, 4579 NvU64 limit 4580 ) 4581 { 4582 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 4583 4584 if (memmgrIsPmaInitialized(pMemoryManager)) 4585 { 4586 NvU32 i; 4587 4588 NV_ASSERT(offset <= limit); 4589 4590 for (i = 0; i < pHeap->pmaObject.regSize; i++) 4591 { 4592 if ((offset >= pHeap->pmaObject.pRegDescriptors[i]->base) && 4593 (limit <= pHeap->pmaObject.pRegDescriptors[i]->limit)) 4594 { 4595 NV_PRINTF(LEVEL_INFO, 4596 "range %llx..%llx resides in PMA region=%llx..%llx\n", 4597 offset, limit, 4598 pHeap->pmaObject.pRegDescriptors[i]->base, 4599 pHeap->pmaObject.pRegDescriptors[i]->limit); 4600 return NV_TRUE; 4601 } 4602 #if defined(DEBUG) 4603 // Check for straddling 4604 else if ( 4605 (limit >= pHeap->pmaObject.pRegDescriptors[i]->base) && 4606 (offset <= pHeap->pmaObject.pRegDescriptors[i]->limit)) 4607 { 4608 NV_PRINTF(LEVEL_ERROR, 4609 "range %llx..%llx straddles in PMA region=%llx..%llx\n", 4610 offset, limit, 4611 pHeap->pmaObject.pRegDescriptors[i]->base, 4612 pHeap->pmaObject.pRegDescriptors[i]->limit); 4613 } 4614 #endif //defined(DEBUG) 4615 } 4616 } 4617 4618 return(NV_FALSE); 4619 } 4620 4621 /*! 4622 * @brief Increase the reference count 4623 * 4624 * @param[in] pGpu OBJGPU pointer 4625 * @param[in] pHeap Heap pointer 4626 * 4627 * @return Current refcount value 4628 */ 4629 NvU32 4630 heapAddRef_IMPL 4631 ( 4632 Heap *pHeap 4633 ) 4634 { 4635 if (pHeap == NULL) 4636 return 0; 4637 4638 return portAtomicExIncrementU64(&pHeap->refCount); 4639 } 4640 4641 /*! 4642 * @brief Increase the reference count 4643 * 4644 * @param[in] pGpu OBJGPU pointer 4645 * @param[in] pHeap Heap pointer 4646 * 4647 * @return Current refcount value 4648 */ 4649 NvU32 4650 heapRemoveRef_IMPL 4651 ( 4652 Heap *pHeap 4653 ) 4654 { 4655 NvU64 refCount = 0; 4656 4657 if (pHeap == NULL) 4658 return 0; 4659 4660 refCount = portAtomicExDecrementU64(&pHeap->refCount); 4661 if (refCount == 0) 4662 { 4663 objDelete(pHeap); 4664 } 4665 4666 return refCount; 4667 } 4668 4669 /*! 4670 * @brief Adjust the heap size 4671 * 4672 * @param[in] pHeap Heap pointer 4673 * @param[in] resizeBy NVS64 resizeBy value 4674 */ 4675 4676 NV_STATUS heapResize_IMPL 4677 ( 4678 Heap *pHeap, 4679 NvS64 resizeBy 4680 ) 4681 { 4682 MEM_BLOCK *pBlockLast; 4683 MEM_BLOCK *pBlockNew; 4684 NV_STATUS status = NV_OK; 4685 OBJGPU *pGpu = ENG_GET_GPU(pHeap); 4686 4687 NV_ASSERT_OR_RETURN(pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR, NV_ERR_NOT_SUPPORTED); 4688 4689 // Free all blacklisted pages 4690 if ((pHeap->blackListAddresses.count != 0) && 4691 pGpu->getProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT) && 4692 gpuCheckPageRetirementSupport_HAL(pGpu)) 4693 { 4694 heapFreeBlackListedPages(pGpu, pHeap); 4695 } 4696 4697 // Go to last block if the heap w.r.t. the start address 4698 pBlockLast = pHeap->pBlockList; 4699 while (pBlockLast->next != pHeap->pBlockList) 4700 pBlockLast = pBlockLast->next; 4701 4702 if (resizeBy < 0) // Shrink the allocation 4703 { 4704 NvS64 newSize; 4705 4706 NV_ASSERT_OR_RETURN(pBlockLast->owner == NVOS32_BLOCK_TYPE_FREE, NV_ERR_NO_MEMORY); 4707 NV_CHECK_OR_RETURN(LEVEL_ERROR, portSafeAddS64(pBlockLast->end - pBlockLast->begin, resizeBy, &newSize) && 4708 (newSize > 0), NV_ERR_INVALID_LIMIT); 4709 pBlockLast->end += resizeBy; 4710 } 4711 else // Grow the allocation 4712 { 4713 if (pBlockLast->owner == NVOS32_BLOCK_TYPE_FREE) 4714 { 4715 // Found a free block at the end Just resize it. 4716 pBlockLast->end += resizeBy; 4717 } 4718 else 4719 { 4720 // Could not find a free block at the end. Add a new free block. 4721 pBlockNew = portMemAllocNonPaged(sizeof(MEM_BLOCK)); 4722 if (pBlockNew != NULL) 4723 { 4724 4725 portMemSet(pBlockNew, 0, sizeof(MEM_BLOCK)); 4726 4727 pBlockNew->owner = NVOS32_BLOCK_TYPE_FREE; 4728 pBlockNew->refCount = 1; 4729 4730 // Set block boundaries 4731 pBlockNew->begin = pBlockLast->end + 1; 4732 pBlockNew->end = pBlockLast->end + resizeBy; 4733 4734 if (pHeap->pFreeBlockList == NULL) 4735 pHeap->pFreeBlockList = pBlockNew; 4736 4737 // Add the block in the free blocks list 4738 pBlockNew->u1.nextFree = pHeap->pFreeBlockList; 4739 pBlockNew->u0.prevFree = pHeap->pFreeBlockList->u0.prevFree; 4740 pBlockNew->u1.nextFree->u0.prevFree = pBlockNew; 4741 pBlockNew->u0.prevFree->u1.nextFree = pBlockNew; 4742 4743 // Add the block in the blocks list 4744 pBlockNew->next = pBlockLast->next; 4745 pBlockNew->prev = pBlockLast; 4746 pBlockNew->next->prev = pBlockNew; 4747 pBlockNew->prev->next = pBlockNew; 4748 4749 if ((status = _heapUpdate(pHeap, pBlockNew, BLOCK_ADD)) != NV_OK) 4750 { 4751 NV_PRINTF(LEVEL_ERROR, 4752 "_heapUpdate failed to _ADD block\n"); 4753 4754 if (pHeap->pFreeBlockList == pBlockNew) // There was no free block in the heap. 4755 pHeap->pFreeBlockList = NULL; // We had added this one. 4756 portMemFree(pBlockNew); 4757 } 4758 else 4759 { 4760 pHeap->numBlocks++; 4761 } 4762 } 4763 } 4764 } 4765 4766 if (status == NV_OK) 4767 { 4768 pHeap->total += resizeBy; 4769 pHeap->free += resizeBy; 4770 4771 status = memmgrGetBlackListPagesForHeap_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), pHeap); 4772 if (status != NV_OK) 4773 { 4774 NV_PRINTF(LEVEL_INFO, 4775 "Failed to read blackList pages (0x%x).\n", 4776 status); 4777 } 4778 4779 heapFilterBlackListPages(pHeap, pHeap->base, pHeap->total); 4780 4781 if (pHeap->blackListAddresses.count != 0) 4782 { 4783 status = heapBlackListPages(pGpu, pHeap); 4784 4785 if (status != NV_OK) 4786 { 4787 NV_PRINTF(LEVEL_WARNING, 4788 "Error 0x%x creating blacklist\n", 4789 status); 4790 } 4791 } 4792 } 4793 return status; 4794 } 4795