1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2016-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @file pool_alloc.c 26 * @brief Defines the interfaces for managing the memory pools used for 27 * allocating and freeing the RM allocations. RM's 28 * internal page directories/tables are NOT managed by PMA and 29 * DO NOT use the interfaces defined in this file. 30 */ 31 32 /* ------------------------------------ Includes ----------------------------------- */ 33 #include "mem_mgr/pool_alloc.h" 34 #include "mem_mgr/vaspace.h" 35 #include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h" 36 #include "class/cl90f1.h" 37 #include "mmu/gmmu_fmt.h" 38 #include "gpu/gpu.h" 39 40 /* ------------------------------------ Local Defines ------------------------------ */ 41 #define PMA_CHUNK_SIZE_512M (512 * 1024 * 1024) 42 #define PMA_CHUNK_SIZE_4M (4 * 1024 * 1024) 43 #define PMA_CHUNK_SIZE_2M (2 * 1024 * 1024) 44 #define PMA_CHUNK_SIZE_512K (512 * 1024) 45 #define PMA_CHUNK_SIZE_256K (256 * 1024) 46 #define PMA_CHUNK_SIZE_64K (64 * 1024) 47 48 /*! PAGE SIZES FOR DIFFERENT POOL ALLOCATOR LEVELS 49 * 50 * CONTEXT BUFFER allocations 51 * 52 * When allocator is used for context buffers three page sizes 53 * are supported as follows: 54 * 55 * For buffers >= 2MB, page size = 2MB 56 * For buffers >= 32KB and < 2MB, page size = 64KB 57 * For buffers < 32KB, page siez = 4KB 58 * 59 * PAGE TABLE allocations 60 * 61 * When the allocator is used for page tables the page sizes 62 * supported by different allocator levels are calculated as follows: 63 * 64 * Pre-Pascal [Big page size = 128K] 65 * Size of a full PD0 (Root) = 64 KBytes 66 * Size of a full Small Page Table = 256 KBytes 67 * Size of a full Big Page Table = 8 KBytes 68 * 69 * Pre-Pascal [Big page size = 64K] 70 * Size of a full PD0 (Root) = 128 KBytes 71 * Size of a full Small Page Table = 128 KBytes 72 * Size of a full Big Page Table = 8 KBytes 73 * 74 * Pascal+ 75 * Size of a full PD3 (Root) = 4 KBytes 76 * Size of a full PD2 = 4 KBytes 77 * Size of a full PD1 = 4 KBytes 78 * Size of a full PD0 = 4 KBytes 79 * Size of a full Small Page Table = 4 KBytes 80 * Size of a full Big Page Table = 256 Bytes 81 * 82 */ 83 typedef enum 84 { 85 RM_POOL_IDX_512M, 86 RM_POOL_IDX_2M, 87 RM_POOL_IDX_256K, 88 RM_POOL_IDX_128K, 89 RM_POOL_IDX_64K, 90 RM_POOL_IDX_8K, 91 RM_POOL_IDX_4K, 92 RM_POOL_IDX_256B, 93 NUM_POOLS // This should always be the last entry! 94 }POOL_IDX; 95 96 /*! 97 * This array contains the alloction sizes (in bytes) of each pool. 98 */ 99 static const NvU64 poolAllocSizes[] = { 100 0x20000000, 0x200000, 0x40000, 0x20000, 0x10000, 0x2000, 0x1000, 0x100 101 }; 102 103 #define POOL_CONFIG_POOL_IDX 0 104 #define POOL_CONFIG_CHUNKSIZE_IDX 1 105 106 static const NvU64 poolConfig[POOL_CONFIG_MAX_SUPPORTED][POOL_CONFIG_CHUNKSIZE_IDX + 1] = { 107 // page size // chunk size 108 { RM_POOL_IDX_256K, PMA_CHUNK_SIZE_512K}, // pool with pageSize = 256K for GMMU_FMT_VERSION_1 109 { RM_POOL_IDX_4K, PMA_CHUNK_SIZE_64K }, // pool with pageSize = 4K for GMMU_FMT_VERSION_2 110 { RM_POOL_IDX_512M, PMA_CHUNK_SIZE_512M }, // pool with pageSize = 512MB for RM allocated buffers (unused as of ampere) 111 { RM_POOL_IDX_2M, PMA_CHUNK_SIZE_4M }, // pool with pageSize = 2MB for RM allocated buffers 112 { RM_POOL_IDX_64K, PMA_CHUNK_SIZE_256K }, // pool with pageSize = 64K for RM allocated buffers 113 { RM_POOL_IDX_4K, PMA_CHUNK_SIZE_64K } // pool with pageSize = 4K for RM allocated buffers 114 }; 115 116 /*! 117 * Locking in the RM internal pool allocator 118 * =================================== 119 * 120 * - pPoolLock 121 * Mutex (a PORT_MUTEX instance) 122 * 123 * The data inside RM_POOL_ALLOC_MEM_RESERVE_INFO is protected from concurrent access 124 * by this mutex. Any function accessing the RM_POOL_ALLOC_MEM_RESERVE_INFO data should 125 * acquire the mutex. 126 * We're using a mutex instead of a spinlock since we are allocating memory inside the 127 * lock. The allocation thread (pmaAllocatePages) may sleep on a semaphore (if scrubbing 128 * is in progress). So, spinlock is not an appropriate choice. The current assumption is that 129 * none of the functions defined here gets called in an interrupt/atomic context. We"ll 130 * assert in portSyncMutexAcquire() if any of this code ever gets called in an atomic 131 * context. The order in which locks are grabbed all the way from the point of entry into RM 132 * to the functions defined here is as follows. 133 * 134 * @ref rmMemPoolReserve API Lock -> pPoolLock (mutex) -> Locks inside PMA. 135 * @ref rmMemPoolAllocate API Lock -> GPU Lock -> pPoolLock (mutex) 136 * @ref rmMemPoolFree API Lock -> GPU Lock -> pPoolLock (mutex) 137 * @ref rmMemPoolRelease API Lock -> GPU Lock -> pPoolLock (mutex) 138 */ 139 140 // State of memory pool 141 struct RM_POOL_ALLOC_MEM_RESERVE_INFO 142 { 143 /*! 144 * Pointer to the PMA object. 145 */ 146 PMA *pPma; 147 148 /*! 149 * Mutex to provide exclusive access to the data inside this struct 150 */ 151 PORT_MUTEX *pPoolLock; 152 153 /*! 154 * Index of the topmost pool in the hierarchy 155 */ 156 POOL_IDX topmostPoolIndex; 157 158 /*! 159 * Size of topmost pool's upstream chunk allocated by PMA. 160 */ 161 NvU64 pmaChunkSize; 162 163 /*! 164 * Array of memory pools. 165 */ 166 POOLALLOC *pPool[NUM_POOLS]; 167 168 /*! 169 * Num of allocations made from the pool. 170 */ 171 NvU64 validAllocCount; 172 173 /*! 174 * Skip scrubbing for all allocations made from the pool. 175 */ 176 NvBool bSkipScrub; 177 178 /*! 179 * Automatically trim memory pool when allocation is freed. 180 */ 181 NvBool bTrimOnFree; 182 183 /*! 184 * Allocate pool in protected memory 185 */ 186 NvBool bProtected; 187 }; 188 189 /* ------------------------------------ Static functions --------------------------- */ 190 191 /*! 192 * @brief Used for allocating pages by the upstream allocator for the topmost 193 * pool. 194 * 195 * @param[in] pCtx Context for upstream allocator. 196 * @param[in] pageSize Only for debugging. 197 * @param[in] pPage Output page handle from upstream. 198 * 199 * @return NV_STATUS 200 */ 201 static NV_STATUS 202 allocUpstreamTopPool 203 ( 204 void *pCtx, 205 NvU64 pageSize, 206 POOLALLOC_HANDLE *pPage 207 ) 208 { 209 PMA_ALLOCATION_OPTIONS allocOptions = {0}; 210 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo; 211 NV_STATUS status; 212 213 NV_ASSERT_OR_RETURN(NULL != pCtx, NV_ERR_INVALID_ARGUMENT); 214 NV_ASSERT_OR_RETURN(NULL != pPage, NV_ERR_INVALID_ARGUMENT); 215 216 // TODO: Replace the direct call to PMA with function pointer. 217 pMemReserveInfo = (RM_POOL_ALLOC_MEM_RESERVE_INFO *)pCtx; 218 allocOptions.flags = PMA_ALLOCATE_PINNED | PMA_ALLOCATE_PERSISTENT | 219 PMA_ALLOCATE_CONTIGUOUS; 220 221 if (pMemReserveInfo->bSkipScrub) 222 { 223 allocOptions.flags |= PMA_ALLOCATE_NO_ZERO; 224 } 225 226 if (pMemReserveInfo->bProtected) 227 { 228 allocOptions.flags |= PMA_ALLOCATE_PROTECTED_REGION; 229 } 230 231 status = pmaAllocatePages(pMemReserveInfo->pPma, 232 pMemReserveInfo->pmaChunkSize/PMA_CHUNK_SIZE_64K, 233 PMA_CHUNK_SIZE_64K, 234 &allocOptions, 235 &pPage->address); 236 if (status != NV_OK) 237 { 238 return status; 239 } 240 241 pPage->pMetadata = NULL; 242 243 return status; 244 } 245 246 /*! 247 * @brief Used for allocating pages by the upstream allocator for the lower 248 * pools. 249 * 250 * @param[in] pCtx Context for upstream allocator. 251 * @param[in] pageSize Only for debugging. 252 * @param[in] pPage Output page handle from upstream. 253 * 254 * @return NV_STATUS 255 */ 256 static NV_STATUS 257 allocUpstreamLowerPools 258 ( 259 void *pCtx, 260 NvU64 pageSize, 261 POOLALLOC_HANDLE *pPage 262 ) 263 { 264 NV_STATUS status; 265 266 NV_ASSERT_OR_RETURN(NULL != pCtx, NV_ERR_INVALID_ARGUMENT); 267 NV_ASSERT_OR_RETURN(NULL != pPage, NV_ERR_INVALID_ARGUMENT); 268 269 status = poolAllocate((POOLALLOC *)pCtx, pPage); 270 NV_ASSERT_OR_RETURN(status == NV_OK, status); 271 272 return status; 273 } 274 275 /*! 276 * @brief Used for freeing pages by the upstream allocator for the topmost 277 * pool. 278 * 279 * @param[in] pCtx Context for upstream allocator. 280 * @param[in] pageSize Only for debugging. 281 * @param[in] pPage Page handle of page to be freed. 282 * 283 * @return 284 */ 285 static void 286 freeUpstreamTopPool 287 ( 288 void *pCtx, 289 NvU64 pageSize, 290 POOLALLOC_HANDLE *pPage 291 ) 292 { 293 NvU32 flags = 0; 294 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo; 295 296 NV_ASSERT_OR_RETURN_VOID(NULL != pCtx); 297 NV_ASSERT_OR_RETURN_VOID(NULL != pPage); 298 299 // TODO: Replace the direct call to PMA with function pointer. 300 pMemReserveInfo = (RM_POOL_ALLOC_MEM_RESERVE_INFO *)pCtx; 301 302 if (pMemReserveInfo->bSkipScrub) 303 { 304 flags |= PMA_FREE_SKIP_SCRUB; 305 } 306 307 pmaFreePages(pMemReserveInfo->pPma, &(pPage->address), 1, 308 pMemReserveInfo->pmaChunkSize, flags); 309 } 310 311 /*! 312 * @brief Used for freeing pages by the upstream allocator for the lower 313 * pools. 314 * 315 * @param[in] pCtx Context for upstream allocator. 316 * @param[in] pageSize Only for debugging. 317 * @param[in] pPage Page handle of page to be freed. 318 * 319 * @return 320 */ 321 static void 322 freeUpstreamLowerPools 323 ( 324 void *pCtx, 325 NvU64 pageSize, 326 POOLALLOC_HANDLE *pPage 327 ) 328 { 329 NV_ASSERT_OR_RETURN_VOID(NULL != pCtx); 330 NV_ASSERT_OR_RETURN_VOID(NULL != pPage); 331 332 poolFree((POOLALLOC *)pCtx, pPage); 333 } 334 335 /*! 336 * @brief Increments the refcount whenever an allocation is made 337 * from the pool. 338 * 339 * @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data 340 * 341 * @return 342 */ 343 static void 344 rmMemPoolAddRef 345 ( 346 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo 347 ) 348 { 349 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 350 351 pMemReserveInfo->validAllocCount++; 352 } 353 354 /*! 355 * @brief Decrements the refcount whenever an allocation is freed and 356 * returned to the pool. 357 * 358 * @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data 359 * 360 * @return 361 */ 362 static void 363 rmMemPoolRemoveRef 364 ( 365 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo 366 ) 367 { 368 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 369 NV_ASSERT_OR_RETURN_VOID(pMemReserveInfo->validAllocCount > 0); 370 371 pMemReserveInfo->validAllocCount--; 372 } 373 374 /*! 375 * @brief Gets the number of vaspaces that are being served by the pools. 376 * 377 * @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data 378 * 379 * @return 380 */ 381 static NvU64 382 rmMemPoolGetRef 383 ( 384 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo 385 ) 386 { 387 NV_ASSERT_OR_RETURN(NULL != pMemReserveInfo, 0); 388 389 return pMemReserveInfo->validAllocCount; 390 } 391 392 /* -------------------------------------- Public functions ---------------------------------- */ 393 394 NV_STATUS 395 rmMemPoolSetup 396 ( 397 void *pCtx, 398 RM_POOL_ALLOC_MEM_RESERVE_INFO **ppMemReserveInfo, 399 POOL_CONFIG_MODE configMode 400 ) 401 { 402 NvS32 poolIndex; 403 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo; 404 NV_STATUS status; 405 NvU32 flags = 0; 406 407 if (configMode >= POOL_CONFIG_MAX_SUPPORTED) 408 { 409 return NV_ERR_INVALID_PARAMETER; 410 } 411 412 pMemReserveInfo = (RM_POOL_ALLOC_MEM_RESERVE_INFO *)portMemAllocNonPaged(sizeof(*pMemReserveInfo)); 413 if (NULL == pMemReserveInfo) 414 { 415 return NV_ERR_NO_MEMORY; 416 } 417 418 portMemSet(pMemReserveInfo, 0, sizeof(*pMemReserveInfo)); 419 420 *ppMemReserveInfo = pMemReserveInfo; 421 422 pMemReserveInfo->pPma = (PMA *)pCtx; 423 424 // 425 // poolConfig is a 2D array where each row is a pre-defined configuration mode 426 // For example, POOL_CONFIG_GMMU_FMT_1 means this pool is used for allocating PTE/PDE entries for the GMMU_FMT_VERSION_1 427 // First column in poolConfig corresponds to topmostPoolIndex for a given config 428 // Second column in poolConfig corresponds to chunk size for a given config 429 // 430 pMemReserveInfo->topmostPoolIndex = poolConfig[configMode][POOL_CONFIG_POOL_IDX]; 431 pMemReserveInfo->pmaChunkSize = poolConfig[configMode][POOL_CONFIG_CHUNKSIZE_IDX]; 432 433 // 434 // The topmost pool is fed pages directly by PMA. 435 // 436 // Calling into PMA with GPU lock acquired may cause deadlocks in case RM 437 // is operating along side UVM. Currently, we don't support UVM on Windows. 438 // So, allow the topmost pool to call into PMA on Windows. This is not 439 // permissible on platforms that support UVM like Linux. 440 // TODO: Remove this special handling for Windows once we have taken care 441 // of reserving memory for page tables required for mapping GR context buffers 442 // in the channel vaspace. See bug 200590870 and 200614517. 443 // 444 if (RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM) 445 { 446 flags = FLD_SET_DRF(_RMPOOL, _FLAGS, _AUTO_POPULATE, _ENABLE, flags); 447 } 448 else 449 { 450 flags = FLD_SET_DRF(_RMPOOL, _FLAGS, _AUTO_POPULATE, _DISABLE, flags); 451 } 452 pMemReserveInfo->pPool[pMemReserveInfo->topmostPoolIndex] = poolInitialize( 453 pMemReserveInfo->pmaChunkSize, 454 poolAllocSizes[pMemReserveInfo->topmostPoolIndex], 455 allocUpstreamTopPool, 456 freeUpstreamTopPool, 457 (void *)pMemReserveInfo, 458 portMemAllocatorGetGlobalNonPaged(), 459 flags); 460 if (NULL == pMemReserveInfo->pPool[pMemReserveInfo->topmostPoolIndex]) 461 { 462 status = NV_ERR_NO_MEMORY; 463 goto done; 464 } 465 466 // 467 // The pools are nested. Starting with the second pool, each is fed 468 // pages by the pool immediately above it in hierarchy. 469 // 470 flags = FLD_SET_DRF(_RMPOOL, _FLAGS, _AUTO_POPULATE, _ENABLE, flags); 471 for (poolIndex = pMemReserveInfo->topmostPoolIndex + 1; poolIndex < NUM_POOLS; poolIndex++) 472 { 473 pMemReserveInfo->pPool[poolIndex] = poolInitialize( 474 poolAllocSizes[poolIndex - 1], 475 poolAllocSizes[poolIndex], 476 allocUpstreamLowerPools, 477 freeUpstreamLowerPools, 478 (void *)pMemReserveInfo->pPool[poolIndex - 1], 479 portMemAllocatorGetGlobalNonPaged(), 480 flags); 481 if (NULL == pMemReserveInfo->pPool[poolIndex]) 482 { 483 status = NV_ERR_NO_MEMORY; 484 goto done; 485 } 486 } 487 488 pMemReserveInfo->pPoolLock = (PORT_MUTEX *)portMemAllocNonPaged(portSyncMutexSize); 489 if (NULL == pMemReserveInfo->pPoolLock) 490 { 491 status = NV_ERR_NO_MEMORY; 492 goto done; 493 } 494 495 status = portSyncMutexInitialize(pMemReserveInfo->pPoolLock); 496 if (NV_OK != status) 497 { 498 portMemFree(pMemReserveInfo->pPoolLock); 499 pMemReserveInfo->pPoolLock = NULL; 500 goto done; 501 } 502 503 if ((configMode == POOL_CONFIG_CTXBUF_4K) || 504 (configMode == POOL_CONFIG_CTXBUF_64K) || 505 (configMode == POOL_CONFIG_CTXBUF_2M) || 506 (configMode == POOL_CONFIG_CTXBUF_512M)) 507 { 508 pMemReserveInfo->bTrimOnFree = NV_FALSE; 509 } 510 else 511 { 512 pMemReserveInfo->bTrimOnFree = NV_TRUE; 513 } 514 done: 515 if (NV_OK != status) 516 { 517 rmMemPoolDestroy(pMemReserveInfo); 518 } 519 return status; 520 } 521 522 523 NV_STATUS 524 rmMemPoolReserve 525 ( 526 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 527 NvU64 size, 528 NvU32 flags 529 ) 530 { 531 NvU64 numChunks; 532 NV_STATUS status = NV_ERR_NO_MEMORY; 533 NvBool bPrevSkipScrubState = NV_FALSE; 534 535 NV_ASSERT_OR_RETURN((NULL != pMemReserveInfo), NV_ERR_INVALID_ARGUMENT); 536 537 portSyncMutexAcquire(pMemReserveInfo->pPoolLock); 538 539 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 540 { 541 bPrevSkipScrubState = pMemReserveInfo->bSkipScrub; 542 pMemReserveInfo->bSkipScrub = NV_TRUE; 543 } 544 545 numChunks = NV_DIV_AND_CEIL(size, pMemReserveInfo->pmaChunkSize); 546 547 // Reserve pages only in the topmost pool. 548 if (NULL != pMemReserveInfo->pPool[pMemReserveInfo->topmostPoolIndex]) 549 { 550 NV_CHECK_OK(status, LEVEL_WARNING, 551 poolReserve(pMemReserveInfo->pPool[pMemReserveInfo->topmostPoolIndex], numChunks)); 552 } 553 554 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 555 { 556 pMemReserveInfo->bSkipScrub = bPrevSkipScrubState; 557 } 558 559 portSyncMutexRelease(pMemReserveInfo->pPoolLock); 560 return status; 561 } 562 563 NV_STATUS 564 rmMemPoolAllocate 565 ( 566 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 567 RM_POOL_ALLOC_MEMDESC *pPoolMemDesc 568 ) 569 { 570 POOLALLOC_HANDLE *pPageHandle = NULL; 571 PoolPageHandleList *pPageHandleList = NULL; 572 NvS32 poolIndex = 0; 573 NvS32 topPool; 574 NV_STATUS status = NV_OK; 575 NvU64 allocSize; 576 NvU32 freeListLength; 577 MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR*)pPoolMemDesc; 578 NvU64 *pPhysicalAddresses = NULL; 579 NvU32 numPages = 0; 580 581 NV_ASSERT_OR_RETURN((NULL != pMemReserveInfo), NV_ERR_INVALID_ARGUMENT); 582 NV_ASSERT_OR_RETURN((NULL != pMemDesc), NV_ERR_INVALID_ARGUMENT); 583 584 topPool = pMemReserveInfo->topmostPoolIndex; 585 pPageHandleList = (PoolPageHandleList *)portMemAllocNonPaged(sizeof(*pPageHandleList)); 586 NV_ASSERT_OR_RETURN(pPageHandleList != NULL, NV_ERR_NO_MEMORY); 587 588 portMemSet(pPageHandleList, 0, sizeof(*pPageHandleList)); 589 listInit(pPageHandleList, portMemAllocatorGetGlobalNonPaged()); 590 591 portSyncMutexAcquire(pMemReserveInfo->pPoolLock); 592 593 poolGetListLength(pMemReserveInfo->pPool[topPool], 594 &freeListLength, NULL, NULL); 595 NV_PRINTF(LEVEL_INFO, 596 "Total size of memory reserved for allocation = 0x%llx Bytes\n", 597 freeListLength * pMemReserveInfo->pmaChunkSize); 598 599 // 600 // The onus is on the caller to pass the correct size info after factoring 601 // in any alignment requirements. The size after factoring in all alignment 602 // requirements is tracked in the ActualSize field. The Size field tracks 603 // the requested size and doesn't take any alignment requirements into 604 // consideration. 605 // 606 allocSize = pMemDesc->ActualSize; 607 608 if (allocSize > poolAllocSizes[topPool]) 609 { 610 numPages = NvU64_LO32(NV_DIV_AND_CEIL(allocSize, poolAllocSizes[topPool])); 611 poolIndex = topPool; 612 } 613 else 614 { 615 for (poolIndex = NUM_POOLS - 1; poolIndex >= topPool; poolIndex--) 616 { 617 if (allocSize <= poolAllocSizes[poolIndex]) 618 { 619 NV_PRINTF(LEVEL_INFO, 620 "Allocating from pool with alloc size = 0x%llx Bytes\n", 621 poolAllocSizes[poolIndex]); 622 break; 623 } 624 } 625 626 if (poolIndex < 0) 627 { 628 status = NV_ERR_NO_MEMORY; 629 goto done; 630 } 631 } 632 633 // 634 // If allocation request is greater than page size of top level pool then 635 // allocate multiple pages from top-level pool 636 // 637 if (numPages > 1) 638 { 639 NvU32 index; 640 641 NV_PRINTF(LEVEL_INFO, 642 "Allocating from pool with alloc size = 0x%llx Bytes\n", 643 poolAllocSizes[topPool] * numPages); 644 645 if (memdescGetContiguity(pMemDesc, AT_GPU)) 646 { 647 status = poolAllocateContig(pMemReserveInfo->pPool[topPool], numPages, pPageHandleList); 648 if (status != NV_OK) 649 { 650 goto done; 651 } 652 pPageHandle = listHead(pPageHandleList); 653 memdescDescribe(pMemDesc, ADDR_FBMEM, pPageHandle->address, pMemDesc->Size); 654 } 655 else 656 { 657 pPhysicalAddresses = (NvU64*)portMemAllocNonPaged(sizeof(*pPhysicalAddresses) * numPages); 658 if (pPhysicalAddresses == NULL) 659 { 660 status = NV_ERR_NO_MEMORY; 661 goto done; 662 } 663 portMemSet(pPhysicalAddresses, 0, sizeof(*pPhysicalAddresses) * numPages); 664 665 for (index = 0; index < numPages; index++) 666 { 667 pPageHandle = listAppendNew(pPageHandleList); 668 if (pPageHandle == NULL) 669 { 670 status = NV_ERR_NO_MEMORY; 671 NV_ASSERT_OR_GOTO((pPageHandle != NULL), done); 672 } 673 status = poolAllocate(pMemReserveInfo->pPool[topPool], pPageHandle); 674 if (status != NV_OK) 675 { 676 // 677 // Remove current pageHandle from the list as its invalid 678 // and we don't want poolFree being called on it as a part of cleanup 679 // 680 listRemove(pPageHandleList, pPageHandle); 681 NV_ASSERT_OR_GOTO(0, done); 682 pPageHandle = NULL; 683 } 684 pPhysicalAddresses[index] = pPageHandle->address; 685 pPageHandle = NULL; 686 } 687 memdescFillPages(pMemDesc, 0, pPhysicalAddresses, numPages, poolAllocSizes[topPool]); 688 portMemFree(pPhysicalAddresses); 689 pPhysicalAddresses = NULL; 690 } 691 } 692 else 693 { 694 pPageHandle = listAppendNew(pPageHandleList); 695 NV_ASSERT_OR_GOTO((NULL != pPageHandle), done); 696 697 status = poolAllocate(pMemReserveInfo->pPool[poolIndex], pPageHandle); 698 if (status != NV_OK) 699 { 700 listRemove(pPageHandleList, pPageHandle); 701 goto done; 702 } 703 704 memdescDescribe(pMemDesc, ADDR_FBMEM, pPageHandle->address, pMemDesc->Size); 705 // memdescDescribe() sets Size and ActualSize to same values. Hence, reassigning 706 pMemDesc->ActualSize = allocSize; 707 pPageHandle = NULL; 708 } 709 710 // save list of page handles in memdesc 711 pMemDesc->pPageHandleList = pPageHandleList; 712 713 // Refcount the pool. 714 rmMemPoolAddRef(pMemReserveInfo); 715 716 done: 717 portMemFree(pPhysicalAddresses); 718 719 if ((status != NV_OK) && (pPageHandleList != NULL)) 720 { 721 if (poolIndex >= 0) 722 { 723 PoolPageHandleListIter it = listIterAll(pPageHandleList); 724 while (listIterNext(&it)) 725 { 726 poolFree(pMemReserveInfo->pPool[poolIndex], it.pValue); 727 } 728 } 729 730 listClear(pPageHandleList); 731 portMemFree(pPageHandleList); 732 } 733 portSyncMutexRelease(pMemReserveInfo->pPoolLock); 734 return status; 735 } 736 737 /*! 738 * @brief Returns any unused nodes from the topmost level of a pool hierarchy 739 * back to PMA. 740 * 741 * @param[in] pMemReserveInfo Pointer to the RM_POOL_ALLOC_MEM_RESERVE_INFO data 742 * @param[in] nodesToPreserve Number of nodes to preserve in the topmost pool 743 * @param[in] flags VASpace flags to skip scrubbing 744 * 745 * @return 746 */ 747 void 748 rmMemPoolTrim 749 ( 750 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 751 NvU32 nodesToPreserve, 752 NvU32 flags 753 ) 754 { 755 NvBool bPrevSkipScrubState = NV_FALSE; 756 757 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 758 759 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 760 { 761 bPrevSkipScrubState = pMemReserveInfo->bSkipScrub; 762 pMemReserveInfo->bSkipScrub = NV_TRUE; 763 } 764 765 poolTrim(pMemReserveInfo->pPool[pMemReserveInfo->topmostPoolIndex], 766 nodesToPreserve); 767 768 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 769 { 770 pMemReserveInfo->bSkipScrub = bPrevSkipScrubState; 771 } 772 } 773 774 void 775 rmMemPoolFree 776 ( 777 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 778 RM_POOL_ALLOC_MEMDESC *pPoolAllocMemDesc, 779 NvU32 flags 780 ) 781 { 782 MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR*)pPoolAllocMemDesc; 783 NvS32 poolIndex = 0; 784 NvU64 allocSize; 785 PoolPageHandleListIter it; 786 NvU32 topPool; 787 788 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 789 NV_ASSERT_OR_RETURN_VOID(NULL != pMemDesc); 790 NV_ASSERT_OR_RETURN_VOID((pMemDesc->pPageHandleList != NULL) && 791 (listCount(pMemDesc->pPageHandleList) != 0)); 792 793 portSyncMutexAcquire(pMemReserveInfo->pPoolLock); 794 795 // 796 // Refcount can be greater than 1 in case of shared vaspaces (as in UVM). 797 // In this case, RM's internal PDB may be refcounted and a reference 798 // stored internally for later revoke. 799 // 800 if (pMemDesc->RefCount > 1) 801 { 802 goto done; 803 } 804 805 topPool = pMemReserveInfo->topmostPoolIndex; 806 807 // Use the ActualSize value to look up the pools 808 allocSize = pMemDesc->ActualSize; 809 810 // 811 // If allocation was greater than page size of top level pool then 812 // multiple pages were allocated from top pool and we need to free them all. 813 // 814 if (allocSize > poolAllocSizes[topPool]) 815 { 816 poolIndex = topPool; 817 } 818 else 819 { 820 for (poolIndex = NUM_POOLS - 1; poolIndex >= 0; poolIndex--) 821 { 822 if ((NULL != pMemReserveInfo->pPool[poolIndex]) && 823 (allocSize <= poolAllocSizes[poolIndex])) 824 { 825 break; 826 } 827 } 828 } 829 NV_ASSERT_OR_GOTO((poolIndex >= 0), done); 830 831 it = listIterAll(pMemDesc->pPageHandleList); 832 while (listIterNext(&it)) 833 { 834 poolFree(pMemReserveInfo->pPool[poolIndex], it.pValue); 835 } 836 listClear(pMemDesc->pPageHandleList); 837 portMemFree(pMemDesc->pPageHandleList); 838 pMemDesc->pPageHandleList = NULL; 839 840 rmMemPoolRemoveRef(pMemReserveInfo); 841 842 // Trim the topmost pool so that any unused pages are returned to PMA. 843 if (pMemReserveInfo->bTrimOnFree) 844 { 845 rmMemPoolTrim(pMemReserveInfo, 1, flags); 846 } 847 done: 848 portSyncMutexRelease(pMemReserveInfo->pPoolLock); 849 } 850 851 void 852 rmMemPoolRelease 853 ( 854 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 855 NvU32 flags 856 ) 857 { 858 NvS32 poolIndex; 859 NvU32 freeListLength; 860 NvU32 partialListLength; 861 NvU32 fullListLenght; 862 NvBool bPrevSkipScrubState = NV_FALSE; 863 864 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 865 866 portSyncMutexAcquire(pMemReserveInfo->pPoolLock); 867 868 // 869 // A refcount equal to zero implies that there are no unfreed page level 870 // instances. At this point the lowermost pool should have only its freelist 871 // non empty. An unfreed allocation in a lower level pool implies non empty 872 // partial lists and full lists in the pools above it. We free the pools 873 // from bottom to top so that by the time we come to the topmost pool, all 874 // allocations are present in the freelist of the topmost pool. The topmost 875 // pool can then return all the memory back to PMA. The pools can get memory 876 // via a call to rmMemPoolReserve() 877 // 878 if (rmMemPoolGetRef(pMemReserveInfo) != 0) 879 { 880 goto done; 881 } 882 883 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 884 { 885 bPrevSkipScrubState = pMemReserveInfo->bSkipScrub; 886 pMemReserveInfo->bSkipScrub = NV_TRUE; 887 } 888 889 for (poolIndex = NUM_POOLS - 1; poolIndex >= 0; poolIndex--) 890 { 891 if (NULL != pMemReserveInfo->pPool[poolIndex]) 892 { 893 // 894 // Since this function gets called only when validAlloCount is zero, 895 // the fullList and the partialList are expected to be empty. All 896 // allocations (if any) should be only in the freelist at this point. 897 // 898 poolGetListLength(pMemReserveInfo->pPool[poolIndex], &freeListLength, 899 &partialListLength, &fullListLenght); 900 NV_ASSERT(partialListLength == 0); 901 NV_ASSERT(fullListLenght == 0); 902 903 // poolTrim() trims only the freelist. 904 poolTrim(pMemReserveInfo->pPool[poolIndex], 0); 905 } 906 } 907 908 if (flags & VASPACE_FLAGS_SKIP_SCRUB_MEMPOOL) 909 { 910 pMemReserveInfo->bSkipScrub = bPrevSkipScrubState; 911 } 912 913 done: 914 portSyncMutexRelease(pMemReserveInfo->pPoolLock); 915 } 916 917 void 918 rmMemPoolDestroy 919 ( 920 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo 921 ) 922 { 923 NvS32 poolIndex; 924 NvU32 freeListLength; 925 NvU32 partialListLength; 926 NvU32 fullListLenght; 927 928 NV_ASSERT_OR_RETURN_VOID(NULL != pMemReserveInfo); 929 930 NV_ASSERT(rmMemPoolGetRef(pMemReserveInfo) == 0); 931 932 // 933 // Always free pools from bottom to top since the lower pools return 934 // their pages to the pool just above during free. The topmost pool will 935 // return the it's pages back to PMA. 936 // 937 for (poolIndex = NUM_POOLS - 1; poolIndex >= 0; poolIndex--) 938 { 939 if (NULL != pMemReserveInfo->pPool[poolIndex]) 940 { 941 poolGetListLength(pMemReserveInfo->pPool[poolIndex], &freeListLength, 942 &partialListLength, &fullListLenght); 943 NV_ASSERT(freeListLength == 0); 944 NV_ASSERT(partialListLength == 0); 945 NV_ASSERT(fullListLenght == 0); 946 947 poolDestroy(pMemReserveInfo->pPool[poolIndex]); 948 } 949 } 950 951 if (NULL != pMemReserveInfo->pPoolLock) 952 { 953 portSyncMutexDestroy(pMemReserveInfo->pPoolLock); 954 portMemFree(pMemReserveInfo->pPoolLock); 955 pMemReserveInfo->pPoolLock = NULL; 956 } 957 958 portMemFree(pMemReserveInfo); 959 pMemReserveInfo = NULL; 960 } 961 962 NvBool 963 rmMemPoolIsScrubSkipped 964 ( 965 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo 966 ) 967 { 968 NV_ASSERT_OR_RETURN(pMemReserveInfo != NULL, NV_FALSE); 969 return pMemReserveInfo->bSkipScrub; 970 } 971 972 void 973 rmMemPoolSkipScrub 974 ( 975 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 976 NvBool bSkipScrub 977 ) 978 { 979 NV_ASSERT_OR_RETURN_VOID(pMemReserveInfo != NULL); 980 pMemReserveInfo->bSkipScrub = bSkipScrub; 981 } 982 983 984 NV_STATUS 985 rmMemPoolGetChunkAndPageSize 986 ( 987 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 988 NvU64 *pChunkSize, 989 NvU64 *pPageSize 990 ) 991 { 992 NV_ASSERT_OR_RETURN(pMemReserveInfo != NULL, NV_ERR_INVALID_ARGUMENT); 993 NV_ASSERT_OR_RETURN((pChunkSize != NULL) && (pPageSize != NULL), NV_ERR_INVALID_ARGUMENT); 994 *pChunkSize = pMemReserveInfo->pmaChunkSize; 995 *pPageSize = poolAllocSizes[pMemReserveInfo->topmostPoolIndex]; 996 return NV_OK; 997 } 998 999 void 1000 rmMemPoolAllocateProtectedMemory 1001 ( 1002 RM_POOL_ALLOC_MEM_RESERVE_INFO *pMemReserveInfo, 1003 NvBool bProtected 1004 ) 1005 { 1006 NV_ASSERT_OR_RETURN_VOID(pMemReserveInfo != NULL); 1007 pMemReserveInfo->bProtected = bProtected; 1008 } 1009