1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.h" 25 #include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h" 26 #include "gpu/mem_mgr/mem_scrub.h" 27 #include "utils/nvprintf.h" 28 #include "utils/nvassert.h" 29 30 #if !defined(SRT_BUILD) 31 // These files are not found on SRT builds 32 #include "os/os.h" 33 #else 34 static NvU64 osGetPageRefcount(NvU64 sysPagePhysAddr) 35 { 36 return 0; 37 } 38 39 static NvU64 osCountTailPages(NvU64 sysPagePhysAddr) 40 { 41 return 0; 42 } 43 44 static void osAllocReleasePage(NvU64 sysPagePhysAddr, NvU32 pageCount) 45 { 46 return; 47 } 48 49 static NV_STATUS osOfflinePageAtAddress(NvU64 address) 50 { 51 return NV_ERR_GENERIC; 52 } 53 54 static NvU8 osGetPageShift(void) 55 { 56 return 0; 57 } 58 59 NV_STATUS scrubCheck(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *size) 60 { 61 return NV_ERR_GENERIC; 62 } 63 64 NV_STATUS scrubSubmitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages, 65 NvU64 pageCount, PSCRUB_NODE *ppList, NvU64 *size) 66 { 67 return NV_ERR_GENERIC; 68 } 69 70 NV_STATUS scrubWaitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages, NvU32 pageCount) 71 { 72 return NV_ERR_GENERIC; 73 } 74 75 NV_STATUS scrubCheckAndWaitForSize (OBJMEMSCRUB *pScrubber, NvU64 numPages, 76 NvU64 pageSize, PSCRUB_NODE *ppList, NvU64 *pSize) 77 { 78 return NV_ERR_GENERIC; 79 } 80 #endif 81 82 // Local helpers 83 NvU32 84 findRegionID(PMA *pPma, NvU64 address) 85 { 86 NvU32 i; 87 88 for (i = 0; i < pPma->regSize; i++) 89 { 90 NvU64 start, limit; 91 start = pPma->pRegDescriptors[i]->base; 92 limit = pPma->pRegDescriptors[i]->limit; 93 if (address >= start && address <= limit) 94 { 95 return i; 96 } 97 } 98 99 // Should never get here 100 NV_ASSERT(0); 101 return 0; 102 } 103 104 105 void 106 pmaPrintBlockStatus(PMA_PAGESTATUS blockStatus) 107 { 108 // Use DBG_PRINTF so as not to prepend "NVRM:" everytime, as NV_PRINTF does 109 if ((blockStatus & STATE_MASK) == STATE_FREE) { 110 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "STATE_FREE "); 111 } 112 else if ((blockStatus & STATE_MASK) == STATE_UNPIN) { 113 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "STATE_UNPIN "); 114 } 115 else if ((blockStatus & STATE_MASK) == STATE_PIN) { 116 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "STATE_PIN "); 117 } 118 else { 119 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "UNKNOWN STATE"); 120 } 121 122 if (blockStatus & ATTRIB_PERSISTENT) { 123 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " | ATTRIB_PERSISTENT"); 124 } 125 else { 126 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " "); 127 } 128 129 if (blockStatus & ATTRIB_SCRUBBING) { 130 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " | ATTRIB_SCRUBBING "); 131 } 132 else { 133 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " "); 134 } 135 136 if (blockStatus & ATTRIB_EVICTING) { 137 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " | ATTRIB_EVICTING "); 138 } 139 else { 140 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " "); 141 } 142 143 if (blockStatus & ATTRIB_BLACKLIST) { 144 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " | ATTRIB_BLACKLIST "); 145 } 146 else { 147 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, " "); 148 } 149 150 NV_PRINTF_EX(NV_PRINTF_MODULE, LEVEL_INFO, "\n"); 151 } 152 153 void 154 pmaRegionPrint(PMA *pPma, PMA_REGION_DESCRIPTOR *pRegion, void *pMap) 155 { 156 NvU32 i; 157 PMA_PAGESTATUS currStatus, blockStatus = STATE_FREE; 158 NvU64 addrBase, addrLimit, numFrames, blockStart = 0; 159 160 NV_ASSERT(pRegion != NULL); 161 NV_ASSERT(pMap != NULL); 162 163 (void)blockStart; //Silence the compiler 164 165 addrBase = pRegion->base; 166 addrLimit = pRegion->limit; 167 numFrames = (addrLimit - addrBase + 1) >> PMA_PAGE_SHIFT; 168 169 NV_PRINTF(LEVEL_INFO, "Region: 0x%llx..0x%llx\n", addrBase, addrLimit); 170 NV_PRINTF(LEVEL_INFO, "Total frames: 0x%llx\n", numFrames); 171 172 for (i = 0; i < numFrames; i++) 173 { 174 currStatus = pPma->pMapInfo->pmaMapRead(pMap, i, NV_TRUE); 175 if (i == 0) 176 { 177 blockStatus = currStatus; 178 blockStart = i; 179 } 180 181 if (blockStatus != currStatus) 182 { 183 NV_PRINTF(LEVEL_INFO, "%8llx..%8x: ", blockStart, i-1); 184 pmaPrintBlockStatus(blockStatus); 185 186 blockStatus = currStatus; 187 blockStart = i; 188 } 189 } 190 NV_PRINTF(LEVEL_INFO, "%8llx..%8x: ", blockStart, i-1); 191 pmaPrintBlockStatus(blockStatus); 192 } 193 194 NvBool 195 pmaStateCheck(PMA *pPma) 196 { 197 NvU32 size, i; 198 PMA_REGION_DESCRIPTOR *pState; 199 void *pMap = NULL; 200 201 if (pPma == NULL) return NV_FALSE; 202 203 size = pPma->regSize; 204 if (size >= PMA_REGION_SIZE) return NV_FALSE; 205 206 if (pPma->bNuma) 207 { 208 if (!pPma->nodeOnlined) 209 { 210 NV_PRINTF(LEVEL_INFO, "Warning: NUMA state not onlined.\n"); 211 return NV_TRUE; 212 } 213 else if (pPma->numaNodeId == PMA_NUMA_NO_NODE) 214 { 215 NV_PRINTF(LEVEL_INFO, "NUMA node ID invalid.\n"); 216 return NV_FALSE; 217 } 218 } 219 220 for (i = 0; i < size; i++) 221 { 222 pMap = pPma->pRegions[i]; 223 pState = pPma->pRegDescriptors[i]; 224 225 if (pMap == NULL || pState == NULL) return NV_FALSE; 226 } 227 228 return NV_TRUE; 229 } 230 231 NV_STATUS 232 pmaCheckRangeAgainstRegionDesc 233 ( 234 PMA *pPma, 235 NvU64 base, 236 NvU64 size 237 ) 238 { 239 PMA_REGION_DESCRIPTOR *pRegionDesc; 240 NvU32 regId = findRegionID(pPma, base); 241 pRegionDesc = pPma->pRegDescriptors[regId]; 242 243 if ((base < pRegionDesc->base) || 244 ((base + size - 1) > pRegionDesc->limit)) 245 { 246 return NV_ERR_INVALID_STATE; 247 } 248 249 return NV_OK; 250 } 251 252 void 253 pmaSetBlockStateAttribUnderPmaLock 254 ( 255 PMA *pPma, 256 NvU64 base, 257 NvU64 size, 258 PMA_PAGESTATUS pmaState, 259 PMA_PAGESTATUS pmaStateWriteMask 260 ) 261 { 262 NvU64 numFrames, baseFrame, i; 263 NvS32 regId; 264 void *pMap; 265 266 NV_ASSERT(pPma != NULL); 267 NV_ASSERT(NV_IS_ALIGNED(base, PMA_GRANULARITY)); 268 NV_ASSERT(NV_IS_ALIGNED(size, PMA_GRANULARITY)); 269 270 regId = findRegionID(pPma, base); // assume same id for base+size TODO check this 271 pMap = pPma->pRegions[regId]; 272 273 numFrames = size >> PMA_PAGE_SHIFT; 274 baseFrame = (base - pPma->pRegDescriptors[regId]->base) >> PMA_PAGE_SHIFT; 275 276 // Ensure accessing the frame data would not go out of bound in lower level 277 NV_ASSERT_OR_RETURN_VOID((base + size - 1) <= pPma->pRegDescriptors[regId]->limit); 278 279 for (i = 0; i < numFrames; i++) 280 { 281 pPma->pMapInfo->pmaMapChangeStateAttribEx(pMap, (baseFrame + i), pmaState, pmaStateWriteMask); 282 } 283 } 284 285 void 286 pmaSetBlockStateAttrib 287 ( 288 PMA *pPma, 289 NvU64 base, 290 NvU64 size, 291 PMA_PAGESTATUS pmaState, 292 PMA_PAGESTATUS pmaStateWriteMask 293 ) 294 { 295 NV_ASSERT(pPma != NULL); 296 297 portSyncSpinlockAcquire(pPma->pPmaLock); 298 299 pmaSetBlockStateAttribUnderPmaLock(pPma, base, size, pmaState, pmaStateWriteMask); 300 301 portSyncSpinlockRelease(pPma->pPmaLock); 302 } 303 304 // This must be called with the PMA lock held! 305 void 306 pmaStatsUpdateState 307 ( 308 NvU64 *pNumFree, 309 NvU64 numPages, 310 PMA_PAGESTATUS oldState, 311 PMA_PAGESTATUS newState 312 ) 313 { 314 NV_ASSERT(pNumFree != NULL); 315 316 oldState &= STATE_MASK; 317 newState &= STATE_MASK; 318 319 if ((oldState == STATE_FREE) && (newState != STATE_FREE)) 320 { 321 (*pNumFree) -= numPages; 322 // NV_PRINTF(LEVEL_INFO, "Decrease to 0x%llx \n", *pNumFree); 323 } 324 else if ((oldState != STATE_FREE) && (newState == STATE_FREE)) 325 { 326 (*pNumFree) += numPages; 327 // NV_PRINTF(LEVEL_INFO, "Increase to 0x%llx \n", *pNumFree); 328 } 329 } 330 331 NvBool pmaIsEvictionPending(PMA *pPma) 332 { 333 NvU32 i; 334 void *pMap = NULL; 335 336 for (i = 0; i < pPma->regSize; ++i) 337 { 338 pMap = pPma->pRegions[i]; 339 if (pPma->pMapInfo->pmaMapGetEvictingFrames(pMap) > 0) 340 return NV_TRUE; 341 } 342 343 return NV_FALSE; 344 } 345 346 void pmaOsSchedule(void) 347 { 348 // TODO Move osSchedule() to nvport? 349 #if !defined(SRT_BUILD) 350 osSchedule(); 351 #endif 352 } 353 354 /*! 355 * @brief Handle eviction results from UVM and free the reuse pages to 356 * OS if eviction failed half-way. 357 * If eviction was successful, we have to double check the refcount and 358 * decide if it's ok to reuse the pages for this eviction. 359 * See bug 2019754. 360 */ 361 static NV_STATUS 362 _pmaCleanupNumaReusePages 363 ( 364 PMA *pPma, 365 NvU64 evictStart, 366 NvU64 numFrames, 367 NvBool bEvictionSucceeded 368 ) 369 { 370 NvU32 regId; 371 NvU64 sysPhysAddr = 0, sysPagePhysAddr = 0; 372 NvU64 frameNum, addrBase, i; 373 PMA_PAGESTATUS currentStatus; 374 NvBool bRaisedRefcount = NV_FALSE; 375 376 regId = findRegionID(pPma, evictStart); 377 addrBase = pPma->pRegDescriptors[regId]->base; 378 frameNum = PMA_ADDR2FRAME(evictStart, addrBase); 379 sysPhysAddr = evictStart + pPma->coherentCpuFbBase; 380 381 if (bEvictionSucceeded == NV_TRUE) 382 { 383 // 384 // If eviction from UVM succeeded, we double check the refcount and 385 // update whether we should reuse these pages or not. If refcount is 386 // greater than the appropriate number (1 for non-compound pages; for 387 // compound pages, refcount should be equal to the number of pages 388 // in this compound page), that means someone called get_user_pages 389 // on those pages and we need to fail this eviction. 390 // 391 for (i = 0; i < numFrames; i++) 392 { 393 sysPagePhysAddr = sysPhysAddr + (i << PMA_PAGE_SHIFT); 394 395 if (osGetPageRefcount(sysPagePhysAddr) > osCountTailPages(sysPagePhysAddr)) 396 { 397 bRaisedRefcount = NV_TRUE; 398 break; 399 } 400 } 401 } 402 403 if (!bEvictionSucceeded || bRaisedRefcount) 404 { 405 // 406 // Eviction Failed. Need to clean up. 407 // Since we set the NUMA_REUSE bit when we decide to reuse the pages, 408 // we know exactly which pages to free both to OS and in PMA bitmap. 409 // 410 NvU8 osPageShift = osGetPageShift(); 411 412 NV_ASSERT_OR_RETURN(PMA_PAGE_SHIFT >= osPageShift, NV_ERR_INVALID_STATE); 413 414 for (i = 0; i < numFrames; i++) 415 { 416 currentStatus = pPma->pMapInfo->pmaMapRead(pPma->pRegions[regId], (frameNum + i), NV_TRUE); 417 sysPagePhysAddr = sysPhysAddr + (i << PMA_PAGE_SHIFT); 418 419 if (currentStatus & ATTRIB_NUMA_REUSE) 420 { 421 osAllocReleasePage(sysPagePhysAddr, 1 << (PMA_PAGE_SHIFT - osPageShift)); 422 pPma->pMapInfo->pmaMapChangeStateAttribEx(pPma->pRegions[regId], (frameNum + i), 423 STATE_FREE, (STATE_MASK | ATTRIB_NUMA_REUSE)); 424 } 425 } 426 427 return NV_ERR_NO_MEMORY; 428 } 429 430 return NV_OK; 431 } 432 433 434 /*! 435 * @brief Eviction for contiguous allocation always evicts the full 436 * range to be allocated and the pmaMapScanContiguous() 437 * function sets the address to start eviction at as the first 438 * entry in the array of pages. 439 */ 440 NV_STATUS 441 _pmaEvictContiguous 442 ( 443 PMA *pPma, 444 void *pMap, 445 NvU64 evictStart, 446 NvU64 evictEnd, 447 MEMORY_PROTECTION prot 448 ) 449 { 450 NV_STATUS status; 451 NvU64 numFramesToEvict; 452 NvU64 evictSize; 453 NvU64 frameEvictionsInProcess = pPma->pMapInfo->pmaMapGetEvictingFrames(pMap); 454 NvBool pmaNumaEnabled = pPma->bNuma; 455 456 evictSize = evictEnd - evictStart + 1; 457 numFramesToEvict = evictSize >> PMA_PAGE_SHIFT; 458 frameEvictionsInProcess += numFramesToEvict; 459 pPma->pMapInfo->pmaMapSetEvictingFrames(pMap, frameEvictionsInProcess); 460 461 pmaSetBlockStateAttribUnderPmaLock(pPma, evictStart, evictSize, ATTRIB_EVICTING, ATTRIB_EVICTING); 462 463 // Release PMA lock before calling into UVM for eviction. 464 portSyncSpinlockRelease(pPma->pPmaLock); 465 466 if (pPma->bScrubOnFree) 467 { 468 PSCRUB_NODE pPmaScrubList = NULL; 469 portSyncMutexRelease(pPma->pAllocLock); 470 471 status = pPma->evictRangeCb(pPma->evictCtxPtr, evictStart, evictEnd, prot); 472 473 portSyncMutexAcquire(pPma->pAllocLock); 474 475 NV_PRINTF(LEVEL_INFO, "evictRangeCb returned with status %llx\n", (NvU64)status); 476 477 if (status != NV_OK) 478 { 479 goto evict_cleanup; 480 } 481 // For NUMA we will scrub only what's being evicted and returned to client. 482 if (pmaNumaEnabled) 483 { 484 // 485 // The evicting contiguous range is marked as ATTRIB_EVICTING 486 // and hence there will be no page stealing. 487 // 488 NvU64 count; 489 490 if ((status = scrubSubmitPages(pPma->pScrubObj, (NvU32)evictSize, &evictStart, 491 1, &pPmaScrubList, &count)) != NV_OK) 492 { 493 status = NV_ERR_INSUFFICIENT_RESOURCES; 494 goto scrub_exit; 495 } 496 497 if (count > 0) 498 _pmaClearScrubBit(pPma, pPmaScrubList, count); 499 } 500 501 if ((status = _pmaCheckScrubbedPages(pPma, evictSize, &evictStart, 1)) != NV_OK) 502 { 503 status = NV_ERR_INSUFFICIENT_RESOURCES; 504 goto scrub_exit; // just incase someone adds anything below. 505 } 506 507 scrub_exit: 508 portMemFree(pPmaScrubList); 509 510 if (!pmaNumaEnabled && 511 (status == NV_ERR_INSUFFICIENT_RESOURCES)) 512 { 513 NV_PRINTF(LEVEL_INFO, "ERROR: scrubber OOM!\n"); 514 goto exit; // fix this later, never exit early violating lock semantics 515 } 516 } 517 else 518 { 519 status = pPma->evictRangeCb(pPma->evictCtxPtr, evictStart, evictEnd, prot); 520 NV_PRINTF(LEVEL_INFO, "evictRangeCb returned with status %llx\n", (NvU64)status); 521 } 522 523 evict_cleanup: 524 // Reacquire PMA lock after returning from UVM and scrubber. 525 portSyncSpinlockAcquire(pPma->pPmaLock); 526 527 // 528 // When we are in NUMA mode, we need to double check the NUMA_REUSE page attribute 529 // to possibly return these pages to OS. 530 // 531 if (pmaNumaEnabled) 532 { 533 status = _pmaCleanupNumaReusePages(pPma, evictStart, numFramesToEvict, (status == NV_OK)); 534 } 535 536 pmaSetBlockStateAttribUnderPmaLock(pPma, evictStart, evictSize, 0, ATTRIB_EVICTING | ATTRIB_NUMA_REUSE); 537 538 frameEvictionsInProcess = pPma->pMapInfo->pmaMapGetEvictingFrames(pMap); 539 NV_ASSERT(frameEvictionsInProcess >= numFramesToEvict); 540 pPma->pMapInfo->pmaMapSetEvictingFrames(pMap, (frameEvictionsInProcess - numFramesToEvict)); 541 542 exit: 543 return status; 544 } 545 546 /*! 547 * @brief Eviction for a non-contiguous range will allow the UVM driver to pick 548 * and evict the specific pages being evicted. The UVM driver is required to hand 549 * back pages to PMA in STATE_PIN state to prevent page stealing. 550 */ 551 NV_STATUS 552 _pmaEvictPages 553 ( 554 PMA *pPma, 555 void *pMap, 556 NvU64 *evictPages, 557 NvU64 evictPageCount, 558 NvU64 *allocPages, 559 NvU64 allocPageCount, 560 NvU64 pageSize, 561 NvU64 physBegin, 562 NvU64 physEnd, 563 MEMORY_PROTECTION prot 564 ) 565 { 566 NvU64 i; 567 NV_STATUS status; 568 NvU64 numFramesToEvict = evictPageCount * (pageSize >> PMA_PAGE_SHIFT); 569 NvU64 frameEvictionsInProcess = pPma->pMapInfo->pmaMapGetEvictingFrames(pMap); 570 NvBool pmaNumaEnabled = pPma->bNuma; 571 572 frameEvictionsInProcess += numFramesToEvict; 573 pPma->pMapInfo->pmaMapSetEvictingFrames(pMap, frameEvictionsInProcess); 574 575 // 576 // Pin all the already allocated pages before unlocking the PMA 577 // lock to prevent them from being allocated while eviction is 578 // happening. 579 // 580 for (i = 0; i < allocPageCount; i++) 581 pmaSetBlockStateAttribUnderPmaLock(pPma, allocPages[i], pageSize, STATE_PIN, STATE_PIN); 582 583 // Release PMA lock before calling into UVM for eviction. 584 portSyncSpinlockRelease(pPma->pPmaLock); 585 586 if (pPma->bScrubOnFree) 587 { 588 PSCRUB_NODE pPmaScrubList = NULL; 589 NvU64 count = 0; 590 591 portSyncMutexRelease(pPma->pAllocLock); 592 status = pPma->evictPagesCb(pPma->evictCtxPtr, pageSize, evictPages, 593 (NvU32)evictPageCount, physBegin, physEnd, prot); 594 portSyncMutexAcquire(pPma->pAllocLock); 595 596 NV_PRINTF(LEVEL_INFO, "evictPagesCb returned with status %llx\n", (NvU64)status); 597 598 if (status != NV_OK) 599 { 600 goto evict_cleanup; 601 } 602 603 // Don't need to mark ATTRIB_SCRUBBING to protect the pages because they are already pinned 604 status = scrubSubmitPages(pPma->pScrubObj, pageSize, evictPages, 605 (NvU32)evictPageCount, &pPmaScrubList, &count); 606 NV_ASSERT_OR_GOTO((status == NV_OK), scrub_exit); 607 608 if (count > 0) 609 _pmaClearScrubBit(pPma, pPmaScrubList, count); 610 611 // Wait for our scrubbing to complete 612 status = _pmaCheckScrubbedPages(pPma, pageSize, evictPages, (NvU32)evictPageCount); 613 scrub_exit: 614 // Free the actual list, although allocated by objscrub 615 portMemFree(pPmaScrubList); 616 617 if ((status != NV_OK) && !pmaNumaEnabled) 618 { 619 status = NV_ERR_INSUFFICIENT_RESOURCES; // Caller expects this status. 620 NV_PRINTF(LEVEL_ERROR, "ERROR: scrubber OOM!\n"); 621 NV_ASSERT_OK_OR_RETURN(status); 622 } 623 } 624 else 625 { 626 status = pPma->evictPagesCb(pPma->evictCtxPtr, pageSize, evictPages, 627 (NvU32)evictPageCount, physBegin, physEnd, prot); 628 NV_PRINTF(LEVEL_INFO, "evictPagesCb returned with status %llx\n", (NvU64)status); 629 } 630 631 evict_cleanup: 632 // Reacquire PMA lock after returning from UVM. 633 portSyncSpinlockAcquire(pPma->pPmaLock); 634 635 // Unpin the allocations now that we reacquired the PMA lock. 636 for (i = 0; i < allocPageCount; i++) 637 pmaSetBlockStateAttribUnderPmaLock(pPma, allocPages[i], pageSize, 0, STATE_PIN); 638 639 frameEvictionsInProcess = pPma->pMapInfo->pmaMapGetEvictingFrames(pMap); 640 NV_ASSERT(frameEvictionsInProcess >= numFramesToEvict); 641 pPma->pMapInfo->pmaMapSetEvictingFrames(pMap, (frameEvictionsInProcess - numFramesToEvict)); 642 643 return status; 644 } 645 646 // 647 // Region selector 648 // Given specific PMA_ALLOCATE_* requirements, generate a list of possible intersecting regions 649 // Invalid regionList IDs set to -1 650 // 651 NV_STATUS 652 pmaSelector 653 ( 654 PMA *pPma, 655 PMA_ALLOCATION_OPTIONS *allocationOptions, 656 NvS32 *regionList 657 ) 658 { 659 // regSize never decreases + registered states don't change, so lock-free 660 NvU32 i; 661 NvU32 flags = allocationOptions->flags; 662 NvU32 regionCount = 0; 663 NV_STATUS status = NV_OK; 664 665 NV_ASSERT(regionList != NULL); 666 NV_ASSERT(allocationOptions != NULL); 667 668 for (i=0; i < pPma->regSize; i++) 669 { 670 if (flags & PMA_ALLOCATE_SPECIFY_REGION_ID) 671 { 672 if (i != allocationOptions->regionId) 673 { 674 // Skip: wrong region ID 675 continue; 676 } 677 } 678 679 if (!!(flags & PMA_ALLOCATE_PROTECTED_REGION) ^ 680 (pPma->pRegDescriptors[i]->bProtected)) 681 { 682 // Don't allow unprotected allocations in protected region 683 // OR protected allocations in unprotected region. 684 continue; 685 } 686 687 if (flags & PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE) 688 { 689 NvU64 regionBegin, regionEnd; 690 NvU64 rangeBegin, rangeEnd; 691 PMA_REGION_DESCRIPTOR *regionDes; 692 693 rangeBegin = allocationOptions->physBegin; 694 rangeEnd = allocationOptions->physEnd; 695 696 regionDes = pPma->pRegDescriptors[i]; 697 regionBegin = regionDes->base; 698 regionEnd = regionDes->limit; 699 700 if ((rangeEnd < regionBegin) || (rangeBegin > regionEnd)) 701 { 702 // Skip: Requested range doesn't intersect region 703 continue; 704 } 705 } 706 707 if (flags & PMA_ALLOCATE_SPECIFY_MINIMUM_SPEED) 708 { 709 if (pPma->pRegDescriptors[i]->performance < allocationOptions->minimumSpeed) 710 { 711 // Skip: Region perf less than minimum threshold 712 continue; 713 } 714 } 715 716 if (regionCount > 0) 717 { 718 NvU32 j = regionCount; 719 720 if (flags & PMA_ALLOCATE_REVERSE_ALLOC) 721 { 722 // Find insertion point (highest memory address to lowest) 723 while ((j > 0) && 724 (pPma->pRegDescriptors[i]->limit > pPma->pRegDescriptors[regionList[j-1]]->limit)) 725 { 726 regionList[j] = regionList[j-1]; 727 j--; 728 } 729 } 730 else if (flags & PMA_ALLOCATE_PREFER_SLOWEST) 731 { 732 // Find insertion point (slowest to fastest) 733 while ((j > 0) && 734 (pPma->pRegDescriptors[i]->performance < pPma->pRegDescriptors[regionList[j-1]]->performance)) 735 { 736 regionList[j] = regionList[j-1]; 737 j--; 738 } 739 } 740 else 741 { 742 // Find insertion point (fastest to slowest) 743 while ((j > 0) && 744 (pPma->pRegDescriptors[i]->performance > pPma->pRegDescriptors[regionList[j-1]]->performance)) 745 { 746 regionList[j] = regionList[j-1]; 747 j--; 748 } 749 } 750 751 // Insert in sorted order 752 regionList[j] = i; 753 regionCount++; 754 } 755 else 756 { 757 regionList[regionCount++] = i; 758 } 759 } 760 761 // Invalidate the unused slots 762 for (i = regionCount; i < pPma->regSize; i++) 763 { 764 regionList[i] = -1; 765 } 766 767 if (regionCount == 0) 768 { 769 status = NV_ERR_NO_MEMORY; 770 } 771 772 return status; 773 } 774 775 /*! 776 * @brief This function will get a list of base+size and then goes in and 777 * clear the scrubbing bit on any pages in these ranges. It is only called 778 * when we know something is done scrubbing. 779 * 780 * @param[in] pPmaScrubList The list of ranges that need to be cleared 781 * @param[in] count Length of the list 782 * 783 * Note: 784 * - This call takes the PMA lock! Do not call this with PMA lock held. 785 */ 786 void 787 _pmaClearScrubBit 788 ( 789 PMA *pPma, 790 PSCRUB_NODE pPmaScrubList, 791 NvU64 count 792 ) 793 { 794 NvU32 i; 795 NvU64 base; 796 NvU64 size; 797 798 NV_ASSERT(count > 0); 799 portSyncSpinlockAcquire(pPma->pPmaLock); 800 801 for (i = 0; i < count; i++) 802 { 803 base = pPmaScrubList[i].base; 804 size = pPmaScrubList[i].size; 805 NV_ASSERT(size > 0); 806 pmaSetBlockStateAttribUnderPmaLock(pPma, base, size, 0, ATTRIB_SCRUBBING); 807 } 808 portSyncSpinlockRelease(pPma->pPmaLock); 809 } 810 811 /*! 812 * @brief This function will optionally wait for scrubbing to be finished for a 813 * list of pages, then check the scrubber status and clear the ATTRIB_SCRUBBING 814 * page attribute on any pages that have completed scrubbing 815 * 816 * @param[in] chunkSize The size of each page being waited on 817 * @param[in] pPages The list of pages being waited on 818 * @param[in] pageCount The number of pages we are waiting for 819 * If pageCount == 0, then we don't wait for any pages 820 * 821 * Locking: 822 * - This needs to be called without the PMA lock! 823 * - This call will take the PMA lock internally to modify page attributes. 824 */ 825 NV_STATUS 826 _pmaCheckScrubbedPages 827 ( 828 PMA *pPma, 829 NvU64 chunkSize, 830 NvU64 *pPages, 831 NvU32 pageCount 832 ) 833 { 834 PSCRUB_NODE pPmaScrubList = NULL; 835 NvU64 count = 0; 836 NV_STATUS status = NV_OK; 837 838 // If the caller wants to wait for something, we wait first before checking 839 if (pageCount != 0) 840 { 841 if ((status = scrubWaitPages(pPma->pScrubObj, chunkSize, pPages, pageCount)) != NV_OK) 842 return status; 843 } 844 845 status = scrubCheck(pPma->pScrubObj, &pPmaScrubList, &count); 846 NV_ASSERT_OR_GOTO((status == NV_OK), exit); 847 848 // This call takes the PMA lock! 849 if (count > 0) 850 _pmaClearScrubBit(pPma, pPmaScrubList, count); 851 852 exit: 853 // Free the actual list, although allocated by objscrub 854 portMemFree(pPmaScrubList); 855 856 return status; 857 } 858 859 860 NV_STATUS 861 _pmaPredictOutOfMemory 862 ( 863 PMA *pPma, 864 NvLength allocationCount, 865 NvU64 pageSize, 866 PMA_ALLOCATION_OPTIONS *allocationOptions 867 ) 868 { 869 NvU32 alignFlag, partialFlag; 870 NvU64 alignment; 871 NvU64 free2mbPages = 0; 872 NvU64 bytesFree = 0; 873 874 alignFlag = !!((allocationOptions->flags) & PMA_ALLOCATE_FORCE_ALIGNMENT); 875 partialFlag = !!((allocationOptions->flags) & PMA_ALLOCATE_ALLOW_PARTIAL); 876 alignment = allocationOptions->alignment; 877 878 if ((alignFlag && (alignment == _PMA_2MB)) || pageSize == _PMA_2MB) 879 { 880 if (allocationOptions->flags & PMA_ALLOCATE_PROTECTED_REGION) 881 { 882 free2mbPages = pPma->pmaStats.numFree2mbPagesProtected; 883 } 884 else 885 { 886 free2mbPages = pPma->pmaStats.numFree2mbPages - 887 pPma->pmaStats.numFree2mbPagesProtected; 888 } 889 890 // If we have at least one page free, don't fail a partial allocation 891 if (partialFlag && (free2mbPages > 0)) 892 { 893 return NV_OK; 894 } 895 896 if (free2mbPages < allocationCount) 897 { 898 return NV_ERR_NO_MEMORY; 899 } 900 } 901 902 // Do a quick check and exit early if we are in OOM case 903 if (allocationOptions->flags & PMA_ALLOCATE_PROTECTED_REGION) 904 { 905 bytesFree = pPma->pmaStats.numFreeFramesProtected << PMA_PAGE_SHIFT; 906 } 907 else 908 { 909 bytesFree = (pPma->pmaStats.numFreeFrames - 910 pPma->pmaStats.numFreeFramesProtected) << PMA_PAGE_SHIFT; 911 } 912 913 // If we have at least one page free, don't fail a partial allocation 914 if (partialFlag && (bytesFree >= pageSize)) 915 { 916 return NV_OK; 917 } 918 919 if (bytesFree < (pageSize * allocationCount)) 920 { 921 return NV_ERR_NO_MEMORY; 922 } 923 924 return NV_OK; 925 } 926 927 /*! 928 * @brief Internal function to intermittently free the blacklisted pages in the 929 * range of allocation request. This will enable PMA to allow OS to manage those 930 * blacklisted pages after being allocated. 931 * 932 * @param[in] pPma PMA Object 933 * @param[in] regId PMA Region ID , where the allocation falls into 934 * @param[in] rangeBegin Start address for the allocation range 935 * @param[in] rangeSize Size of the allocation region 936 * 937 * Locking: 938 * - This needs to be called with the PMA lock! 939 */ 940 941 void 942 _pmaFreeBlacklistPages 943 ( 944 PMA *pPma, 945 NvU32 regId, 946 NvU64 rangeBegin, 947 NvU64 rangeSize 948 ) 949 { 950 NvU32 blacklistCount = 0; 951 NvU32 chunk; 952 NvU64 alignedBlacklistAddr; 953 NvBool bClientManagedBlacklist = NV_FALSE; 954 PMA_BLACKLIST_CHUNK *pBlacklistChunks, *pBlacklistChunk; 955 956 pmaQueryBlacklistInfo(pPma, &blacklistCount, &bClientManagedBlacklist, &pBlacklistChunks); 957 958 if(blacklistCount == 0) 959 return; // return early, nothing to do. 960 961 for (chunk = 0; chunk < blacklistCount; chunk++) 962 { 963 pBlacklistChunk = &pBlacklistChunks[chunk]; 964 if (pBlacklistChunk->bIsValid && (pBlacklistChunk->physOffset >= rangeBegin && 965 pBlacklistChunk->physOffset <= (rangeBegin + rangeSize - 1))) 966 { 967 // 968 // Clear the blacklist attribute of the pages 969 // Since physOffset here is the absolute address, make sure we align it to 64k 970 // 971 alignedBlacklistAddr = NV_ALIGN_DOWN64(pBlacklistChunk->physOffset, PMA_GRANULARITY); 972 pmaSetBlockStateAttribUnderPmaLock(pPma, alignedBlacklistAddr, PMA_GRANULARITY, 0, ATTRIB_BLACKLIST); 973 pBlacklistChunk->bIsValid = NV_FALSE; 974 bClientManagedBlacklist = NV_TRUE; 975 } 976 } 977 978 pmaSetClientManagedBlacklist(pPma, bClientManagedBlacklist); 979 980 return; 981 } 982 983 /*! 984 * @brief Internal function to reallocate blacklisted pages in the 985 * range of allocation request.This is called, when the allocation requesting 986 * blacklisting OFF fails or when the allocation with blacklisting OFF gets free-d. 987 * 988 * @param[in] pPma PMA Object 989 * @param[in] regId PMA Region ID , where the allocation falls into 990 * @param[in] rangeBegin Start address for the allocation range 991 * @param[in] rangeSize Size of the allocation region 992 * 993 * Locking: 994 * - This needs to be called with the PMA lock! 995 */ 996 997 void _pmaReallocBlacklistPages 998 ( 999 PMA *pPma, 1000 NvU32 regId, 1001 NvU64 rangeBegin, 1002 NvU64 rangeSize 1003 ) 1004 { 1005 NvU32 blacklistCount = 0; 1006 NvU32 chunk; 1007 NvU64 alignedBlacklistAddr; 1008 NvU32 reallocatedBlacklistCount = 0; 1009 1010 NvBool bClientManagedBlacklist = NV_FALSE; 1011 PMA_BLACKLIST_CHUNK *pBlacklistChunks, *pBlacklistChunk; 1012 pmaQueryBlacklistInfo(pPma, &blacklistCount, &bClientManagedBlacklist, &pBlacklistChunks); 1013 1014 if (blacklistCount == 0 || !bClientManagedBlacklist) 1015 { 1016 return; 1017 } 1018 1019 // Assert if scrub on free is enabled for client managed blacklist 1020 NV_ASSERT(pPma->bScrubOnFree == NV_FALSE); 1021 1022 for (chunk = 0; chunk < blacklistCount; chunk++) 1023 { 1024 pBlacklistChunk = &pBlacklistChunks[chunk]; 1025 if (!pBlacklistChunk->bIsValid && 1026 (pBlacklistChunk->physOffset >= rangeBegin && 1027 pBlacklistChunk->physOffset <= (rangeBegin + rangeSize -1))) 1028 { 1029 // Since physOffset here is the absolute address, make sure we align it to 64k 1030 alignedBlacklistAddr = NV_ALIGN_DOWN64(pBlacklistChunk->physOffset, PMA_GRANULARITY); 1031 pmaSetBlockStateAttribUnderPmaLock(pPma, alignedBlacklistAddr, PMA_GRANULARITY, ATTRIB_BLACKLIST, ATTRIB_BLACKLIST); 1032 pBlacklistChunk->bIsValid = NV_TRUE; 1033 } 1034 reallocatedBlacklistCount = (pBlacklistChunk->bIsValid == NV_TRUE) ? reallocatedBlacklistCount+1: 1035 reallocatedBlacklistCount; 1036 } 1037 1038 // Reset the flag if client handed over the blacklisted pages in their region to RM. 1039 if (chunk == reallocatedBlacklistCount) 1040 { 1041 pmaSetClientManagedBlacklist(pPma, NV_FALSE); 1042 } 1043 } 1044 1045 /*! 1046 * @brief Internal function to lookup if the current frame is blacklisted already 1047 * If so, we will return NV_TRUE, otherwise NV_FALSE. 1048 * 1049 * @param[in] pPma PMA Object 1050 * @param[in] regId PMA Region ID , where the allocation falls into 1051 * @param[in] frameNum Frame Number which needs to be checked. 1052 * 1053 * Locking: 1054 * - This needs to be called with the PMA lock! 1055 */ 1056 NvBool 1057 _pmaLookupBlacklistFrame 1058 ( 1059 PMA *pPma, 1060 NvU32 regId, 1061 NvU64 frameNum 1062 ) 1063 { 1064 NvU32 blacklistCount; 1065 NvU64 addrBase; 1066 NvU32 chunk; 1067 NvU64 cliManagedBlackFrame = 0; 1068 1069 NvBool bClientManagedBlacklist = NV_FALSE; 1070 PMA_BLACKLIST_CHUNK *pBlacklistChunks, *pBlacklistChunk; 1071 1072 pmaQueryBlacklistInfo(pPma, &blacklistCount, &bClientManagedBlacklist, &pBlacklistChunks); 1073 1074 if (blacklistCount == 0 || !bClientManagedBlacklist) 1075 return NV_FALSE; 1076 1077 addrBase = pPma->pRegDescriptors[regId]->base; 1078 for (chunk = 0; chunk < blacklistCount; chunk++) 1079 { 1080 pBlacklistChunk = &pBlacklistChunks[chunk]; 1081 if (pBlacklistChunk->bIsValid) 1082 { 1083 // calculate the frame addr 1084 cliManagedBlackFrame = PMA_ADDR2FRAME(pBlacklistChunk->physOffset, addrBase); 1085 if (cliManagedBlackFrame == frameNum) 1086 { 1087 return NV_TRUE; 1088 } 1089 } 1090 } 1091 return NV_FALSE; 1092 } 1093 1094 /*! 1095 * @brief Returns a list of PMA-managed blocks with the specified state and 1096 * attributes. 1097 * 1098 * @param[in] pPma PMA pointer 1099 * @param[in/out] ppList Pointer to list of segments having specified 1100 * state and attributes 1101 * @param[in] pageStatus PMA page state and attribute 1102 * 1103 * @return 1104 * NV_OK Success 1105 * NV_ERR_NO_MEMORY Failure to allocate list 1106 */ 1107 NV_STATUS 1108 pmaBuildList 1109 ( 1110 PMA *pPma, 1111 PRANGELISTTYPE *ppList, 1112 PMA_PAGESTATUS pageStatus 1113 ) 1114 { 1115 NvU32 regionIdx, frameNum; 1116 NvU64 addrBase, addrLimit, numFrames; 1117 NvU64 blockStart = 0, blockEnd = 0; 1118 NvBool bBlockValid; 1119 PMA_PAGESTATUS pageState; 1120 PRANGELISTTYPE pRangeCurr, pRangeList = NULL; 1121 NV_STATUS status = NV_OK; 1122 void *pMap = NULL; 1123 1124 for (regionIdx = 0; regionIdx < pPma->regSize; regionIdx++) 1125 { 1126 pMap = pPma->pRegions[regionIdx]; 1127 addrBase = pPma->pRegDescriptors[regionIdx]->base; 1128 addrLimit = pPma->pRegDescriptors[regionIdx]->limit; 1129 numFrames = (addrLimit - addrBase + 1) >> PMA_PAGE_SHIFT; 1130 bBlockValid = NV_FALSE; 1131 1132 for (frameNum = 0; frameNum < numFrames; frameNum++) 1133 { 1134 pageState = pPma->pMapInfo->pmaMapRead(pMap, frameNum, NV_TRUE); 1135 if (pageState & pageStatus) 1136 { 1137 if (bBlockValid) 1138 { 1139 // Block start already found. Find the end 1140 blockEnd = frameNum; 1141 } 1142 else 1143 { 1144 // Block start found. Now find the end 1145 blockStart = frameNum; 1146 blockEnd = frameNum; 1147 bBlockValid = NV_TRUE; 1148 } 1149 } 1150 else if (bBlockValid) 1151 { 1152 // Block found having required PMA page state. Store it in the list 1153 pRangeCurr = (PRANGELISTTYPE) portMemAllocNonPaged(sizeof(RANGELISTTYPE)); 1154 if (pRangeCurr) 1155 { 1156 pRangeCurr->base = addrBase + blockStart * PMA_GRANULARITY; 1157 pRangeCurr->limit = addrBase + blockEnd * PMA_GRANULARITY + PMA_GRANULARITY - 1; 1158 pRangeCurr->pNext = pRangeList; 1159 pRangeList = pRangeCurr; 1160 } 1161 else 1162 { 1163 // Allocation failed 1164 pmaFreeList(pPma, &pRangeList); 1165 pRangeList = NULL; 1166 status = NV_ERR_NO_MEMORY; 1167 break; 1168 } 1169 1170 bBlockValid = NV_FALSE; 1171 } 1172 } 1173 1174 // No point checking further if we are already out of memory 1175 if (status == NV_ERR_NO_MEMORY) 1176 break; 1177 1178 // Check if last frame was part of a block. 1179 if (bBlockValid) 1180 { 1181 // Block found having required PMA page state. Store it in the list 1182 pRangeCurr = (PRANGELISTTYPE) portMemAllocNonPaged(sizeof(RANGELISTTYPE)); 1183 if (pRangeCurr) 1184 { 1185 pRangeCurr->base = addrBase + blockStart * PMA_GRANULARITY; 1186 pRangeCurr->limit = addrBase + blockEnd * PMA_GRANULARITY + PMA_GRANULARITY - 1; 1187 pRangeCurr->pNext = pRangeList; 1188 pRangeList = pRangeCurr; 1189 } 1190 else 1191 { 1192 // Allocation failed 1193 pmaFreeList(pPma, &pRangeList); 1194 pRangeList = NULL; 1195 status = NV_ERR_NO_MEMORY; 1196 break; 1197 } 1198 } 1199 } 1200 1201 *ppList = pRangeList; 1202 1203 return status; 1204 } 1205 1206 /*! 1207 * @brief Frees previously generated list of PMA-managed blocks with 1208 * function pmaBuildList() 1209 * 1210 * @param[in] pPma PMA pointer 1211 * @param[in/out] ppList Pointer to list of PMA segments 1212 * 1213 * @return 1214 * None 1215 */ 1216 void 1217 pmaFreeList 1218 ( 1219 PMA *pPma, 1220 PRANGELISTTYPE *ppList 1221 ) 1222 { 1223 PRANGELISTTYPE pRangeCurr = *ppList; 1224 PRANGELISTTYPE pRangeNext; 1225 1226 while (pRangeCurr) 1227 { 1228 pRangeNext = pRangeCurr->pNext;; 1229 portMemFree(pRangeCurr); 1230 pRangeCurr = pRangeNext; 1231 } 1232 1233 *ppList = NULL; 1234 } 1235 1236 NV_STATUS 1237 pmaRegisterBlacklistInfo 1238 ( 1239 PMA *pPma, 1240 NvU64 physAddrBase, 1241 PMA_BLACKLIST_ADDRESS *pBlacklistPageBase, 1242 NvU32 blacklistCount, 1243 NvBool bBlacklistFromInforom 1244 ) 1245 { 1246 NvU32 i; 1247 NvU64 alignedBlacklistAddr; 1248 PMA_BLACKLIST_CHUNK *pBlacklistChunk = NULL; 1249 NvU32 nextBlacklistEntry = 0; 1250 NvU32 blacklistEntryIn = 0; 1251 1252 if (blacklistCount == 0 || pBlacklistPageBase == NULL) 1253 { 1254 return NV_OK; 1255 } 1256 1257 if (pPma->pBlacklistChunks == NULL) 1258 { 1259 pPma->pBlacklistChunks = (PMA_BLACKLIST_CHUNK *) 1260 portMemAllocNonPaged( PMA_MAX_BLACKLIST_ENTRIES * sizeof(PMA_BLACKLIST_CHUNK)); 1261 if (pPma->pBlacklistChunks == NULL) 1262 { 1263 pPma->blacklistCount = 0; 1264 NV_PRINTF(LEVEL_ERROR, "ERROR: Insufficient memory to allocate blacklisting tracking structure.\n"); 1265 return NV_ERR_NO_MEMORY; 1266 } 1267 portMemSet(pPma->pBlacklistChunks, 0, PMA_MAX_BLACKLIST_ENTRIES * sizeof(PMA_BLACKLIST_CHUNK)); 1268 } 1269 1270 nextBlacklistEntry = pPma->blacklistCount; 1271 1272 for (i = nextBlacklistEntry; i < (nextBlacklistEntry + blacklistCount); i++) 1273 { 1274 pBlacklistChunk = &pPma->pBlacklistChunks[i]; 1275 pBlacklistChunk->physOffset = pBlacklistPageBase[blacklistEntryIn].physOffset; 1276 pBlacklistChunk->bIsDynamic = pBlacklistPageBase[blacklistEntryIn].bIsDynamic; 1277 1278 // Since physOffset here is the absolute address, make sure we align it to 64K 1279 alignedBlacklistAddr = NV_ALIGN_DOWN64(pBlacklistPageBase[blacklistEntryIn].physOffset, PMA_GRANULARITY); 1280 pmaSetBlockStateAttrib(pPma, alignedBlacklistAddr, PMA_GRANULARITY, ATTRIB_BLACKLIST, ATTRIB_BLACKLIST); 1281 pBlacklistChunk->bIsValid = NV_TRUE; 1282 1283 // 1284 // In NUMA systems, memory allocation comes directly from kernel, which 1285 // won't check for ATTRIB_BLACKLIST. So pages need to be blacklisted 1286 // directly through the kernel. 1287 // 1288 // This is only needed for NUMA systems that auto online NUMA memory. 1289 // Other systems (e.g., P9) already do blacklisting in nvidia-persistenced. 1290 // 1291 if (pPma->bNuma && pPma->bNumaAutoOnline) 1292 { 1293 // 1294 // Only blacklist pages from inforom (i.e., during heap/PMA init) need 1295 // to be blacklisted with kernel here. The blacklist pages stored in 1296 // inforom need to remain blacklisted persistently across GPU resets - 1297 // kernel won't automatically blacklist these so RM must do it 1298 // explicitly here. 1299 // 1300 // Blacklist pages not from inforom (i.e., from ECC interrupt handling) 1301 // do not need to be blacklisted with kernel. This is because the ECC 1302 // interrupt will automatically trigger kernel itself to blacklist the page. 1303 // 1304 if (bBlacklistFromInforom) 1305 { 1306 NV_STATUS status; 1307 1308 // Use physOffset without 64K alignment, because kernel may use a different page size. 1309 NV_PRINTF(LEVEL_INFO, 1310 "NUMA enabled - blacklisting page through kernel at address 0x%llx (GPA) 0x%llx (SPA)\n", 1311 pBlacklistPageBase[blacklistEntryIn].physOffset, 1312 pBlacklistPageBase[blacklistEntryIn].physOffset + pPma->coherentCpuFbBase); 1313 1314 status = osOfflinePageAtAddress(pBlacklistPageBase[blacklistEntryIn].physOffset + pPma->coherentCpuFbBase); 1315 if (status != NV_OK) 1316 { 1317 NV_PRINTF(LEVEL_ERROR, "osOfflinePageAtAddress() failed with status: %d\n", status); 1318 } 1319 } 1320 } 1321 1322 blacklistEntryIn++; 1323 } 1324 1325 pPma->blacklistCount += blacklistCount; 1326 1327 return NV_OK; 1328 } 1329 1330 void 1331 pmaSetClientManagedBlacklist 1332 ( 1333 PMA *pPma, 1334 NvBool bClientManagedBlacklist 1335 ) 1336 { 1337 pPma->bClientManagedBlacklist = bClientManagedBlacklist; 1338 } 1339 1340 void 1341 pmaQueryBlacklistInfo 1342 ( 1343 PMA *pPma, 1344 NvU32 *pBlacklistCount, 1345 NvBool *pbClientManagedBlacklist, 1346 PMA_BLACKLIST_CHUNK **ppBlacklistChunks 1347 ) 1348 { 1349 if (pBlacklistCount) 1350 { 1351 *pBlacklistCount = pPma->blacklistCount; 1352 } 1353 1354 if (pbClientManagedBlacklist) 1355 { 1356 *pbClientManagedBlacklist = pPma->bClientManagedBlacklist; 1357 } 1358 1359 if (ppBlacklistChunks) 1360 { 1361 *ppBlacklistChunks = pPma->pBlacklistChunks; 1362 } 1363 } 1364 1365 NvBool 1366 pmaIsBlacklistingAddrUnique 1367 ( 1368 PMA *pPma, 1369 NvU64 physAddr 1370 ) 1371 { 1372 NvU32 count = 0; 1373 PMA_BLACKLIST_CHUNK *pBlacklistChunk = NULL; 1374 if (pPma->blacklistCount == 0) 1375 { 1376 return NV_TRUE; 1377 } 1378 for (count = 0; count < pPma->blacklistCount; count++) 1379 { 1380 pBlacklistChunk = &pPma->pBlacklistChunks[count]; 1381 if (pBlacklistChunk->physOffset == physAddr) 1382 { 1383 return NV_FALSE; 1384 } 1385 } 1386 return NV_TRUE; 1387 } 1388 1389