1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @brief This file exposes the PMA interfaces. 26 * The PMA module interacts with the RM and UVM components. UVM will use the 27 * ops layer to call out to PMA, while RM directly calls into PMA. 28 * The PMA module takes a global lock to protect its internal structure. It 29 * uses a bitmap structure called regmap. 30 * 31 * @bug 32 * 1. status code -- decide if we need to add to global 33 * 2. suspend/resume -- might add one more function to support 34 * 35 * @TODO 36 * 1. external fragmentation 37 * 2. use new scrubber API and remove the initScrubbing atomic variable 38 */ 39 40 #ifndef PHYS_MEM_ALLOCATOR_H 41 #define PHYS_MEM_ALLOCATOR_H 42 43 #include "nvport/nvport.h" 44 #include "regmap.h" 45 #include "nvmisc.h" 46 47 #if defined(SRT_BUILD) 48 #define RMCFG_MODULE_x 1 49 #else 50 #include "rmconfig.h" 51 #endif 52 53 #ifdef __cplusplus 54 extern "C" { 55 #endif 56 57 typedef struct OBJMEMSCRUB OBJMEMSCRUB; 58 typedef struct SCRUB_NODE SCRUB_NODE; 59 60 #define PMA_REGION_SIZE 64 61 #define PMA_ADDR2FRAME(addr, base) (((addr) - (base)) >> PMA_PAGE_SHIFT) 62 #define PMA_FRAME2ADDR(frame, base) ((base) + ((frame) << PMA_PAGE_SHIFT)) 63 64 // 65 // These flags are used for initialization in order to set global PMA states, 66 // in case we need to wait for scrubber to be initialized or wait for a NUMA 67 // node being onlined, etc. 68 // 69 #define PMA_INIT_NONE NVBIT(0) 70 #define PMA_INIT_SCRUB_ON_FREE NVBIT(1) 71 #define PMA_INIT_NUMA NVBIT(2) 72 #define PMA_INIT_INTERNAL NVBIT(3) // Used after heap is removed 73 #define PMA_INIT_FORCE_PERSISTENCE NVBIT(4) 74 // unused 75 #define PMA_INIT_NUMA_AUTO_ONLINE NVBIT(6) 76 77 // These flags are used for querying PMA's config and/or state. 78 #define PMA_QUERY_SCRUB_ENABLED NVBIT(0) 79 #define PMA_QUERY_SCRUB_VALID NVBIT(1) 80 #define PMA_QUERY_NUMA_ENABLED NVBIT(2) 81 #define PMA_QUERY_NUMA_ONLINED NVBIT(3) 82 83 // 84 // When modifying flags, make sure they are compatible with the mirrored 85 // UVM_PMA_* flags in nv_uvm_types.h. 86 // 87 // Input flags 88 #define PMA_ALLOCATE_DONT_EVICT NVBIT(0) 89 #define PMA_ALLOCATE_PINNED NVBIT(1) 90 #define PMA_ALLOCATE_SPECIFY_MINIMUM_SPEED NVBIT(2) 91 #define PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE NVBIT(3) 92 #define PMA_ALLOCATE_SPECIFY_REGION_ID NVBIT(4) 93 #define PMA_ALLOCATE_PREFER_SLOWEST NVBIT(5) 94 #define PMA_ALLOCATE_CONTIGUOUS NVBIT(6) 95 #define PMA_ALLOCATE_PERSISTENT NVBIT(7) 96 #define PMA_ALLOCATE_PROTECTED_REGION NVBIT(8) 97 #define PMA_ALLOCATE_FORCE_ALIGNMENT NVBIT(9) 98 #define PMA_ALLOCATE_NO_ZERO NVBIT(10) 99 #define PMA_ALLOCATE_TURN_BLACKLIST_OFF NVBIT(11) 100 #define PMA_ALLOCATE_ALLOW_PARTIAL NVBIT(12) 101 #define PMA_ALLOCATE_REVERSE_ALLOC NVBIT(13) 102 103 // Output flags 104 #define PMA_ALLOCATE_RESULT_IS_ZERO NVBIT(0) 105 106 // These are flags input to the pmaFreePages call 107 #define PMA_FREE_SKIP_SCRUB NVBIT(0) 108 109 // State bits for debugging utilities like nvwatch 110 #define PMA_SCRUB_INITIALIZE 0 111 #define PMA_SCRUB_IN_PROGRESS 1 112 #define PMA_SCRUB_DONE 2 113 114 #define PMA_SCRUBBER_VALID 1 115 #define PMA_SCRUBBER_INVALID 0 116 117 #define PMA_NUMA_NO_NODE -1 118 119 // Maximum blacklist entries possible 120 #define PMA_MAX_BLACKLIST_ENTRIES 512 121 122 typedef struct 123 { 124 NvU32 flags; 125 NvU32 minimumSpeed; // valid if flags & PMA_ALLOCATE_SPECIFY_MININUM_SPEED 126 NvU64 physBegin, physEnd; // valid if flags & PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE 127 NvU32 regionId; // valid if flags & PMA_ALLOCATE_SPECIFY_REGION_ID 128 NvU64 alignment; // valid if flags & PMA_ALLOCATE_FORCE_ALIGNMENT 129 NvLength numPagesAllocated; // valid if flags & PMA_ALLOCATE_ALLOW_PARTIAL 130 131 NvU32 resultFlags; // valid if the allocation function returns NV_OK 132 } PMA_ALLOCATION_OPTIONS; 133 134 // 135 // Explanation: This struct will be provided when UVM/RM registers a region with PMA, 136 // after which the struct is stored locally in PMA. The internal "filter" function will 137 // use the information everytime a request comes in. 138 // 139 typedef struct 140 { 141 NvU64 base; // Base/start address of the region 142 NvU64 limit; // Last/end address of region 143 NvU32 performance; // Relative performance. Higher is faster 144 NvBool bSupportCompressed; // Support compressed kinds 145 NvBool bSupportISO; // Support ISO (display, cursor, video) surfaces 146 NvBool bProtected; // Represents a protected region of memory. 147 } PMA_REGION_DESCRIPTOR; 148 149 typedef struct _PMA_MAP_INFO PMA_MAP_INFO; 150 typedef struct _PMA PMA; 151 152 // Range descriptors for managing persistent range lists 153 typedef struct _RANGELISTTYPE 154 { 155 NvU64 base; 156 NvU64 limit; 157 struct _RANGELISTTYPE *pNext; 158 } RANGELISTTYPE, *PRANGELISTTYPE; 159 160 typedef enum 161 { 162 MEMORY_PROTECTION_UNPROTECTED = 0, 163 MEMORY_PROTECTION_PROTECTED = 1 164 } MEMORY_PROTECTION; 165 166 /*! 167 * @brief Callback to update stats in RUSD 168 */ 169 typedef void (*pmaUpdateStatsCb_t)(void *pCtx, NvU64 freeFrames); 170 171 /*! 172 * @brief Callbacks to UVM for eviction 173 */ 174 typedef NV_STATUS (*pmaEvictPagesCb_t)(void *ctxPtr, NvU64 pageSize, NvU64 *pPages, 175 NvU32 count, NvU64 physBegin, NvU64 physEnd, 176 MEMORY_PROTECTION prot); 177 typedef NV_STATUS (*pmaEvictRangeCb_t)(void *ctxPtr, NvU64 physBegin, NvU64 physEnd, 178 MEMORY_PROTECTION prot); 179 180 /*! 181 * @brief Pluggable data structure management. Currently we have regmap and address tree. 182 */ 183 typedef void *(*pmaMapInit_t)(NvU64 numFrames, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected); 184 typedef void (*pmaMapDestroy_t)(void *pMap); 185 typedef void (*pmaMapChangeStateAttrib_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask); 186 typedef void (*pmaMapChangePageStateAttrib_t)(void *pMap, NvU64 startFrame, NvU64 pageSize, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask); 187 typedef void (*pmaMapChangeBlockStateAttrib_t)(void *pMap, NvU64 frameNum, NvU64 numFrames, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask); 188 typedef PMA_PAGESTATUS (*pmaMapRead_t)(void *pMap, NvU64 frameNum, NvBool readAttrib); 189 typedef NV_STATUS (*pmaMapScanContiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd, 190 NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment, 191 NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc); 192 typedef NV_STATUS (*pmaMapScanDiscontiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd, 193 NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment, 194 NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc); 195 typedef void (*pmaMapGetSize_t)(void *pMap, NvU64 *pBytesTotal); 196 typedef void (*pmaMapGetLargestFree_t)(void *pMap, NvU64 *pLargestFree); 197 typedef NV_STATUS (*pmaMapScanContiguousNumaEviction_t)(void *pMap, NvU64 addrBase, NvLength actualSize, 198 NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd); 199 typedef NvU64 (*pmaMapGetEvictingFrames_t)(void *pMap); 200 typedef void (*pmaMapSetEvictingFrames_t)(void *pMap, NvU64 frameEvictionsInProcess); 201 202 struct _PMA_MAP_INFO 203 { 204 NvU32 mode; 205 pmaMapInit_t pmaMapInit; 206 pmaMapDestroy_t pmaMapDestroy; 207 pmaMapChangeStateAttrib_t pmaMapChangeStateAttrib; 208 pmaMapChangePageStateAttrib_t pmaMapChangePageStateAttrib; 209 pmaMapChangeBlockStateAttrib_t pmaMapChangeBlockStateAttrib; 210 pmaMapRead_t pmaMapRead; 211 pmaMapScanContiguous_t pmaMapScanContiguous; 212 pmaMapScanDiscontiguous_t pmaMapScanDiscontiguous; 213 pmaMapGetSize_t pmaMapGetSize; 214 pmaMapGetLargestFree_t pmaMapGetLargestFree; 215 pmaMapScanContiguousNumaEviction_t pmaMapScanContiguousNumaEviction; 216 pmaMapGetEvictingFrames_t pmaMapGetEvictingFrames; 217 pmaMapSetEvictingFrames_t pmaMapSetEvictingFrames; 218 }; 219 220 struct _PMA 221 { 222 PORT_SPINLOCK *pPmaLock; // PMA-wide lock 223 PORT_MUTEX *pEvictionCallbacksLock; // Eviction callback registration lock 224 225 // Only used when free scrub-on-free feature is turned on 226 PORT_RWLOCK *pScrubberValidLock; // A reader-writer lock to protect the scrubber valid bit 227 PORT_MUTEX *pAllocLock; // Used to protect page stealing in the allocation path 228 229 // Region related states 230 NvU32 regSize; // Actual size of regions array 231 void * pRegions[PMA_REGION_SIZE]; // All the region maps stored as opaque pointers 232 NvU32 *pSortedFastFirst; // Pre-sorted array of region IDs 233 PMA_REGION_DESCRIPTOR *pRegDescriptors [PMA_REGION_SIZE]; // Stores the descriptions of each region 234 PMA_MAP_INFO *pMapInfo; // The pluggable layer for managing scanning 235 236 // Allocation related states 237 void * evictCtxPtr; // Opaque context pointer for eviction callback 238 pmaEvictPagesCb_t evictPagesCb; // Discontiguous eviction callback 239 pmaEvictRangeCb_t evictRangeCb; // Contiguous eviction callback 240 NvU64 frameAllocDemand; // Frame count of allocations in-process 241 NvBool bForcePersistence; // Force all allocations to persist across suspend/resume 242 PMA_STATS pmaStats; // PMA statistics used for client heuristics 243 244 // Scrubber related states 245 NvSPtr initScrubbing; // If the init scrubber has finished in this PMA 246 NvBool bScrubOnFree; // If "scrub on free" is enabled for this PMA object 247 NvSPtr scrubberValid; // If scrubber object is valid, using atomic variable to prevent races 248 OBJMEMSCRUB *pScrubObj; // Object to store the FreeScrub header 249 250 // NUMA states 251 NvBool bNuma; // If we are allocating for a NUMA system 252 NvBool nodeOnlined; // If node is onlined 253 NvS32 numaNodeId; // Current Node ID, set at initialization. -1 means invalid 254 NvU64 coherentCpuFbBase; // Used to calculate FB offset from bus address 255 NvU64 coherentCpuFbSize; // Used for error checking only 256 NvU32 numaReclaimSkipThreshold; // percent value below which __GFP_RECLAIM will not be used. 257 NvBool bNumaAutoOnline; // If NUMA memory is auto-onlined 258 259 // Blacklist related states 260 PMA_BLACKLIST_CHUNK *pBlacklistChunks; // Tracking for blacklist pages 261 NvU32 blacklistCount; // Number of blacklist pages 262 NvBool bClientManagedBlacklist; // Blacklisted pages in PMA that will be taken over by Client 263 264 // RUSD Callback 265 pmaUpdateStatsCb_t pStatsUpdateCb; // RUSD update free pages 266 void *pStatsUpdateCtx; // Context for RUSD update 267 }; 268 269 /*! 270 * @brief This must be called before any other PMA functions. Returns a PMA 271 * object for later use 272 * 273 * @param[in] gpuId The UVM global GPU ID. Defined in nvCpuUuid.h as a 16 274 * byte digest. PMA will only store reference to it. 275 * @param[in] pma Pointer to PMA object being initialized 276 * @param[in] initFlags PMA initialization flags for special modes 277 * @return 278 * NV_ERR_INVALID_ARGUMENT: 279 * If the combination of initFlags is invalid. 280 * NV_ERR_NO_MEMORY: 281 * Internal memory allocation failed. 282 * NV_ERR_GENERIC: 283 * Unexpected error. We try hard to avoid returning this error 284 * code, because it is not very informative. 285 * 286 */ 287 NV_STATUS pmaInitialize(PMA *pPma, NvU32 initFlags); 288 289 290 /*! 291 * @brief Release a PMA object. Also frees memory. 292 * 293 * All eviction handlers must have been unregistered by this point. 294 * 295 * @param[in] pma Pointer to PMA object being destroyed. 296 */ 297 void pmaDestroy(PMA *pPma); 298 299 300 /*! 301 * @brief Queries PMA configuration and state. 302 * 303 * Any clients of PMA can query config and state with a valid PMA object. 304 * Querying at different times may return different values when states change. 305 * 306 * @param[in] pma Pointer to PMA object being destroyed. 307 * @param[in/out] pConfigs Configs/states to query. See PMA_QUERY_* above. 308 */ 309 NV_STATUS pmaQueryConfigs(PMA* pPma, NvU32 *pConfigs); 310 311 312 /*! 313 * @brief Attaches a region of physical memory to be managed by the PMA. 314 * 315 * Systems with floorswept memory configurations will have (multi gigabyte) 316 * holes in memory. Each contiguous region should be reported by this 317 * registration function. This is also intended to be used by systems with 318 * two speed memories. 319 * 320 * Note: Some 0FB configurations may choose to skip registering any regions. 321 * At most 64 regions may be registered with a single PMA. 322 * 323 * @param[in] id 324 * A unique value in the range [0, 64) that uniquely identifies this 325 * region. Passed in by RM in region-order. Should be continuous for 326 * best performance. 327 * 328 * @param[in] bAsyncEccScrub 329 * RM will set this when it is performing the initial ECC scrub 330 * asynchronously. All pages in the pma will be marked 'allocated pinned'. 331 * RM will call pmaFreeContiguous as memory is scrubbed. 332 * 333 * Until the scrub is complete (by calling pmaScrubComplete), no 334 * allocation calls will fail with out of memory. Instead, it will hang 335 * until scrubbing is complete. One exception is when the client passes 336 * in the PMA_DONT_EVICT flag, in which case the call will actually fail 337 * regardless of whether RM is scrubbing memory. 338 * 339 * CAUTION! RM is responsible for ensuring black-listed pages are not 340 * marked free during the scrub. 341 * 342 * @param[in] regionState: 343 * Contains physical information about the region. 344 * 345 * @param[in] pBlacklistPageBase: 346 * List of base addresses of bad GPU pages. 347 * Each address is assumed to reference to a page of size 348 * PMA_GRANULARITY (64kb). 349 * 350 * Implementors note: PMA will simply mark these pages as "allocatedPinned". 351 * This list should be saved so that during pmaDestroy we can verify that only 352 * these pages are still allocated. 353 * 354 * @param[in] blacklistCount: 355 * Number of pages in above list. 356 * 357 * @return 358 * NV_ERR_NO_MEMORY: 359 * Internal memory allocation failed. 360 * NV_ERR_GENERIC: 361 * Unexpected error. We try hard to avoid returning this error code, 362 * because it is not very informative. 363 * 364 */ 365 NV_STATUS pmaRegisterRegion(PMA *pPma, NvU32 id, NvBool bAsyncEccScrub, 366 PMA_REGION_DESCRIPTOR *pRegionDesc, NvU32 blacklistCount, PPMA_BLACKLIST_ADDRESS pBlacklistPage); 367 368 369 /*! 370 * @brief Synchronous API for allocating pages from the PMA. 371 * PMA will decide which pma regions to allocate from based on the provided 372 * flags. PMA will also initiate UVM evictions to make room for this 373 * allocation unless prohibited by PMA_FLAGS_DONT_EVICT. UVM callers must pass 374 * this flag to avoid deadlock. Only UVM may allocate unpinned memory from this 375 * API and note that eviction callbacks need to be registered before that 376 * happens. 377 * 378 * Alignment of the allocated pages is guaranteed to be greater or equal to the 379 * requested page size. For contiguous allocations, a greater alignment can be 380 * specified with the PMA_ALLOCATE_FORCE_ALIGNMENT flag and the alignment 381 * allocation option. For non-contiguous allocations, it's an error to specify 382 * an alignment larger than the page size. 383 * 384 * For broadcast methods, PMA will guarantee the same physical frames are 385 * allocated on multiple GPUs, specified by the PMA objects passed in. 386 * 387 * Implementors note: 388 * If region registered with asyncEccScrub and pmaScrubComplete 389 * has not yet been issued then we cannot return NV_ERR_NO_MEMORY. 390 * We must instead drop the lock and wait until the next call to 391 * either pmaScrubComplete or pmaFreeContiguous/Pages to retry. 392 * Exception: PMA_ALLOCATE_DONT_EVICT 393 * 394 * @param[in] pPma 395 * The input PMA object 396 * 397 * @param[in] pageCount 398 * Number of pages to allocate. 399 * 400 * @param[in] pageSize 401 * 64kb, 128kb or 2mb. No other values are permissible. 402 * 403 * @param[in/out] allocationOptions 404 * Input flags: 405 * PMA_ALLOCATE_DONT_EVICT 406 * Do not evict in order to service this allocation. 407 * Do not wait for ECC scrub completion if out of memory. 408 * PMA_ALLOCATE_PINNED 409 * The allocation is pinned (RM must pass this) 410 * PMA_ALLOCATE_SPECIFY_MININUM_SPEED 411 * Only examines regions whose speed is greater than 412 * minimumSpeed. 413 * PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE 414 * Restrict the allowable physical address range for 415 * allocation to [physBegin, physEnd). 416 * PMA_ALLOCATE_SPECIFY_REGION_ID 417 * Only service allocations out of 'regionId'. The 418 * regionId is assigned in pmaRegisterRegion. 419 * PMA_ALLOCATE_PREFER_SLOWEST 420 * Prefer slower memory over faster. 421 * PMA_ALLOCATE_CONTIGUOUS 422 * If this allocation is a contiguous allocation 423 * PMA_ALLOCATE_PERSISTENT 424 * If this allocation should persist across suspend/resume 425 * PMA_ALLOCATE_FORCE_ALIGNMENT 426 * Force a specific alignment of the allocation. For non-contiguous 427 * allocations has to be less or equal to the page size. 428 * 429 * Output flags: 430 * PMA_ALLOCATE_RESULT_IS_ZERO 431 * If the allocated pages have been scrubbed. 432 * 433 * @param[out] pPages 434 * Return array of base addresses of allocated pages. 435 * 436 * @return 437 * NV_ERR_NO_MEMORY: 438 * Internal memory allocation failed. 439 * NV_ERR_GENERIC: 440 * Unexpected error. We try hard to avoid returning this error 441 * code,because it is not very informative. 442 * 443 */ 444 NV_STATUS pmaAllocatePages(PMA *pPma, NvLength pageCount, NvU64 pageSize, 445 PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages); 446 447 // allocate on multiple GPU, thus pmaCount 448 NV_STATUS pmaAllocatePagesBroadcast(PMA **pPma, NvU32 pmaCount, NvLength allocationCount, 449 NvU64 pageSize, PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages); 450 451 452 /*! 453 * @brief Marks previously unpinned pages as pinned. 454 * 455 * It will return an error and rollback any change if any page is not 456 * previously marked "unpinned". 457 * 458 * @param[in] pPages 459 * Array of base addresses of pages to pin 460 * 461 * @param[in] pageCount 462 * Number of pages to pin 463 * 464 * @param[in] pageSize 465 * Page size of each page being pinned 466 * 467 * @return 468 * NV_OK: 469 * The pages have been pinned successfully. 470 * 471 * NV_ERR_IN_USE: 472 * Some of the pages requested to be pinned are being evicted and thus 473 * cannot be pinned. None of the pages have been pinned and the caller 474 * needs to make sure the pages can be successfully evicted. 475 * 476 * NV_ERR_INVALID_STATE: 477 * Some of the pages requested to be pinned weren't in the allocated 478 * unpinned state. 479 * 480 * TODO some error for rollback 481 * 482 */ 483 NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize); 484 485 /*! 486 * @brief Marks a list of pages as free. 487 * This operation is also used by RM to mark pages as "scrubbed" for the 488 * initial ECC sweep. This function does not fail. 489 * 490 * @param[in] pPages 491 * Array of base addresses of pages to free 492 * 493 * @param[in] pageCount 494 * Number of pages to free 495 * If the value is 1, this is a contiguous free 496 * 497 * @param[in] size 498 * When freeing contiguous memory, this is the total size; 499 * When freeing discontiguous memory, this is page size of each page. 500 * 501 * @param[in] flag 502 * PMA_FREE_SKIP_SCRUB 503 * This flag is used to disable scrub on free when PMA frees the page 504 * 505 * @return Void 506 * 507 */ 508 void pmaFreePages(PMA *pPma, NvU64 *pPages, NvU64 pageCount, NvU64 size, NvU32 flag); 509 510 /*! 511 * @brief Clears scrubbing bit on PMA pages within the supplied range. 512 * 513 * @param[in] pma 514 * PMA object 515 * 516 * @param[in] rangeBase 517 * The start address 518 * 519 * @param[in] rangeLimit 520 * The end address 521 * 522 * @return 523 * void 524 */ 525 void pmaClearScrubRange(PMA *pPma, NvU64 rangeBase, NvU64 rangeLimit); 526 527 528 /*! 529 * @brief Notifies the PMA that the ECC scrub is complete. 530 * 531 * Until this function is called no PMA allocations will fail with 532 * "insufficient memory". They will hang and wait for the scrubbing. 533 * 534 * TODO consider suspend/resume behavior! 535 * 536 * Design Note: 537 * pmaRegisterRegion(w/ asyncEccScrub) leaves all pages as 538 * "allocated and pinned". 539 * 540 * As the ECC scrub progresses, RM will call PMA and "pmaFreeContiguous" 541 * the regions as they are scrubbed. 542 * 543 * @param[in] pma 544 * PMA object 545 * 546 * @return 547 * NV_ERR_GENERIC: 548 * Unexpected error. We try hard to avoid returning this error 549 * code,because it is not very informative. 550 * 551 */ 552 NV_STATUS pmaScrubComplete(PMA *pPma); 553 554 555 /*! 556 * Register the eviction callbacks. 557 * 558 * Only one set of callbacks can be registered at a time and they need to be 559 * unregistered with pmaUnregisterEvictionCb() before a new set can be 560 * registered. 561 * 562 * Note that eviction callbacks need to be registered before allocating any unpinned memory. 563 * 564 * See the documentation of the callbacks above for details of the eviction. 565 * 566 * @param[in] pma 567 * PMA object 568 * 569 * @param[in] evictPageCb 570 * The callback function for evicting pages at a time 571 * 572 * @param[in] evictRangeCb 573 * The callback function for evicting a range 574 * 575 * @param[in] ctxPtr 576 * The callback context pointer to be passed back on callback 577 * 578 * @return 579 * NV_ERR_INVALID_ARGUMENT: 580 * One of the callbacks or PMA object was NULL. 581 * 582 * NV_ERR_INVALID_STATE: 583 * Callbacks already registered. 584 */ 585 NV_STATUS pmaRegisterEvictionCb(PMA *pPma, pmaEvictPagesCb_t evictPagesCb, pmaEvictRangeCb_t evictRangeCb, void *ctxPtr); 586 587 /*! 588 * Register the stats update callback. 589 * 590 * Register callback to call when number of free pages changes. Currently only used for RUSD. 591 * 592 * @param[in] pma 593 * PMA object 594 * 595 * @param[in] pUpdateCb 596 * The callback to call when updating free page count 597 * 598 * @param[in] ctxPtr 599 * The callback context pointer to be passed back on callback 600 */ 601 void pmaRegisterUpdateStatsCb(PMA *pPma, pmaUpdateStatsCb_t pUpdateCb, void *ctxPtr); 602 603 604 /*! 605 * Unregister the eviction callbacks. 606 * 607 * Guarantees that all pending eviction callbacks complete before returning. 608 * 609 * All unpinned allocations must be freed before the callbacks are unregistered 610 * and the caller needs to guarantee that any pending eviction callbacks won't 611 * block on the thread unregistering the callbacks. 612 * 613 * The call does nothing if the PMA object is NULL. 614 * 615 * @param[in] pma 616 * PMA object. 617 */ 618 void pmaUnregisterEvictionCb(PMA *pPma); 619 620 /*! 621 * @brief Returns information about the total FB memory. 622 * 623 * @param[in] pPma PMA pointer 624 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 625 * 626 * @return 627 * void 628 */ 629 void pmaGetTotalMemory(PMA *pPma, NvU64 *pBytesTotal); 630 631 /*! 632 * @brief Returns information about each region managed by PMA 633 * 634 * @param[in] pPma PMA pointer 635 * @param[out] pRegSize Pointer to size of region descriptor array 636 * @param[out] ppRegionDesc Pointer to the array of region descriptors 637 * 638 * @return 639 * NV_STATUS codes based on convention 640 */ 641 NV_STATUS pmaGetRegionInfo(PMA *pPma, NvU32 *pRegSize, PMA_REGION_DESCRIPTOR **ppRegionDesc); 642 643 /*! 644 * @brief Returns information about the total free FB memory. 645 * 646 * @param[in] pPma PMA pointer 647 * @param[in] pBytesFree Pointer that will return the free FB memory size. 648 * 649 * @return 650 * void 651 */ 652 void pmaGetFreeMemory(PMA *pPma, NvU64 *pBytesFree); 653 654 /*! 655 * @brief Returns information about the largest free FB memory chunk across all regions. 656 * 657 * @param[in] pPma PMA pointer 658 * @param[in] pLargestFree Pointer that will return the largest free FB memory size. 659 * @param[in] pRegionBase Pointer that will return the region base of largest free FB memory. 660 * @param[in] pLargestOffset Pointer that will return the offset in region for largest free FB memory. 661 * 662 * @return 663 * void 664 */ 665 void pmaGetLargestFree(PMA *pPma, NvU64 *pLargestFree, NvU64 *pRegionBase, NvU64 *pLargestOffset); 666 667 /*! 668 * @brief Returns a list of PMA allocated blocks which has ATTRIB_PERSISTENT 669 * attribute set. It will be used by FBSR module to save/restore 670 * clients PMA allocations during system suspend/resume. 671 * 672 * @param[in] pPma PMA pointer 673 * @param[in/out] ppPersistList Pointer to list of persistent segments 674 * 675 * @return 676 * NV_OK Success 677 * NV_ERR_NO_MEMORY Failure to allocate list 678 */ 679 NV_STATUS pmaBuildPersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList); 680 681 682 /*! 683 * @brief Returns a list of all PMA allocated blocks. For all the PMA 684 * allocated blocks, either STATE_PIN or STATE_UNPIN attribute will 685 * be set. It will be used by FBSR module to save/restore clients 686 * PMA allocations for Unix GC-OFF based power management. 687 * 688 * @param[in] pPma PMA pointer 689 * @param[in/out] ppList Pointer to list of all the PMA allocated blocks. 690 * 691 * @return 692 * NV_OK Success 693 * NV_ERR_NO_MEMORY Failure to allocate list 694 */ 695 NV_STATUS pmaBuildAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList); 696 697 698 /*! 699 * @brief Frees previously generated list by function pmaBuildPersistentList(). 700 * 701 * @param[in] pPma PMA pointer 702 * @param[in/out] ppPersistList Pointer to list of persistent segments 703 * 704 * @return 705 * void 706 */ 707 void pmaFreePersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList); 708 709 710 /*! 711 * @brief Frees previously generated list by function 712 * pmaBuildAllocatedBlocksList(). 713 * 714 * @param[in] pPma PMA pointer 715 * @param[in/out] ppList Pointer to list of all the PMA allocated blocks. 716 * 717 * @return 718 * void 719 */ 720 void pmaFreeAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList); 721 722 /*! 723 * @brief Registers a memory scrubber to PMA. Currently only used for 724 * the scrub-on-free feature. 725 * 726 * This function will take the PMA lock to protect against races between 727 * the the use of the MemScrub object and any Reg/Unreg calls. 728 * 729 * @param[in] pPma PMA pointer 730 * @param[in] pScrubObj Pointer to the scrubber 731 * 732 * @return 733 * NV_OK Success 734 * NV_INVALID_STATE When PMA is NULL or bMemScrub is NV_FALSE 735 */ 736 NV_STATUS pmaRegMemScrub(PMA *pPma, OBJMEMSCRUB *pScrubObj); 737 738 739 /*! 740 * @brief Unregisters the memory scrubber, when the scrubber is torn 741 * down. If any feature is turned on that would require this scrubber to be 742 * present, after this call, the PMA object will be unavailable to give out 743 * any free pages for use, until pmaRegMemScrub is called. 744 * 745 * This function will take the PMA lock to protect against races between 746 * the use of the MemScrub object and any Reg/Unreg calls. 747 * 748 * @param[in] pPma PMA pointer 749 * 750 * @return 751 * void 752 */ 753 void pmaUnregMemScrub(PMA *pPma); 754 755 756 /*! 757 * @brief Notifies PMA that node onlining is complete and we can make pass-through 758 * calls to OS. 759 * 760 * This function will take the PMA lock to protect against races between 761 * the use of online/offline calls. 762 * 763 * TODO: Under development! 764 * Currently: 765 * - We online at hardcoded offset and size of PMA memory 766 * - Normal PMA allocation will go to the part of PMA that is not onlined 767 * 768 * Future: 769 * - Online all of PMA memory 770 * - PMA requests will suballocate from OS 771 * - Any mapping created over the PMA range will have to use kmap instead of ioremap 772 * 773 * Bug #1973975, 1858365, 1943187 774 * 775 * @param[in] pPma PMA pointer 776 * @param[in] numaNodeId NUMA node ID that PMA is managing 777 * @param[in] coherentCpuFbBase The ATS aperture base corresponding to start of FB 778 * @param[in] coherentCpuFbSize The ATS aperture size. The actual size we 779 * onlined. This could be different from the HW 780 * ATS aperture size. 781 * 782 * @return 783 * NV_OK Success 784 * NV_INVALID_STATE When PMA is NULL or bNuma is NV_FALSE or nodeId is too big 785 */ 786 NV_STATUS pmaNumaOnlined(PMA *pPma, NvS32 numaNodeId, 787 NvU64 coherentCpuFbBase, NvU64 coherentCpuFbSize); 788 789 790 /*! 791 * @brief Notifies PMA that node offlining has started and PMA should start failing 792 * all allocation calls. 793 * 794 * This function will take the PMA lock to protect against races between 795 * the use of online/offline calls. 796 * 797 * @param[in] pPma PMA pointer 798 * 799 * @return 800 * void 801 */ 802 void pmaNumaOfflined(PMA *pPma); 803 804 /*! 805 * @brief Returns client managed blacklisted pages in the PMA region 806 * 807 * @param[in] pPma PMA pointer 808 * @param[in] pChunks pointer to blacklist addresses in the PMA region 809 * @param[in] pPageSize pointer to Size of each blacklist page addresses 810 * @param[in] pNumChunks pointer to valid client managed blacklist pages 811 * 812 * @return 813 * void 814 */ 815 void pmaGetClientBlacklistedPages(PMA *pPma, NvU64 *pChunks, NvU64 *pPageSize, NvU32 *pNumChunks); 816 817 /*! 818 * @brief Returns the PMA blacklist size in bytes for 819 * both statically and dynamically blacklisted pages. 820 * pDynamicBlacklistSize and pStaticBlacklistSize are only copied-out if non-NULL. 821 * 822 * @param[in] pPma PMA pointer 823 * @param[in] pDynamicBlacklistSize pointer to dynamic blacklist size (bytes) 824 * @param[in] pStaticBlacklistSize pointer to static blacklist size (bytes) 825 * 826 * @return 827 * void 828 */ 829 void pmaGetBlacklistSize(PMA *pPma, NvU32 *pDynamicBlacklistSize, NvU32 *pStaticBlacklistSize); 830 831 /*! 832 * @brief Clear scrub bit for pages from the list of scrub items 833 * that fall in the base to base+size range to return these pages to PMA. 834 * 835 * @param[in] pPma PMA pointer 836 * @param[in] pPmaScrubList list of scrub items each with a id, base and size 837 * @param[in] count count of scrub items 838 * 839 * @return 840 * void 841 */ 842 void pmaClearScrubbedPages(PMA *pPma, SCRUB_NODE *pPmaScrubList, NvU64 count); 843 844 /*! 845 * @brief Print states of all regions 846 */ 847 void pmaPrintMapState(PMA *pPma); 848 849 /*! 850 * @brief Track the given physical address as blacklisted page in PMA. This call will blacklist 851 * the entire PMA page frame of size 64KB which contains the physical address. 852 * 853 * @param[in] pPma PMA pointer 854 * @param[in] physAddr Address of the blacklisted page 855 * 856 * Locking: 857 * - DO NOT call this function with the PMA lock already held. 858 * - This function will internally grab the PMA lock to update the attribute bits. 859 * 860 * @return 861 * void 862 */ 863 NV_STATUS pmaAddToBlacklistTracking(PMA *pPma, NvU64 physBase); 864 865 /*! 866 * @brief Returns total protected video memory. 867 * 868 * @param[in] pPma PMA pointer 869 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 870 * 871 * @return 872 * void 873 */ 874 void pmaGetTotalProtectedMemory(PMA *pPma, NvU64 *pBytesTotal); 875 876 /*! 877 * @brief Returns total unprotected video memory. 878 * 879 * @param[in] pPma PMA pointer 880 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 881 * 882 * @return 883 * void 884 */ 885 void pmaGetTotalUnprotectedMemory(PMA *pPma, NvU64 *pBytesTotal); 886 887 /*! 888 * @brief Returns information about the total free protected FB memory. 889 * In confidential compute use cases, memory will be split into 890 * protected and unprotected regions 891 * 892 * @param[in] pPma PMA pointer 893 * @param[in] pBytesFree Pointer that will return the free protected memory size. 894 * 895 * @return 896 * void 897 */ 898 void pmaGetFreeProtectedMemory(PMA *pPma, NvU64 *pBytesFree); 899 900 /*! 901 * @brief Returns information about the total free unprotected FB memory. 902 * In confidential compute use cases, memory will be split into 903 * protected and unprotected regions 904 * 905 * @param[in] pPma PMA pointer 906 * @param[in] pBytesFree Pointer that will return the free unprotected memory size. 907 * 908 * @return 909 * void 910 */ 911 void pmaGetFreeUnprotectedMemory(PMA *pPma, NvU64 *pBytesFree); 912 913 #ifdef __cplusplus 914 } 915 #endif 916 917 #endif // PHYS_MEM_ALLOCATOR_H 918