1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /*! 25 * @brief This file exposes the PMA interfaces. 26 * The PMA module interacts with the RM and UVM components. UVM will use the 27 * ops layer to call out to PMA, while RM directly calls into PMA. 28 * The PMA module takes a global lock to protect its internal structure. It 29 * uses a bitmap structure called regmap. 30 * 31 * @bug 32 * 1. status code -- decide if we need to add to global 33 * 2. suspend/resume -- might add one more function to support 34 * 35 * @TODO 36 * 1. external fragmentation 37 * 2. use new scrubber API and remove the initScrubbing atomic variable 38 */ 39 40 #ifndef PHYS_MEM_ALLOCATOR_H 41 #define PHYS_MEM_ALLOCATOR_H 42 43 #include "nvport/nvport.h" 44 #include "regmap.h" 45 #include "addrtree.h" 46 #include "nvmisc.h" 47 48 #if defined(SRT_BUILD) 49 #define RMCFG_MODULE_x 1 50 #else 51 #include "rmconfig.h" 52 #endif 53 54 #ifdef __cplusplus 55 extern "C" { 56 #endif 57 58 typedef struct OBJMEMSCRUB OBJMEMSCRUB; 59 typedef struct SCRUB_NODE SCRUB_NODE; 60 61 #define PMA_REGION_SIZE 64 62 #define PMA_ADDR2FRAME(addr, base) (((addr) - (base)) >> PMA_PAGE_SHIFT) 63 #define PMA_FRAME2ADDR(frame, base) ((base) + ((frame) << PMA_PAGE_SHIFT)) 64 65 // 66 // These flags are used for initialization in order to set global PMA states, 67 // in case we need to wait for scrubber to be initialized or wait for a NUMA 68 // node being onlined, etc. 69 // 70 #define PMA_INIT_NONE NVBIT(0) 71 #define PMA_INIT_SCRUB_ON_FREE NVBIT(1) 72 #define PMA_INIT_NUMA NVBIT(2) 73 #define PMA_INIT_INTERNAL NVBIT(3) // Used after heap is removed 74 #define PMA_INIT_FORCE_PERSISTENCE NVBIT(4) 75 #define PMA_INIT_ADDRTREE NVBIT(5) 76 #define PMA_INIT_NUMA_AUTO_ONLINE NVBIT(6) 77 78 // These flags are used for querying PMA's config and/or state. 79 #define PMA_QUERY_SCRUB_ENABLED NVBIT(0) 80 #define PMA_QUERY_SCRUB_VALID NVBIT(1) 81 #define PMA_QUERY_NUMA_ENABLED NVBIT(2) 82 #define PMA_QUERY_NUMA_ONLINED NVBIT(3) 83 84 // 85 // When modifying flags, make sure they are compatible with the mirrored 86 // UVM_PMA_* flags in nv_uvm_types.h. 87 // 88 // Input flags 89 #define PMA_ALLOCATE_DONT_EVICT NVBIT(0) 90 #define PMA_ALLOCATE_PINNED NVBIT(1) 91 #define PMA_ALLOCATE_SPECIFY_MINIMUM_SPEED NVBIT(2) 92 #define PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE NVBIT(3) 93 #define PMA_ALLOCATE_SPECIFY_REGION_ID NVBIT(4) 94 #define PMA_ALLOCATE_PREFER_SLOWEST NVBIT(5) 95 #define PMA_ALLOCATE_CONTIGUOUS NVBIT(6) 96 #define PMA_ALLOCATE_PERSISTENT NVBIT(7) 97 #define PMA_ALLOCATE_PROTECTED_REGION NVBIT(8) 98 #define PMA_ALLOCATE_FORCE_ALIGNMENT NVBIT(9) 99 #define PMA_ALLOCATE_NO_ZERO NVBIT(10) 100 #define PMA_ALLOCATE_TURN_BLACKLIST_OFF NVBIT(11) 101 #define PMA_ALLOCATE_ALLOW_PARTIAL NVBIT(12) 102 #define PMA_ALLOCATE_REVERSE_ALLOC NVBIT(13) 103 104 // Output flags 105 #define PMA_ALLOCATE_RESULT_IS_ZERO NVBIT(0) 106 107 // These are flags input to the pmaFreePages call 108 #define PMA_FREE_SKIP_SCRUB NVBIT(0) 109 110 // State bits for debugging utilities like nvwatch 111 #define PMA_SCRUB_INITIALIZE 0 112 #define PMA_SCRUB_IN_PROGRESS 1 113 #define PMA_SCRUB_DONE 2 114 115 #define PMA_SCRUBBER_VALID 1 116 #define PMA_SCRUBBER_INVALID 0 117 118 #define PMA_NUMA_NO_NODE -1 119 120 // Maximum blacklist entries possible 121 #define PMA_MAX_BLACKLIST_ENTRIES 512 122 123 typedef struct 124 { 125 NvU32 flags; 126 NvU32 minimumSpeed; // valid if flags & PMA_ALLOCATE_SPECIFY_MININUM_SPEED 127 NvU64 physBegin, physEnd; // valid if flags & PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE 128 NvU32 regionId; // valid if flags & PMA_ALLOCATE_SPECIFY_REGION_ID 129 NvU64 alignment; // valid if flags & PMA_ALLOCATE_FORCE_ALIGNMENT 130 NvLength numPagesAllocated; // valid if flags & PMA_ALLOCATE_ALLOW_PARTIAL 131 132 NvU32 resultFlags; // valid if the allocation function returns NV_OK 133 } PMA_ALLOCATION_OPTIONS; 134 135 // 136 // Explanation: This struct will be provided when UVM/RM registers a region with PMA, 137 // after which the struct is stored locally in PMA. The internal "filter" function will 138 // use the information everytime a request comes in. 139 // 140 typedef struct 141 { 142 NvU64 base; // Base/start address of the region 143 NvU64 limit; // Last/end address of region 144 NvU32 performance; // Relative performance. Higher is faster 145 NvBool bSupportCompressed; // Support compressed kinds 146 NvBool bSupportISO; // Support ISO (display, cursor, video) surfaces 147 NvBool bProtected; // Represents a protected region of memory. 148 } PMA_REGION_DESCRIPTOR; 149 150 typedef struct _PMA_MAP_INFO PMA_MAP_INFO; 151 typedef struct _PMA PMA; 152 153 // Range descriptors for managing persistent range lists 154 typedef struct _RANGELISTTYPE 155 { 156 NvU64 base; 157 NvU64 limit; 158 struct _RANGELISTTYPE *pNext; 159 } RANGELISTTYPE, *PRANGELISTTYPE; 160 161 typedef enum 162 { 163 MEMORY_PROTECTION_UNPROTECTED = 0, 164 MEMORY_PROTECTION_PROTECTED = 1 165 } MEMORY_PROTECTION; 166 167 /*! 168 * @brief Callbacks to UVM for eviction 169 */ 170 typedef NV_STATUS (*pmaEvictPagesCb_t)(void *ctxPtr, NvU64 pageSize, NvU64 *pPages, 171 NvU32 count, NvU64 physBegin, NvU64 physEnd, 172 MEMORY_PROTECTION prot); 173 typedef NV_STATUS (*pmaEvictRangeCb_t)(void *ctxPtr, NvU64 physBegin, NvU64 physEnd, 174 MEMORY_PROTECTION prot); 175 176 /*! 177 * @brief Pluggable data structure management. Currently we have regmap and address tree. 178 */ 179 typedef void *(*pmaMapInit_t)(NvU64 numFrames, NvU64 addrBase, PMA_STATS *pPmaStats, NvBool bProtected); 180 typedef void (*pmaMapDestroy_t)(void *pMap); 181 typedef void (*pmaMapChangeState_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState); 182 typedef void (*pmaMapChangeStateAttrib_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, NvBool writeAttrib); 183 typedef void (*pmaMapChangeStateAttribEx_t)(void *pMap, NvU64 frameNum, PMA_PAGESTATUS newState, PMA_PAGESTATUS newStateMask); 184 typedef void (*pmaMapChangePageStateAttrib_t)(void *pMap, NvU64 startFrame, NvU64 pageSize, PMA_PAGESTATUS newState, NvBool writeAttrib); 185 typedef PMA_PAGESTATUS (*pmaMapRead_t)(void *pMap, NvU64 frameNum, NvBool readAttrib); 186 typedef NV_STATUS (*pmaMapScanContiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd, 187 NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment, 188 NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc); 189 typedef NV_STATUS (*pmaMapScanDiscontiguous_t)(void *pMap, NvU64 addrBase, NvU64 rangeStart, NvU64 rangeEnd, 190 NvU64 numPages, NvU64 *freelist, NvU64 pageSize, NvU64 alignment, 191 NvU64 *pagesAllocated, NvBool bSkipEvict, NvBool bReverseAlloc); 192 typedef void (*pmaMapGetSize_t)(void *pMap, NvU64 *pBytesTotal); 193 typedef void (*pmaMapGetLargestFree_t)(void *pMap, NvU64 *pLargestFree); 194 typedef NV_STATUS (*pmaMapScanContiguousNumaEviction_t)(void *pMap, NvU64 addrBase, NvLength actualSize, 195 NvU64 pageSize, NvU64 *evictStart, NvU64 *evictEnd); 196 typedef NvU64 (*pmaMapGetEvictingFrames_t)(void *pMap); 197 typedef void (*pmaMapSetEvictingFrames_t)(void *pMap, NvU64 frameEvictionsInProcess); 198 199 struct _PMA_MAP_INFO 200 { 201 NvU32 mode; 202 pmaMapInit_t pmaMapInit; 203 pmaMapDestroy_t pmaMapDestroy; 204 pmaMapChangeState_t pmaMapChangeState; 205 pmaMapChangeStateAttrib_t pmaMapChangeStateAttrib; 206 pmaMapChangeStateAttribEx_t pmaMapChangeStateAttribEx; 207 pmaMapChangePageStateAttrib_t pmaMapChangePageStateAttrib; 208 pmaMapRead_t pmaMapRead; 209 pmaMapScanContiguous_t pmaMapScanContiguous; 210 pmaMapScanDiscontiguous_t pmaMapScanDiscontiguous; 211 pmaMapGetSize_t pmaMapGetSize; 212 pmaMapGetLargestFree_t pmaMapGetLargestFree; 213 pmaMapScanContiguousNumaEviction_t pmaMapScanContiguousNumaEviction; 214 pmaMapGetEvictingFrames_t pmaMapGetEvictingFrames; 215 pmaMapSetEvictingFrames_t pmaMapSetEvictingFrames; 216 }; 217 218 struct _PMA 219 { 220 PORT_SPINLOCK *pPmaLock; // PMA-wide lock 221 PORT_MUTEX *pEvictionCallbacksLock; // Eviction callback registration lock 222 223 // Only used when free scrub-on-free feature is turned on 224 PORT_RWLOCK *pScrubberValidLock; // A reader-writer lock to protect the scrubber valid bit 225 PORT_MUTEX *pAllocLock; // Used to protect page stealing in the allocation path 226 227 // Region related states 228 NvU32 regSize; // Actual size of regions array 229 void * pRegions[PMA_REGION_SIZE]; // All the region maps stored as opaque pointers 230 NvU32 *pSortedFastFirst; // Pre-sorted array of region IDs 231 PMA_REGION_DESCRIPTOR *pRegDescriptors [PMA_REGION_SIZE]; // Stores the descriptions of each region 232 PMA_MAP_INFO *pMapInfo; // The pluggable layer for managing scanning 233 234 // Allocation related states 235 void * evictCtxPtr; // Opaque context pointer for eviction callback 236 pmaEvictPagesCb_t evictPagesCb; // Discontiguous eviction callback 237 pmaEvictRangeCb_t evictRangeCb; // Contiguous eviction callback 238 NvU64 frameAllocDemand; // Frame count of allocations in-process 239 NvBool bForcePersistence; // Force all allocations to persist across suspend/resume 240 PMA_STATS pmaStats; // PMA statistics used for client heuristics 241 242 // Scrubber related states 243 NvSPtr initScrubbing; // If the init scrubber has finished in this PMA 244 NvBool bScrubOnFree; // If "scrub on free" is enabled for this PMA object 245 NvSPtr scrubberValid; // If scrubber object is valid, using atomic variable to prevent races 246 OBJMEMSCRUB *pScrubObj; // Object to store the FreeScrub header 247 248 // NUMA states 249 NvBool bNuma; // If we are allocating for a NUMA system 250 NvBool nodeOnlined; // If node is onlined 251 NvS32 numaNodeId; // Current Node ID, set at initialization. -1 means invalid 252 NvU64 coherentCpuFbBase; // Used to calculate FB offset from bus address 253 NvU64 coherentCpuFbSize; // Used for error checking only 254 NvU32 numaReclaimSkipThreshold; // percent value below which __GFP_RECLAIM will not be used. 255 NvBool bNumaAutoOnline; // If NUMA memory is auto-onlined 256 257 // Blacklist related states 258 PMA_BLACKLIST_CHUNK *pBlacklistChunks; // Tracking for blacklist pages 259 NvU32 blacklistCount; // Number of blacklist pages 260 NvBool bClientManagedBlacklist; // Blacklisted pages in PMA that will be taken over by Client 261 }; 262 263 /*! 264 * @brief This must be called before any other PMA functions. Returns a PMA 265 * object for later use 266 * 267 * @param[in] gpuId The UVM global GPU ID. Defined in nvCpuUuid.h as a 16 268 * byte digest. PMA will only store reference to it. 269 * @param[in] pma Pointer to PMA object being initialized 270 * @param[in] initFlags PMA initialization flags for special modes 271 * @return 272 * NV_ERR_INVALID_ARGUMENT: 273 * If the combination of initFlags is invalid. 274 * NV_ERR_NO_MEMORY: 275 * Internal memory allocation failed. 276 * NV_ERR_GENERIC: 277 * Unexpected error. We try hard to avoid returning this error 278 * code, because it is not very informative. 279 * 280 */ 281 NV_STATUS pmaInitialize(PMA *pPma, NvU32 initFlags); 282 283 284 /*! 285 * @brief Release a PMA object. Also frees memory. 286 * 287 * All eviction handlers must have been unregistered by this point. 288 * 289 * @param[in] pma Pointer to PMA object being destroyed. 290 */ 291 void pmaDestroy(PMA *pPma); 292 293 294 /*! 295 * @brief Queries PMA configuration and state. 296 * 297 * Any clients of PMA can query config and state with a valid PMA object. 298 * Querying at different times may return different values when states change. 299 * 300 * @param[in] pma Pointer to PMA object being destroyed. 301 * @param[in/out] pConfigs Configs/states to query. See PMA_QUERY_* above. 302 */ 303 NV_STATUS pmaQueryConfigs(PMA* pPma, NvU32 *pConfigs); 304 305 306 /*! 307 * @brief Attaches a region of physical memory to be managed by the PMA. 308 * 309 * Systems with floorswept memory configurations will have (multi gigabyte) 310 * holes in memory. Each contiguous region should be reported by this 311 * registration function. This is also intended to be used by systems with 312 * two speed memories. 313 * 314 * Note: Some 0FB configurations may choose to skip registering any regions. 315 * At most 64 regions may be registered with a single PMA. 316 * 317 * @param[in] id 318 * A unique value in the range [0, 64) that uniquely identifies this 319 * region. Passed in by RM in region-order. Should be continuous for 320 * best performance. 321 * 322 * @param[in] bAsyncEccScrub 323 * RM will set this when it is performing the initial ECC scrub 324 * asynchronously. All pages in the pma will be marked 'allocated pinned'. 325 * RM will call pmaFreeContiguous as memory is scrubbed. 326 * 327 * Until the scrub is complete (by calling pmaScrubComplete), no 328 * allocation calls will fail with out of memory. Instead, it will hang 329 * until scrubbing is complete. One exception is when the client passes 330 * in the PMA_DONT_EVICT flag, in which case the call will actually fail 331 * regardless of whether RM is scrubbing memory. 332 * 333 * CAUTION! RM is responsible for ensuring black-listed pages are not 334 * marked free during the scrub. 335 * 336 * @param[in] regionState: 337 * Contains physical information about the region. 338 * 339 * @param[in] pBlacklistPageBase: 340 * List of base addresses of bad GPU pages. 341 * Each address is assumed to reference to a page of size 342 * PMA_GRANULARITY (64kb). 343 * 344 * Implementors note: PMA will simply mark these pages as "allocatedPinned". 345 * This list should be saved so that during pmaDestroy we can verify that only 346 * these pages are still allocated. 347 * 348 * @param[in] blacklistCount: 349 * Number of pages in above list. 350 * 351 * @return 352 * NV_ERR_NO_MEMORY: 353 * Internal memory allocation failed. 354 * NV_ERR_GENERIC: 355 * Unexpected error. We try hard to avoid returning this error code, 356 * because it is not very informative. 357 * 358 */ 359 NV_STATUS pmaRegisterRegion(PMA *pPma, NvU32 id, NvBool bAsyncEccScrub, 360 PMA_REGION_DESCRIPTOR *pRegionDesc, NvU32 blacklistCount, PPMA_BLACKLIST_ADDRESS pBlacklistPage); 361 362 363 /*! 364 * @brief Synchronous API for allocating pages from the PMA. 365 * PMA will decide which pma regions to allocate from based on the provided 366 * flags. PMA will also initiate UVM evictions to make room for this 367 * allocation unless prohibited by PMA_FLAGS_DONT_EVICT. UVM callers must pass 368 * this flag to avoid deadlock. Only UVM may allocate unpinned memory from this 369 * API and note that eviction callbacks need to be registered before that 370 * happens. 371 * 372 * Alignment of the allocated pages is guaranteed to be greater or equal to the 373 * requested page size. For contiguous allocations, a greater alignment can be 374 * specified with the PMA_ALLOCATE_FORCE_ALIGNMENT flag and the alignment 375 * allocation option. For non-contiguous allocations, it's an error to specify 376 * an alignment larger than the page size. 377 * 378 * For broadcast methods, PMA will guarantee the same physical frames are 379 * allocated on multiple GPUs, specified by the PMA objects passed in. 380 * 381 * Implementors note: 382 * If region registered with asyncEccScrub and pmaScrubComplete 383 * has not yet been issued then we cannot return NV_ERR_NO_MEMORY. 384 * We must instead drop the lock and wait until the next call to 385 * either pmaScrubComplete or pmaFreeContiguous/Pages to retry. 386 * Exception: PMA_ALLOCATE_DONT_EVICT 387 * 388 * @param[in] pPma 389 * The input PMA object 390 * 391 * @param[in] pageCount 392 * Number of pages to allocate. 393 * 394 * @param[in] pageSize 395 * 64kb, 128kb or 2mb. No other values are permissible. 396 * 397 * @param[in/out] allocationOptions 398 * Input flags: 399 * PMA_ALLOCATE_DONT_EVICT 400 * Do not evict in order to service this allocation. 401 * Do not wait for ECC scrub completion if out of memory. 402 * PMA_ALLOCATE_PINNED 403 * The allocation is pinned (RM must pass this) 404 * PMA_ALLOCATE_SPECIFY_MININUM_SPEED 405 * Only examines regions whose speed is greater than 406 * minimumSpeed. 407 * PMA_ALLOCATE_SPECIFY_ADDRESS_RANGE 408 * Restrict the allowable physical address range for 409 * allocation to [physBegin, physEnd). 410 * PMA_ALLOCATE_SPECIFY_REGION_ID 411 * Only service allocations out of 'regionId'. The 412 * regionId is assigned in pmaRegisterRegion. 413 * PMA_ALLOCATE_PREFER_SLOWEST 414 * Prefer slower memory over faster. 415 * PMA_ALLOCATE_CONTIGUOUS 416 * If this allocation is a contiguous allocation 417 * PMA_ALLOCATE_PERSISTENT 418 * If this allocation should persist across suspend/resume 419 * PMA_ALLOCATE_FORCE_ALIGNMENT 420 * Force a specific alignment of the allocation. For non-contiguous 421 * allocations has to be less or equal to the page size. 422 * 423 * Output flags: 424 * PMA_ALLOCATE_RESULT_IS_ZERO 425 * If the allocated pages have been scrubbed. 426 * 427 * @param[out] pPages 428 * Return array of base addresses of allocated pages. 429 * 430 * @return 431 * NV_ERR_NO_MEMORY: 432 * Internal memory allocation failed. 433 * NV_ERR_GENERIC: 434 * Unexpected error. We try hard to avoid returning this error 435 * code,because it is not very informative. 436 * 437 */ 438 NV_STATUS pmaAllocatePages(PMA *pPma, NvLength pageCount, NvU64 pageSize, 439 PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages); 440 441 // allocate on multiple GPU, thus pmaCount 442 NV_STATUS pmaAllocatePagesBroadcast(PMA **pPma, NvU32 pmaCount, NvLength allocationCount, 443 NvU64 pageSize, PMA_ALLOCATION_OPTIONS *pAllocationOptions, NvU64 *pPages); 444 445 446 /*! 447 * @brief Marks previously unpinned pages as pinned. 448 * 449 * It will return an error and rollback any change if any page is not 450 * previously marked "unpinned". 451 * 452 * @param[in] pPages 453 * Array of base addresses of pages to pin 454 * 455 * @param[in] pageCount 456 * Number of pages to pin 457 * 458 * @param[in] pageSize 459 * Page size of each page being pinned 460 * 461 * @return 462 * NV_OK: 463 * The pages have been pinned successfully. 464 * 465 * NV_ERR_IN_USE: 466 * Some of the pages requested to be pinned are being evicted and thus 467 * cannot be pinned. None of the pages have been pinned and the caller 468 * needs to make sure the pages can be successfully evicted. 469 * 470 * NV_ERR_INVALID_STATE: 471 * Some of the pages requested to be pinned weren't in the allocated 472 * unpinned state. 473 * 474 * TODO some error for rollback 475 * 476 */ 477 NV_STATUS pmaPinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize); 478 479 480 /*! 481 * @brief Marks previously pinned pages as unpinned. 482 * 483 * It will return an error and rollback any change if any page is not 484 * previously marked "pinned". Behaviour is undefined is any blacklisted 485 * pages are unpinned. 486 * 487 * @param[in] pPages 488 * Array of base addresses of pages to pin 489 * 490 * @param[in] pageCount 491 * Number of pages to pin 492 * 493 * @param[in] pageSize 494 * Page size of each page being unpinned 495 * 496 * @return 497 * NV_ERR_GENERIC: 498 * Unexpected error. We try hard to avoid returning this error 499 * code,because it is not very informative. 500 * TODO some error for rollback 501 * 502 */ 503 NV_STATUS pmaUnpinPages(PMA *pPma, NvU64 *pPages, NvLength pageCount, NvU64 pageSize); 504 505 506 /*! 507 * @brief Marks a list of pages as free. 508 * This operation is also used by RM to mark pages as "scrubbed" for the 509 * initial ECC sweep. This function does not fail. 510 * 511 * @param[in] pPages 512 * Array of base addresses of pages to free 513 * 514 * @param[in] pageCount 515 * Number of pages to free 516 * If the value is 1, this is a contiguous free 517 * 518 * @param[in] size 519 * When freeing contiguous memory, this is the total size; 520 * When freeing discontiguous memory, this is page size of each page. 521 * 522 * @param[in] flag 523 * PMA_FREE_SKIP_SCRUB 524 * This flag is used to disable scrub on free when PMA frees the page 525 * 526 * @return Void 527 * 528 */ 529 void pmaFreePages(PMA *pPma, NvU64 *pPages, NvU64 pageCount, NvU64 size, NvU32 flag); 530 531 /*! 532 * @brief Clears scrubbing bit on PMA pages within the supplied range. 533 * 534 * @param[in] pma 535 * PMA object 536 * 537 * @param[in] rangeBase 538 * The start address 539 * 540 * @param[in] rangeLimit 541 * The end address 542 * 543 * @return 544 * void 545 */ 546 void pmaClearScrubRange(PMA *pPma, NvU64 rangeBase, NvU64 rangeLimit); 547 548 549 /*! 550 * @brief Notifies the PMA that the ECC scrub is complete. 551 * 552 * Until this function is called no PMA allocations will fail with 553 * "insufficient memory". They will hang and wait for the scrubbing. 554 * 555 * TODO consider suspend/resume behavior! 556 * 557 * Design Note: 558 * pmaRegisterRegion(w/ asyncEccScrub) leaves all pages as 559 * "allocated and pinned". 560 * 561 * As the ECC scrub progresses, RM will call PMA and "pmaFreeContiguous" 562 * the regions as they are scrubbed. 563 * 564 * @param[in] pma 565 * PMA object 566 * 567 * @return 568 * NV_ERR_GENERIC: 569 * Unexpected error. We try hard to avoid returning this error 570 * code,because it is not very informative. 571 * 572 */ 573 NV_STATUS pmaScrubComplete(PMA *pPma); 574 575 576 /*! 577 * Register the eviction callbacks. 578 * 579 * Only one set of callbacks can be registered at a time and they need to be 580 * unregistered with pmaUnregisterEvictionCb() before a new set can be 581 * registered. 582 * 583 * Note that eviction callbacks need to be registered before allocating any unpinned memory. 584 * 585 * See the documentation of the callbacks above for details of the eviction. 586 * 587 * @param[in] pma 588 * PMA object 589 * 590 * @param[in] evictPageCb 591 * The callback function for evicting pages at a time 592 * 593 * @param[in] evictRangeCb 594 * The callback function for evicting a range 595 * 596 * @param[in] ctxPtr 597 * The callback context pointer to be passed back on callback 598 * 599 * @return 600 * NV_ERR_INVALID_ARGUMENT: 601 * One of the callbacks or PMA object was NULL. 602 * 603 * NV_ERR_INVALID_STATE: 604 * Callbacks already registered. 605 */ 606 NV_STATUS pmaRegisterEvictionCb(PMA *pPma, pmaEvictPagesCb_t evictPagesCb, pmaEvictRangeCb_t evictRangeCb, void *ctxPtr); 607 608 609 /*! 610 * Unregister the eviction callbacks. 611 * 612 * Guarantees that all pending eviction callbacks complete before returning. 613 * 614 * All unpinned allocations must be freed before the callbacks are unregistered 615 * and the caller needs to guarantee that any pending eviction callbacks won't 616 * block on the thread unregistering the callbacks. 617 * 618 * The call does nothing if the PMA object is NULL. 619 * 620 * @param[in] pma 621 * PMA object. 622 */ 623 void pmaUnregisterEvictionCb(PMA *pPma); 624 625 /*! 626 * @brief Returns information about the total FB memory. 627 * 628 * @param[in] pPma PMA pointer 629 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 630 * 631 * @return 632 * void 633 */ 634 void pmaGetTotalMemory(PMA *pPma, NvU64 *pBytesTotal); 635 636 /*! 637 * @brief Returns information about each region managed by PMA 638 * 639 * @param[in] pPma PMA pointer 640 * @param[out] pRegSize Pointer to size of region descriptor array 641 * @param[out] ppRegionDesc Pointer to the array of region descriptors 642 * 643 * @return 644 * NV_STATUS codes based on convention 645 */ 646 NV_STATUS pmaGetRegionInfo(PMA *pPma, NvU32 *pRegSize, PMA_REGION_DESCRIPTOR **ppRegionDesc); 647 648 /*! 649 * @brief Returns information about the total free FB memory. 650 * 651 * @param[in] pPma PMA pointer 652 * @param[in] pBytesFree Pointer that will return the free FB memory size. 653 * 654 * @return 655 * void 656 */ 657 void pmaGetFreeMemory(PMA *pPma, NvU64 *pBytesFree); 658 659 /*! 660 * @brief Returns information about the largest free FB memory chunk across all regions. 661 * 662 * @param[in] pPma PMA pointer 663 * @param[in] pLargestFree Pointer that will return the largest free FB memory size. 664 * @param[in] pRegionBase Pointer that will return the region base of largest free FB memory. 665 * @param[in] pLargestOffset Pointer that will return the offset in region for largest free FB memory. 666 * 667 * @return 668 * void 669 */ 670 void pmaGetLargestFree(PMA *pPma, NvU64 *pLargestFree, NvU64 *pRegionBase, NvU64 *pLargestOffset); 671 672 /*! 673 * @brief Returns a list of PMA allocated blocks which has ATTRIB_PERSISTENT 674 * attribute set. It will be used by FBSR module to save/restore 675 * clients PMA allocations during system suspend/resume. 676 * 677 * @param[in] pPma PMA pointer 678 * @param[in/out] ppPersistList Pointer to list of persistent segments 679 * 680 * @return 681 * NV_OK Success 682 * NV_ERR_NO_MEMORY Failure to allocate list 683 */ 684 NV_STATUS pmaBuildPersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList); 685 686 687 /*! 688 * @brief Returns a list of all PMA allocated blocks. For all the PMA 689 * allocated blocks, either STATE_PIN or STATE_UNPIN attribute will 690 * be set. It will be used by FBSR module to save/restore clients 691 * PMA allocations for Unix GC-OFF based power management. 692 * 693 * @param[in] pPma PMA pointer 694 * @param[in/out] ppList Pointer to list of all the PMA allocated blocks. 695 * 696 * @return 697 * NV_OK Success 698 * NV_ERR_NO_MEMORY Failure to allocate list 699 */ 700 NV_STATUS pmaBuildAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList); 701 702 703 /*! 704 * @brief Frees previously generated list by function pmaBuildPersistentList(). 705 * 706 * @param[in] pPma PMA pointer 707 * @param[in/out] ppPersistList Pointer to list of persistent segments 708 * 709 * @return 710 * void 711 */ 712 void pmaFreePersistentList(PMA *pPma, PRANGELISTTYPE *ppPersistList); 713 714 715 /*! 716 * @brief Frees previously generated list by function 717 * pmaBuildAllocatedBlocksList(). 718 * 719 * @param[in] pPma PMA pointer 720 * @param[in/out] ppList Pointer to list of all the PMA allocated blocks. 721 * 722 * @return 723 * void 724 */ 725 void pmaFreeAllocatedBlocksList(PMA *pPma, PRANGELISTTYPE *ppList); 726 727 /*! 728 * @brief Registers a memory scrubber to PMA. Currently only used for 729 * the scrub-on-free feature. 730 * 731 * This function will take the PMA lock to protect against races between 732 * the the use of the MemScrub object and any Reg/Unreg calls. 733 * 734 * @param[in] pPma PMA pointer 735 * @param[in] pScrubObj Pointer to the scrubber 736 * 737 * @return 738 * NV_OK Success 739 * NV_INVALID_STATE When PMA is NULL or bMemScrub is NV_FALSE 740 */ 741 NV_STATUS pmaRegMemScrub(PMA *pPma, OBJMEMSCRUB *pScrubObj); 742 743 744 /*! 745 * @brief Unregisters the memory scrubber, when the scrubber is torn 746 * down. If any feature is turned on that would require this scrubber to be 747 * present, after this call, the PMA object will be unavailable to give out 748 * any free pages for use, until pmaRegMemScrub is called. 749 * 750 * This function will take the PMA lock to protect against races between 751 * the use of the MemScrub object and any Reg/Unreg calls. 752 * 753 * @param[in] pPma PMA pointer 754 * 755 * @return 756 * void 757 */ 758 void pmaUnregMemScrub(PMA *pPma); 759 760 761 /*! 762 * @brief Notifies PMA that node onlining is complete and we can make pass-through 763 * calls to OS. 764 * 765 * This function will take the PMA lock to protect against races between 766 * the use of online/offline calls. 767 * 768 * TODO: Under development! 769 * Currently: 770 * - We online at hardcoded offset and size of PMA memory 771 * - Normal PMA allocation will go to the part of PMA that is not onlined 772 * 773 * Future: 774 * - Online all of PMA memory 775 * - PMA requests will suballocate from OS 776 * - Any mapping created over the PMA range will have to use kmap instead of ioremap 777 * 778 * Bug #1973975, 1858365, 1943187 779 * 780 * @param[in] pPma PMA pointer 781 * @param[in] numaNodeId NUMA node ID that PMA is managing 782 * @param[in] coherentCpuFbBase The ATS aperture base corresponding to start of FB 783 * @param[in] coherentCpuFbSize The ATS aperture size. The actual size we 784 * onlined. This could be different from the HW 785 * ATS aperture size. 786 * 787 * @return 788 * NV_OK Success 789 * NV_INVALID_STATE When PMA is NULL or bNuma is NV_FALSE or nodeId is too big 790 */ 791 NV_STATUS pmaNumaOnlined(PMA *pPma, NvS32 numaNodeId, 792 NvU64 coherentCpuFbBase, NvU64 coherentCpuFbSize); 793 794 795 /*! 796 * @brief Notifies PMA that node offlining has started and PMA should start failing 797 * all allocation calls. 798 * 799 * This function will take the PMA lock to protect against races between 800 * the use of online/offline calls. 801 * 802 * @param[in] pPma PMA pointer 803 * 804 * @return 805 * void 806 */ 807 void pmaNumaOfflined(PMA *pPma); 808 809 /*! 810 * @brief Returns client managed blacklisted pages in the PMA region 811 * 812 * @param[in] pPma PMA pointer 813 * @param[in] pChunks pointer to blacklist addresses in the PMA region 814 * @param[in] pPageSize pointer to Size of each blacklist page addresses 815 * @param[in] pNumChunks pointer to valid client managed blacklist pages 816 * 817 * @return 818 * void 819 */ 820 void pmaGetClientBlacklistedPages(PMA *pPma, NvU64 *pChunks, NvU64 *pPageSize, NvU32 *pNumChunks); 821 822 /*! 823 * @brief Returns the PMA blacklist size in bytes for 824 * both statically and dynamically blacklisted pages. 825 * pDynamicBlacklistSize and pStaticBlacklistSize are only copied-out if non-NULL. 826 * 827 * @param[in] pPma PMA pointer 828 * @param[in] pDynamicBlacklistSize pointer to dynamic blacklist size (bytes) 829 * @param[in] pStaticBlacklistSize pointer to static blacklist size (bytes) 830 * 831 * @return 832 * void 833 */ 834 void pmaGetBlacklistSize(PMA *pPma, NvU32 *pDynamicBlacklistSize, NvU32 *pStaticBlacklistSize); 835 836 /*! 837 * @brief Clear scrub bit for pages from the list of scrub items 838 * that fall in the base to base+size range to return these pages to PMA. 839 * 840 * @param[in] pPma PMA pointer 841 * @param[in] pPmaScrubList list of scrub items each with a id, base and size 842 * @param[in] count count of scrub items 843 * 844 * @return 845 * void 846 */ 847 void pmaClearScrubbedPages(PMA *pPma, SCRUB_NODE *pPmaScrubList, NvU64 count); 848 849 /*! 850 * @brief Print states of all regions 851 */ 852 void pmaPrintMapState(PMA *pPma); 853 854 /*! 855 * @brief Track the given physical address as blacklisted page in PMA. This call will blacklist 856 * the entire PMA page frame of size 64KB which contains the physical address. 857 * 858 * @param[in] pPma PMA pointer 859 * @param[in] physAddr Address of the blacklisted page 860 * 861 * Locking: 862 * - DO NOT call this function with the PMA lock already held. 863 * - This function will internally grab the PMA lock to update the attribute bits. 864 * 865 * @return 866 * void 867 */ 868 NV_STATUS pmaAddToBlacklistTracking(PMA *pPma, NvU64 physBase); 869 870 /*! 871 * @brief Returns total protected video memory. 872 * 873 * @param[in] pPma PMA pointer 874 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 875 * 876 * @return 877 * void 878 */ 879 void pmaGetTotalProtectedMemory(PMA *pPma, NvU64 *pBytesTotal); 880 881 /*! 882 * @brief Returns total unprotected video memory. 883 * 884 * @param[in] pPma PMA pointer 885 * @param[in] pBytesTotal Pointer that will return the total FB memory size. 886 * 887 * @return 888 * void 889 */ 890 void pmaGetTotalUnprotectedMemory(PMA *pPma, NvU64 *pBytesTotal); 891 892 /*! 893 * @brief Returns information about the total free protected FB memory. 894 * In confidential compute use cases, memory will be split into 895 * protected and unprotected regions 896 * 897 * @param[in] pPma PMA pointer 898 * @param[in] pBytesFree Pointer that will return the free protected memory size. 899 * 900 * @return 901 * void 902 */ 903 void pmaGetFreeProtectedMemory(PMA *pPma, NvU64 *pBytesFree); 904 905 /*! 906 * @brief Returns information about the total free unprotected FB memory. 907 * In confidential compute use cases, memory will be split into 908 * protected and unprotected regions 909 * 910 * @param[in] pPma PMA pointer 911 * @param[in] pBytesFree Pointer that will return the free unprotected memory size. 912 * 913 * @return 914 * void 915 */ 916 void pmaGetFreeUnprotectedMemory(PMA *pPma, NvU64 *pBytesFree); 917 918 #ifdef __cplusplus 919 } 920 #endif 921 922 #endif // PHYS_MEM_ALLOCATOR_H 923