1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2014-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /** 25 * @file 26 * @brief Memory module public interface 27 */ 28 29 #ifndef _NVPORT_H_ 30 #error "This file cannot be included directly. Include nvport.h instead." 31 #endif 32 33 #ifndef _NVPORT_MEMORY_H_ 34 #define _NVPORT_MEMORY_H_ 35 36 /** 37 * Platform-specific inline implementations 38 */ 39 #if NVOS_IS_LIBOS 40 #include "nvport/inline/memory_libos.h" 41 #endif 42 43 // Go straight at the memory or hardware. 44 #define PORT_MEM_RD08(p) (*(p)) 45 #define PORT_MEM_RD16(p) (*(p)) 46 #define PORT_MEM_RD32(p) (*(p)) 47 #define PORT_MEM_RD64(p) (*(p)) 48 #define PORT_MEM_WR08(p, v) (*(p) = (v)) 49 #define PORT_MEM_WR16(p, v) (*(p) = (v)) 50 #define PORT_MEM_WR32(p, v) (*(p) = (v)) 51 #define PORT_MEM_WR64(p, v) (*(p) = (v)) 52 53 /** 54 * @defgroup NVPORT_MEMORY Memory 55 * @brief This module contains memory management related functionality. 56 * 57 * @{ 58 */ 59 60 /** 61 * @brief Single allocation description - forward reference. 62 */ 63 struct PORT_MEM_TRACK_ALLOC_INFO; 64 typedef struct PORT_MEM_TRACK_ALLOC_INFO PORT_MEM_TRACK_ALLOC_INFO; 65 66 67 /** 68 * @name Core Functions 69 * @{ 70 */ 71 72 73 /** 74 * @brief Initializes global Memory tracking structures. 75 * 76 * This function is called by @ref portInitialize. It is available here in case 77 * it is needed to initialize the MEMORY module without initializing all the 78 * others. e.g. for unit tests. 79 */ 80 void portMemInitialize(void); 81 /** 82 * @brief Destroys global Memory tracking structures, and checks for leaks 83 * 84 * This function is called by @ref portShutdown. It is available here in case 85 * it is needed to initialize the MEMORY module without initializing all the 86 * others. e.g. for unit tests. 87 * 88 * @param bForceSilent - Will not print the report, even if 89 * @ref PORT_MEM_TRACK_PRINT_LEVEL isn't PORT_MEM_TRACK_PRINT_LEVEL_SILENT 90 */ 91 void portMemShutdown(NvBool bForceSilent); 92 93 94 /** 95 * @brief Allocates pageable virtual memory of given size. 96 * 97 * Will allocate at least lengthBytes bytes and return a pointer to the 98 * allocated virtual memory. The caller will be able to both read and write 99 * the returned memory via standard pointer accesses. 100 * 101 * The memory is not guaranteed to be initialized before being returned to the 102 * caller. 103 * 104 * An allocation request of size 0 will result in a return value of NULL. 105 * 106 * @par Checked builds only: 107 * Requests of size 0 will breakpoint/assert. 108 * 109 * @par Undefined: 110 * It is possible this function will consume more than lengthBytes of virtual 111 * address space. However behavior is undefined if the caller attempts to read 112 * or write addresses beyond lengthBytes. 113 * 114 * @return Pointer to requested memory, NULL if allocation fails. 115 * 116 * @note Calling this function is identical to calling 117 * @ref PORT_ALLOC ( @ref portMemAllocatorGetGlobalPaged() , lengthBytes) 118 * 119 * @pre Windows: IRQL <= APC_LEVEL 120 * @pre Unix: Non-interrupt context 121 * @note Will not put the thread to sleep. 122 */ 123 NV_FORCERESULTCHECK void *portMemAllocPaged(NvLength lengthBytes); 124 125 /** 126 * @brief Allocates non-paged (i.e. pinned) memory. 127 * 128 * This function is essentially the same to @ref portMemAllocPaged except that 129 * the virtual memory once returned will always be resident in CPU memory. 130 * 131 * @return Pointer to requested memory, NULL if allocation fails. 132 * 133 * @note Calling this function is identical to calling 134 * @ref PORT_ALLOC ( @ref portMemAllocatorGetGlobalNonPaged() , lengthBytes) 135 * 136 * @pre Windows: IRQL <= DISPATCH_LEVEL 137 * @pre Unix: Non-interrupt context 138 * @note Will not put the thread to sleep. 139 */ 140 NV_FORCERESULTCHECK void *portMemAllocNonPaged(NvLength lengthBytes); 141 142 /** 143 * @brief Allocates non-paged (i.e. pinned) memory on the stack or the heap 144 * 145 * USE ONLY FOR MEMORY THAT ALLOCATED AND FREED IN THE SAME FUNCTION! 146 * 147 * This function allocates memory on the stack for platforms with a large stack. 148 * Otherwise it is defined to @ref portMemAllocNonPaged and @ref portMemFree. 149 */ 150 #define portMemExAllocStack(lengthBytes) __builtin_alloca(lengthBytes) 151 #define portMemExAllocStack_SUPPORTED PORT_COMPILER_IS_GCC 152 153 #if portMemExAllocStack_SUPPORTED && NVOS_IS_LIBOS 154 #define portMemAllocStackOrHeap(lengthBytes) portMemExAllocStack(lengthBytes) 155 #define portMemFreeStackOrHeap(pData) 156 #else 157 #define portMemAllocStackOrHeap(size) portMemAllocNonPaged(size) 158 #define portMemFreeStackOrHeap(pData) portMemFree(pData) 159 #endif 160 161 /** 162 * @brief Frees memory allocated by @ref portMemAllocPaged or @ref portMemAllocNonPaged. 163 * 164 * Frees either paged or non-paged virtual memory. The pointer passed in must 165 * have been the exact value returned by the allocation routine. 166 * 167 * Calling with NULL has no effect. 168 * 169 * @par Checked builds only: 170 * Will fill the memory with a pattern to help detect use after free. <br> 171 * Will assert/breakpoint if the memory fenceposts have been corrupted 172 * 173 * @par Undefined: 174 * Freeing the same address multiple times results in undefined behavior. <br> 175 * Accessing memory in the region freed by this function results in undefined 176 * behavior. It may generate a page fault, or if the memory has been 177 * reallocated (or kept around to optimize subsequent allocation requests) then 178 * the access may unexpectedly work. 179 * 180 * @pre Windows: IRQL <= APC_LEVEL (DISPATCH_LEVEL if freeing NonPaged memory) 181 * @pre Unix: Non-interrupt context 182 * @note Will not put the thread to sleep. 183 */ 184 void portMemFree(void *pData); 185 186 /** 187 * @brief Copies data from one address to another. 188 * 189 * Copies srcSize bytes from pSource to pDestination, returning pDestination. 190 * pDestination should be at least destSize bytes, pSource at least srcSize. 191 * destSize should be equal or greater to srcSize. 192 * 193 * If destSize is 0, it is guaranteed to not access either buffer. 194 * 195 * @par Undefined: 196 * Behavior is undefined if memory regions referred to by pSource and 197 * pDestination overlap. 198 * 199 * @par Checked builds only: 200 * Will assert/breakpoint if the regions overlap. <br> 201 * Will assert/breakpoint if destSize < srcSize <br> 202 * Will assert/breakpoint if either pointer is NULL 203 * 204 * @return pDestination on success, NULL if the operation failed. 205 * 206 */ 207 void *portMemCopy(void *pDestination, NvLength destSize, const void *pSource, NvLength srcSize); 208 209 /** 210 * @brief Moves data from one address to another. 211 * 212 * Copies memory from pSource to pDestination, returning pDestination. 213 * pDestination should be at least destSize bytes, pSource at least srcSize. 214 * srcSize should be equal or greater to destSize. 215 * 216 * If destSize is 0, it is guaranteed to not access either buffer. 217 * 218 * Unlike @ref portMemCopy this function allows the regions to overlap. 219 * 220 * @par Checked builds only: 221 * Will assert/breakpoint if destSize < srcSize <br> 222 * Will assert/breakpoint if either pointer is NULL 223 * 224 * @return pDestination on success, NULL if the operation failed. 225 * 226 */ 227 void *portMemMove(void *pDestination, NvLength destSize, const void *pSource, NvLength srcSize); 228 229 /** 230 * @brief Sets given memory to specified value. 231 * 232 * Writes lengthBytes bytes of data starting at pData with value. 233 * The buffer is assumed to have the size of at least lengthBytes. 234 * 235 * if lengthBytes is 0 it is guaranteed to not access pData. 236 * 237 * @return pData 238 */ 239 void *portMemSet(void *pData, NvU8 value, NvLength lengthBytes); 240 241 /** 242 * @brief Sets given memory to specified pattern 243 * 244 * Fills lengthBytes of pData repeating the pPattern pattern. 245 * The pData buffer is assumed to have the size of at least lengthBytes. 246 * The pPattern buffer is assumed to have the size of at least patternBytes. 247 * 248 * If lengthBytes is 0 it is guaranteed to not access pData. 249 * @par Undefined: 250 * Behavior is undefined if patternBytes is zero. <br> 251 * Behavior is undefined if pPattern and pData overlap. 252 * 253 * @return pData 254 */ 255 void *portMemSetPattern(void *pData, NvLength lengthBytes, const NvU8 *pPattern, NvLength patternBytes); 256 257 /** 258 * @brief Compares two memory regions. 259 * 260 * This function does a byte by byte comparison of the 2 memory regions provided. 261 * 262 * It simultaneously scans pData0 and pData1 starting from byte 0 and going 263 * until lengthBytes bytes have been scanned or the bytes in pData0 and pData1 264 * are not equal. 265 * 266 * The return value will be 267 * - 0 if all lengthBytes bytes are equal. 268 * - <0 if pData0 is less than pData1 for the first unequal byte. 269 * - >0 if pData0 is greater than pData1 for the first unequal byte. 270 * 271 * Both buffers are assumed to have the size of at least lengthBytes. 272 * 273 * @par Undefined: 274 * Behavior is undefined if memory regions referred to by pData0 and pData1 275 * overlap. <br> 276 * Behavior is undefined if lengthBytes is 0. 277 * 278 * @par Checked builds only: 279 * The function will return 0 and breakpoint/assert if there is overlap. <br> 280 * The function will return 0 and breakpoint/assert if the length is 0. 281 */ 282 NvS32 portMemCmp(const void *pData0, const void *pData1, NvLength lengthBytes); 283 284 285 typedef struct PORT_MEM_ALLOCATOR PORT_MEM_ALLOCATOR; 286 287 /** 288 * @brief Function signature for PORT_MEM_ALLOCATOR::alloc. 289 * 290 * Basic behavior is similar to @ref portMemAllocPaged. What type of memory 291 * is returned depends on the type of allocator that was created. 292 * 293 * Must be given the same instance of @ref PORT_MEM_ALLOCATOR as that which 294 * contains the calling function pointer. A different copy returned by the 295 * same function is not sufficient. Behavior is undefined if this is not done. 296 */ 297 typedef void *PortMemAllocatorAlloc(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 298 299 /** 300 * @brief Function signature for PORT_MEM_ALLOCATOR::free. 301 * 302 * See @ref portMemFree for details. 303 * 304 * Must be given the same instance of @ref PORT_MEM_ALLOCATOR as that which 305 * contains the calling function pointer. A different copy returned by the 306 * same function is not sufficient. Behavior is undefined if this is not done. 307 * 308 * @par Checked builds only: 309 * Will assert if given a different pointer than the one the memory 310 * was allocated with. 311 */ 312 typedef void PortMemAllocatorFree(PORT_MEM_ALLOCATOR *pAlloc, void *pMemory); 313 314 /** 315 * @brief Function signature for PORT_MEM_ALLOCATOR::release. 316 * 317 * This function is called by @ref portMemAllocatorRelease when the allocator is 318 * released. This is only needed when implementing custom allocators, to be able 319 * to clean up as necessary. 320 */ 321 typedef void PortMemAllocatorRelease(PORT_MEM_ALLOCATOR *pAlloc); 322 323 324 /** 325 * @brief Platform specific allocator implementation. 326 */ 327 typedef struct PORT_MEM_ALLOCATOR_IMPL PORT_MEM_ALLOCATOR_IMPL; 328 329 /** 330 * @brief Opaque structure to hold all memory tracking information. 331 */ 332 typedef struct PORT_MEM_ALLOCATOR_TRACKING PORT_MEM_ALLOCATOR_TRACKING; 333 334 /** 335 * @brief Initializes an allocator tracking structures. 336 * 337 * You only need to call this when creating a custom allocator. The functions 338 * declared in this file call this internally. 339 * 340 * @param pTracking - Pointer to an already allocated tracking structure. 341 */ 342 void portMemInitializeAllocatorTracking(PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_ALLOCATOR_TRACKING *pTracking); 343 344 /** 345 * @brief A set of functions that can be used to manage a specific type of memory. 346 * 347 * The intent of the allocator paradigm is to allow for generic code to be 348 * given an instance of PORT_MEM_ALLOCATOR for use to create memory so it does 349 * not have to embed a policy decision in its implementation. It can also 350 * allow for the implementation of specialized allocators that can be leveraged 351 * through a generic interface. 352 * 353 * Don't call these functions directly, use @ref PORT_ALLOC and @ref PORT_FREE 354 * This is done to provide full tracking support for these calls. 355 */ 356 struct PORT_MEM_ALLOCATOR { 357 /** 358 * @brief see @ref PortMemAllocatorAlloc for documentation 359 */ 360 PortMemAllocatorAlloc *_portAlloc; 361 /** 362 * @brief see @ref PortMemAllocatorFree for documentation 363 */ 364 PortMemAllocatorFree *_portFree; 365 /** 366 * @brief see @ref PortMemAllocatorRelease for documentation 367 */ 368 PortMemAllocatorRelease *_portRelease; 369 /** 370 * @brief Pointer to tracking structure. 371 */ 372 PORT_MEM_ALLOCATOR_TRACKING *pTracking; 373 /** 374 * @brief Pointer to the platform specific implementation. 375 */ 376 PORT_MEM_ALLOCATOR_IMPL *pImpl; 377 }; 378 379 /** 380 * @brief Macro for calling the alloc method of an allocator object. 381 * 382 * Please use this instead of calling the methods directly, to ensure proper 383 * memory tracking in all cases. 384 * 385 * @pre Windows: IRQL <= APC_LEVEL(DISPATCH_LEVEL if allocating NonPaged memory) 386 * @pre Unix: Non-interrupt context 387 * @note Will not put the thread to sleep. 388 */ 389 #define PORT_ALLOC(pAlloc, length) _portMemAllocatorAlloc(pAlloc, length) 390 /** 391 * @brief Macro for calling the free method of an allocator object 392 * 393 * Please use this instead of calling the methods directly, to ensure proper 394 * memory tracking in all cases. 395 * 396 * @pre Windows: IRQL <= APC_LEVEL (DISPATCH_LEVEL if freeing NonPaged memory) 397 * @pre Unix: Non-interrupt context 398 * @note Will not put the thread to sleep. 399 */ 400 #define PORT_FREE(pAlloc, pMem) _portMemAllocatorFree(pAlloc, pMem) 401 402 /** 403 * @brief Creates an allocator for paged memory. 404 * 405 * Returns an allocator instance where @ref PORT_ALLOC will behave 406 * like @ref portMemAllocPaged. Note the memory holding the PORT_MEM_ALLOCATOR 407 * instance may also be paged. 408 * 409 * @return NULL if creation failed. 410 * 411 * @pre Windows: IRQL <= APC_LEVEL 412 * @pre Unix: Non-interrupt context 413 * @note Will not put the thread to sleep. 414 */ 415 NV_FORCERESULTCHECK PORT_MEM_ALLOCATOR *portMemAllocatorCreatePaged(void); 416 417 /** 418 * @brief Creates an allocator for non-paged memory. 419 * 420 * Returns an allocator instance where @ref PORT_ALLOC will 421 * behave like @ref portMemAllocNonPaged. Note the memory holding the 422 * PORT_MEM_ALLOCATOR instance will also be non-paged. 423 * 424 * @return NULL if creation failed. 425 * 426 * @pre Windows: IRQL <= DISPATCH_LEVEL 427 * @pre Unix: Non-interrupt context 428 * @note Will not put the thread to sleep. 429 */ 430 NV_FORCERESULTCHECK PORT_MEM_ALLOCATOR *portMemAllocatorCreateNonPaged(void); 431 432 /** 433 * @brief Creates an allocator over an existing block of memory. 434 * 435 * Adds allocator bookkeeping information to an existing memory block, so that 436 * it can be used with the standard allocator interface. Some of the space of 437 * the preallocated block will be consumed for bookkeeping, so not all of the 438 * memory will be allocatable. 439 * 440 * Use this to create an allocator object on an ISR stack, so memory allocations 441 * can be done at DIQRL. 442 * 443 * @par Implementation details: 444 * The allocator allocates in chunks of 16 bytes, and uses a 2bit-vector to keep 445 * track of free chunks. Thus, the bookkeeping structures for a block of size N 446 * will take about N/64+sizeof(PORT_MEM_ALLOCATOR) bytes. 447 * Use @ref PORT_MEM_PREALLOCATED_BLOCK if you want to specify useful(allocable) 448 * size instead of total size. 449 * 450 * The allocator is only valid while the memory it was created on is valid. 451 * @ref portMemAllocatorRelease must be called on the allocator before the 452 * memory lifecycle ends. 453 * 454 * @return NULL if creation failed. 455 * 456 * @pre Usable at any IRQL/interrupt context 457 * @note Will not put the thread to sleep. 458 * @note This allocator is not thread safe. 459 */ 460 NV_FORCERESULTCHECK PORT_MEM_ALLOCATOR *portMemAllocatorCreateOnExistingBlock(void *pPreallocatedBlock, NvLength blockSizeBytes); 461 462 /** 463 * @brief Extends the given size to fit the required bookkeeping information 464 * 465 * To be used when preallocating blocks that will be used to create an allocator 466 * Consider these two preallocated memory blocks: 467 * ~~~{.c} 468 * NvU8 xxx[1024]; 469 * NvU8 yyy[PORT_MEM_PREALLOCATED_BLOCK(1024)]; 470 * ~~~ 471 * Block @c xxx has a size of 1024, but only ~950 of that can be allocated. 472 * Block @c yyy has a size of ~1100, and exactly 1024 bytes can be allocated. 473 */ 474 #define PORT_MEM_PREALLOCATED_BLOCK(size) \ 475 (size + PORT_MEM_PREALLOCATED_BLOCK_EXTRA_SIZE(size)) 476 477 /** 478 * @brief releases an allocator instance. 479 * 480 * This must be called to release any resources associated with the allocator. 481 * 482 * @par Checked builds only: 483 * Will assert if pAllocator has unfreed allocations 484 * 485 * @par Undefined: 486 * pAllocator must be an instance of PORT_MEM_ALLOCATOR that was provided by one 487 * of the portMemAllocatorCreate* functions. 488 * 489 * These limitations don't apply to allocators created using @ref portMemAllocatorCreateOnExistingBlock and 490 * @ref portMemExAllocatorCreateLockedOnExistingBlock. 491 * 492 * @pre Windows: IRQL <= APC_LEVEL 493 * @pre Unix: Non-interrupt context 494 * @note Will not put the thread to sleep. 495 */ 496 void portMemAllocatorRelease(PORT_MEM_ALLOCATOR *pAllocator); 497 498 /** 499 * @brief Returns the pointer to the global nonpaged allocator. 500 * 501 * This allocator is always initialized does not need to be released. 502 * 503 * Allocations performed using this allocator are identical to the ones done 504 * by @ref portMemAllocNonPaged 505 */ 506 PORT_MEM_ALLOCATOR *portMemAllocatorGetGlobalNonPaged(void); 507 /** 508 * @brief Returns the pointer to the global paged allocator. 509 * 510 * This allocator is always initialized does not need to be released. 511 * 512 * Allocations performed using this allocator are identical to the ones done 513 * by @ref portMemAllocPaged 514 */ 515 PORT_MEM_ALLOCATOR *portMemAllocatorGetGlobalPaged(void); 516 /** 517 * @brief Prints the memory details gathered by whatever tracking mechanism is 518 * enabled. If pTracking is NULL, aggregate tracking information from all 519 * allocators will be printed. 520 * 521 * @note Printing is done using portDbgPrintf, which prints regardless of 522 * build type and debug levels. 523 */ 524 void portMemPrintTrackingInfo(const PORT_MEM_ALLOCATOR_TRACKING *pTracking); 525 /** 526 * @brief Calls @ref portMemPrintTrackingInfo for all current allocator trackers. 527 */ 528 void portMemPrintAllTrackingInfo(void); 529 530 // @} End core functions 531 532 533 /** 534 * @name Extended Functions 535 * @{ 536 */ 537 538 /** 539 * @brief Returns true if it is safe to allocate paged memory. 540 */ 541 NvBool portMemExSafeForPagedAlloc(void); 542 #define portMemExSafeForPagedAlloc_SUPPORTED PORT_IS_KERNEL_BUILD 543 544 /** 545 * @brief Returns true if it is safe to allocate non-paged memory. 546 */ 547 NvBool portMemExSafeForNonPagedAlloc(void); 548 #define portMemExSafeForNonPagedAlloc_SUPPORTED PORT_IS_KERNEL_BUILD 549 550 /** 551 * @brief Public allocator tracking information 552 */ 553 typedef struct PORT_MEM_TRACK_ALLOCATOR_STATS 554 { 555 /** @brief Total number of allocations */ 556 NvU32 numAllocations; 557 /** @brief Total allocated bytes, including all staging */ 558 NvLength allocatedSize; 559 /** @brief Useful size of allocations - What was actually requested */ 560 NvLength usefulSize; 561 /** @brief Extra size allocated for tracking/debugging purposes */ 562 NvLength metaSize; 563 } PORT_MEM_TRACK_ALLOCATOR_STATS; 564 565 /** 566 * @brief Returns the statistics of currently active allocations for the given 567 * allocator. 568 * 569 * If pAllocator is NULL, it returns stats for all allocators, as well as the 570 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 571 */ 572 NV_STATUS portMemExTrackingGetActiveStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 573 574 /** 575 * @brief Returns the statistics of currently active allocations made with the 576 * given gfid. 577 * 578 * If the corresponding pTracking is not found, it returns 579 * NV_ERR_OBJECT_NOT_FOUND 580 */ 581 NV_STATUS portMemExTrackingGetGfidActiveStats( 582 NvU32 gfid, 583 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 584 ); 585 586 /** 587 * @brief Returns the statistics of all allocations made with the given 588 * allocator since it was created. 589 * 590 * If pAllocator is NULL, it returns stats for all allocators, as well as the 591 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 592 */ 593 NV_STATUS portMemExTrackingGetTotalStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 594 595 /** 596 * @brief Returns the statistics of all allocations made with the given 597 * gfid. 598 * 599 * If the corresponding pTracking is not found, it returns 600 * NV_ERR_OBJECT_NOT_FOUND 601 */ 602 NV_STATUS portMemExTrackingGetGfidTotalStats( 603 NvU32 gfid, 604 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 605 ); 606 607 /** 608 * @brief Returns the statistics of peak allocations made with the given 609 * allocator since it was created. 610 * 611 * Peak data reports the high-water mark based on the maximum size (the peak 612 * allocations doesn't report the largest number of allocations, it reports 613 * the number of allocations at the time the peak size was achieved). This is 614 * done so that the other peak stats, which are derived from peak size and 615 * peak allocations, are consistent with each other. 616 * 617 * If pAllocator is NULL, it returns stats for all allocators, as well as the 618 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 619 */ 620 NV_STATUS portMemExTrackingGetPeakStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 621 622 /** 623 * @brief Returns the statistics of peak allocations made with the given 624 * gfid since it was created. 625 * 626 * Peak data reports the high-water mark based on the maximum size (the peak 627 * allocations doesn't report the largest number of allocations, it reports 628 * the number of allocations at the time the peak size was achieved). This is 629 * done so that the other peak stats, which are derived from peak size and 630 * peak allocations, are consistent with each other. 631 * 632 * If the corresponding pTracking is not found, it returns 633 * NV_ERR_OBJECT_NOT_FOUND 634 */ 635 NV_STATUS portMemExTrackingGetGfidPeakStats( 636 NvU32 gfid, 637 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 638 ); 639 640 /** 641 * @brief Cycles through the tracking infos for allocations by pAllocator 642 * If pAllocator is NULL, it will cycle through all allocations. 643 * 644 * @param [out] pInfo The info will be written to this buffer. 645 * @param [in, out] pIterator 646 * Should point to NULL the first time it is called. 647 * Every next call should pass the value returned by previous. 648 * To reset the loop, set the iterator to NULL. 649 * Upon writing the last range, the iterator will be set to NULL. 650 * The iterator is only valid until the next alloc/free from this allocator. 651 * There is no need to release the iterator in any way. 652 * 653 * @return NV_ERR_OBJECT_NOT_FOUND if no allocations exist. 654 */ 655 NV_STATUS portMemExTrackingGetNext(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOC_INFO *pInfo, void **pIterator); 656 657 /** 658 * @brief Gets the total size of the underlying heap, in bytes. 659 */ 660 NvLength portMemExTrackingGetHeapSize(void); 661 662 /** 663 * @brief Gets the usable size in bytes (sans metadata/padding) of the given allocation. 664 */ 665 NvLength portMemExTrackingGetAllocUsableSize(void *pMem); 666 667 /** 668 * @brief Copies from user memory to kernel memory. 669 * 670 * When accepting data as input from user space it is necessary to take 671 * additional precautions to access it safely and securely. This means copy 672 * the user data into a kernel buffer and then using that kernel buffer for all 673 * needed accesses. 674 * 675 * The function will fail if pUser is an invalid user space pointer or if the 676 * memory it refers to is less than length bytes long. A valid kernel pointer 677 * is interpreted as an invalid user pointer. 678 * @par Checked builds only: 679 * Will trigger a breakpoint if pUser is invalid userspace pointer 680 * 681 * The function will fail if pKernel is NULL. 682 * 683 * The function will fail if lengthBytes is 0. 684 * 685 * @return 686 * - NV_OK if successful 687 * - NV_ERR_INVALID_POINTER if pUser is invalid or pKernel is NULL 688 * - NV_ERR_INVALID_ARGUMENT if lengthBytes is 0 689 */ 690 NV_STATUS portMemExCopyFromUser(const NvP64 pUser, void *pKernel, NvLength lengthBytes); 691 #define portMemExCopyFromUser_SUPPORTED PORT_IS_KERNEL_BUILD 692 693 694 /** 695 * @brief Copies from kernel memory to user memory. 696 * 697 * This is the reverse of @ref portMemExCopyFromUser. The copy in this case is 698 * from pKernel to pUser. 699 * 700 * See @ref portMemExCopyFromUser for more details. 701 * 702 */ 703 NV_STATUS portMemExCopyToUser(const void *pKernel, NvP64 pUser, NvLength lengthBytes); 704 #define portMemExCopyToUser_SUPPORTED PORT_IS_KERNEL_BUILD 705 706 /** 707 * @brief Returns the size (in bytes) of a single memory page. 708 */ 709 NvLength portMemExGetPageSize(void); 710 #define portMemExGetPageSize_SUPPORTED PORT_IS_KERNEL_BUILD 711 712 /** 713 * @brief Opaque container holding an allocation of physical system memory. 714 */ 715 typedef struct PORT_PHYSICAL_MEMDESC PORT_PHYSICAL_MEMDESC; 716 717 /** 718 * @brief Creates a handle used to manage and manipulate a physical memory 719 * allocation. 720 * 721 * @param pAllocator the allocator to use the create the allocation's tracking 722 * structures. This allocator is *not* used to allocate physical memory. 723 * 724 * @return NULL if the allocation failed. 725 */ 726 PORT_PHYSICAL_MEMDESC *portMemExPhysicalDescCreate(PORT_MEM_ALLOCATOR *pAllocator); 727 #define portMemExPhysicalDescCreate_SUPPORTED PORT_IS_KERNEL_BUILD 728 729 /** 730 * @brief Types of caching for physical memory mappings. 731 * 732 * In case a target architecture does not support a specific caching mode, 733 * the mapping call will fail. 734 * Specifying PORT_MEM_ANYCACHE lets the implementation pick a caching mode that 735 * is present on the target architecture. This way the mapping will not fail. 736 */ 737 typedef enum 738 { 739 PORT_MEM_UNCACHED, 740 PORT_MEM_CACHED, 741 PORT_MEM_WRITECOMBINED, 742 PORT_MEM_ANYCACHE 743 } PortMemCacheMode; 744 745 /** 746 * @brief Types of access protections for physical memory mappings. 747 */ 748 typedef enum 749 { 750 PORT_MEM_PROT_NO_ACCESS = 0, 751 PORT_MEM_PROT_READ = 1, 752 PORT_MEM_PROT_WRITE = 2, 753 PORT_MEM_PROT_READ_WRITE = 3, 754 PORT_MEM_PROT_EXEC = 4, 755 PORT_MEM_PROT_READ_EXEC = 5, 756 PORT_MEM_PROT_WRITE_EXEC = 6, 757 PORT_MEM_PROT_READ_WRITE_EXEC = 7 758 } PortMemProtectMode; 759 760 /** 761 * @brief Populates a physical memory descriptor with backing pages. 762 * 763 * Populates a descriptor with physical pages. Pages will be zeroed. 764 */ 765 NV_STATUS portMemExPhysicalDescPopulate(PORT_PHYSICAL_MEMDESC *pPmd, NvLength sizeBytes, NvBool bContiguous); 766 #define portMemExPhysicalDescPopulate_SUPPORTED PORT_IS_KERNEL_BUILD 767 768 769 /** 770 * @brief allocates a PMD and populates it with memory 771 * 772 * This is a combination of @ref portMemExPhysicalDescCreate and @ref 773 * portMemExPhysicalDescPopulate. It should be the preferred method to allocate 774 * physical memory when it is possible to do it as a single step. Not only 775 * does the caller require less code and error handling but it allows the 776 * implementation the option to combine the tracking data into fewer 777 * allocations since it knows the size up front. 778 * 779 * @param [out] ppPmd - Pointer to the allocated PMD. 780 * @param pAllocator - Allocator to use when allocating the PMD 781 */ 782 NV_STATUS portMemExPhysicalDescCreateAndPopulate(PORT_MEM_ALLOCATOR *pAllocator, 783 PORT_PHYSICAL_MEMDESC **ppPmd, NvLength sizeBytes, NvBool bContiguous); 784 #define portMemExPhysicalDescCreateAndPopulate_SUPPORTED PORT_IS_KERNEL_BUILD 785 786 787 /** 788 * @brief Adds a contiguous memory range to the physical memory descriptor 789 * 790 * To describe a non-contiguous memory range, call this function once for every 791 * contiguous range. Range order will be determined by function call order, 792 * not the range addresses. 793 */ 794 NV_STATUS portMemExPhysicalDescribeRange(PORT_PHYSICAL_MEMDESC *pPmd, NvU64 start, NvLength length); 795 #define portMemExPhysicalDescribeRange_SUPPORTED PORT_IS_KERNEL_BUILD 796 797 /** 798 * @brief Hands back the next contiguous memory range in the memory descriptor 799 * 800 * @param [out] pStart - Physical address of the range 801 * @param [out] pLength - Length of the range 802 * @param [in, out] pIterator 803 * Should point to NULL the first time it is called. 804 * Every next call should pass the value returned by previous. 805 * To reset the loop, set the iterator to NULL. 806 * Upon writing the last range, the iterator will be set to NULL. 807 * The iterator is valid until pPmd is destroyed. 808 * There is no need to release the iterator in any way. 809 * 810 * @return NV_ERR_OBJECT_NOT_FOUND if no ranges exist. 811 */ 812 NV_STATUS portMemExPhysicalGetNextRange(PORT_PHYSICAL_MEMDESC *pPmd, 813 NvU64 *pStart, NvLength *pLength, void **pIterator); 814 #define portMemExPhysicalGetNextRange_SUPPORTED PORT_IS_KERNEL_BUILD 815 816 /** 817 * @brief Frees the memory descriptor and all tracking data. The descriptor must 818 * have been allocated with @ref portMemExPhysicalDescCreate or 819 * @ref portMemExPhysicalDescCreateAndPopulate 820 * 821 * Freed memory is not automatically unmapped. 822 * 823 * It is guaranteed that after memory has been freed, the original data can no 824 * longer be read in any way. 825 * @par Undefined: 826 * Accessing a mapping that has been freed results in undefined behavior. 827 */ 828 void portMemExPhysicalDescFree(PORT_PHYSICAL_MEMDESC *pPmd); 829 #define portMemExPhysicalDescFree_SUPPORTED PORT_IS_KERNEL_BUILD 830 831 832 /** 833 * @brief Frees physical memory allocated with @ref portMemExPhysicalDescPopulate 834 */ 835 void portMemExPhysicalFree(PORT_PHYSICAL_MEMDESC *pPmd); 836 #define portMemExPhysicalFree_SUPPORTED PORT_IS_KERNEL_BUILD 837 838 839 /** 840 * @brief Maps a region of a @ref PORT_PHYSICAL_MEMDESC 841 * 842 * @param [out] ppMapping - Virtual address where the physical memory is mapped 843 * @param offset - Offset of the physical memory where the region starts. 844 * The region must start on a page boundary. 845 * @param length - Length of the physical memory region. 846 * Needs to be a multiple of page size. 847 * @param protect - Mapping protections 848 * @param cacheMode - Mapping cache mode. 849 * Only PORT_MEM_ANYCACHE is guaranteed to be supported. 850 * 851 * @return NV_ERR_NOT_SUPPORTED if the specified cache mode is not supported by 852 * the current architecture. 853 */ 854 NV_STATUS portMemExPhysicalMap(PORT_PHYSICAL_MEMDESC *pPmd, 855 void **ppMapping, NvU64 offset, NvU64 length, 856 PortMemProtectMode protect, PortMemCacheMode cacheMode); 857 #define portMemExPhysicalMap_SUPPORTED PORT_IS_KERNEL_BUILD 858 859 /** 860 * @brief Unmaps a region created with @ref portMemExPhysicalMap. 861 * 862 * @par Undefined: 863 * Accessing an unmapped memory is undefined, but it is guaranteed that the 864 * actual data can't be read/overwritten. 865 */ 866 NV_STATUS portMemExPhysicalUnmap(PORT_PHYSICAL_MEMDESC *pPmd, void *pMapping); 867 #define portMemExPhysicalUnmap_SUPPORTED PORT_IS_KERNEL_BUILD 868 869 /** 870 * @brief Creates a thread safe allocator over an existing block of memory. 871 * 872 * @note See @ref portMemAllocatorCreateOnExistingBlock for other limitations. 873 * @note User should initialize @p pSpinlock and destroy it after it 874 * has finished using this allocator. 875 */ 876 PORT_MEM_ALLOCATOR *portMemExAllocatorCreateLockedOnExistingBlock(void *pPreallocatedBlock, NvLength blockSizeBytes, void *pSpinlock); 877 #define portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED \ 878 (PORT_IS_MODULE_SUPPORTED(sync)) 879 880 881 /** 882 * @brief Maps the given physical address range to nonpaged system space. 883 * 884 * @param[in] start Specifies the starting physical address of the I/O 885 * range to be mapped. 886 * @param[in] byteSize Specifies the number of bytes to be mapped. 887 * 888 * @return The base virtual address that maps the base physical address for 889 * the range 890 */ 891 void *portMemExMapIOSpace(NvU64 start, NvU64 byteSize); 892 #define portMemExMapIOSpace_SUPPORTED (NVOS_IS_WINDOWS && !PORT_IS_MODS) 893 894 /** 895 * @brief Unmaps a specified range of physical addresses previously mapped by 896 * portMapIOSpace 897 * 898 * @param[in] addr Pointer to the base virtual address to which the 899 * physical pages were mapped. 900 * @param[in] byteSize Specifies the number of bytes that were mapped. 901 */ 902 void portMemExUnmapIOSpace(void *addr, NvU64 byteSize); 903 #define portMemExUnmapIOSpace_SUPPORTED (NVOS_IS_WINDOWS && !PORT_IS_MODS) 904 905 // @} End extended functions 906 907 908 /** 909 * @note Memory tracking is controlled through the following compile-time flags. 910 * The PORT_MEM_TRACK_USE_* constants should be defined to 0 or 1. 911 * If nothing is defined, the default values are assigned here. 912 */ 913 #if !defined(PORT_MEM_TRACK_USE_COUNTER) 914 /** 915 * @brief Use allocations counter for all allocators 916 * 917 * Allocation counter is lightweight and can detect if a leak is present. 918 * Default is always on. 919 */ 920 #define PORT_MEM_TRACK_USE_COUNTER 1 921 #endif 922 #if !defined(PORT_MEM_TRACK_USE_FENCEPOSTS) 923 /** 924 * @brief Use fenceposts around all allocated blocks 925 * 926 * Fenceposts can detect out of bounds writes and improper free calls 927 * Default is on for checked builds (where it will assert if an error occurs) 928 */ 929 #define PORT_MEM_TRACK_USE_FENCEPOSTS PORT_IS_CHECKED_BUILD 930 #endif 931 #if !defined(PORT_MEM_TRACK_USE_ALLOCLIST) 932 /** 933 * @brief Keep a list of all allocations. 934 * 935 * Allocation lists can give more details about detected leaks, and allow 936 * cycling through all allocations. 937 * Default is off. 938 * @todo Perhaps enable for checked builds? 939 */ 940 #define PORT_MEM_TRACK_USE_ALLOCLIST 0 941 #endif 942 #if !defined(PORT_MEM_TRACK_USE_CALLERINFO) 943 /** 944 * @brief Track file:line information for all allocations 945 * 946 * On release builds the filename hash is passed instead of the string. This 947 * requires NvLog to be enabled. 948 * Default is off. 949 */ 950 #define PORT_MEM_TRACK_USE_CALLERINFO 0 951 #endif 952 /** 953 * @brief Track instruction pointer instead of function/file/line information 954 * for all allocations 955 * 956 * Has no effect unless PORT_MEM_TRACK_USE_CALLERINFO is also set. 957 */ 958 #if !defined(PORT_MEM_TRACK_USE_CALLERINFO_IP) 959 #if NVOS_IS_LIBOS 960 #define PORT_MEM_TRACK_USE_CALLERINFO_IP 1 961 #else 962 #define PORT_MEM_TRACK_USE_CALLERINFO_IP 0 963 #endif 964 #endif 965 #if !defined(PORT_MEM_TRACK_USE_LOGGING) 966 /** 967 * @brief Log all alloc and free calls to a binary NvLog buffer 968 * Requires NvLog to be enabled. 969 * 970 * Default is off. 971 */ 972 #define PORT_MEM_TRACK_USE_LOGGING 0 973 #endif 974 #if !defined(PORT_MEM_TRACK_USE_LIMIT) 975 /** 976 * @brief Track and enforce a heap memory usage limit on processes 977 * running in GSP-RM. 978 * 979 * Default is on in GSP-RM only. 980 */ 981 #ifndef GSP_PLUGIN_BUILD 982 #define PORT_MEM_TRACK_USE_LIMIT (NVOS_IS_LIBOS) 983 #else 984 #define PORT_MEM_TRACK_USE_LIMIT 0 985 #endif 986 #endif // !defined(PORT_MEM_TRACK_USE_LIMIT) 987 988 // Memory tracking header can redefine some functions declared here. 989 #include "nvport/inline/memory_tracking.h" 990 991 /** @brief Nothing is printed unless @ref portMemPrintTrackingInfo is called */ 992 #define PORT_MEM_TRACK_PRINT_LEVEL_SILENT 0 993 /** @brief Print when an error occurs and at shutdown */ 994 #define PORT_MEM_TRACK_PRINT_LEVEL_BASIC 1 995 /** @brief Print at every alloc and free, and at any abnormal situation */ 996 #define PORT_MEM_TRACK_PRINT_LEVEL_VERBOSE 2 997 998 #if !defined(PORT_MEM_TRACK_PRINT_LEVEL) 999 #if PORT_IS_CHECKED_BUILD || PORT_MEM_TRACK_ALLOC_SIZE 1000 #define PORT_MEM_TRACK_PRINT_LEVEL PORT_MEM_TRACK_PRINT_LEVEL_BASIC 1001 #else 1002 #define PORT_MEM_TRACK_PRINT_LEVEL PORT_MEM_TRACK_PRINT_LEVEL_SILENT 1003 #endif // PORT_IS_CHECKED_BUILD 1004 #endif // !defined(PORT_MEM_TRACK_PRINT_LEVEL) 1005 1006 /** 1007 * @brief Single allocation description. 1008 * 1009 * Must be defined after memory_tracking.h is included for PORT_MEM_CALLERINFO. 1010 */ 1011 struct PORT_MEM_TRACK_ALLOC_INFO 1012 { 1013 #if PORT_MEM_TRACK_USE_CALLERINFO 1014 /** 1015 * @brief Function / file / line or instruction pointer. 1016 */ 1017 PORT_MEM_CALLERINFO callerInfo; 1018 #endif 1019 /** 1020 * @brief pointer to the allocated memory block. 1021 */ 1022 void *pMemory; 1023 /** 1024 * @brief Size of the allocated memory block 1025 */ 1026 NvLength size; 1027 /** 1028 * @brief Pointer to the allocator that allocated the memory. 1029 * If the memory was allocated globally, this will be NULL 1030 */ 1031 PORT_MEM_ALLOCATOR *pAllocator; 1032 /** 1033 * @brief Timestamp of the allocation. Will be 0 if it wasn't logged. 1034 */ 1035 NvU64 timestamp; 1036 }; 1037 1038 /** 1039 * @} 1040 */ 1041 1042 #endif // _NVPORT_MEMORY_H_ 1043