1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /** 25 * @file 26 * @brief Memory module public interface 27 */ 28 29 #ifndef _NVPORT_H_ 30 #error "This file cannot be included directly. Include nvport.h instead." 31 #endif 32 33 #ifndef _NVPORT_MEMORY_H_ 34 #define _NVPORT_MEMORY_H_ 35 36 /** 37 * Platform-specific inline implementations 38 */ 39 #if NVOS_IS_LIBOS 40 #include "nvport/inline/memory_libos.h" 41 #endif 42 43 /** 44 * @defgroup NVPORT_MEMORY Memory 45 * @brief This module contains memory management related functionality. 46 * 47 * @{ 48 */ 49 50 /** 51 * @brief Single allocation description - forward reference. 52 */ 53 struct PORT_MEM_TRACK_ALLOC_INFO; 54 typedef struct PORT_MEM_TRACK_ALLOC_INFO PORT_MEM_TRACK_ALLOC_INFO; 55 56 57 /** 58 * @name Core Functions 59 * @{ 60 */ 61 62 63 /** 64 * @brief Initializes global Memory tracking structures. 65 * 66 * This function is called by @ref portInitialize. It is available here in case 67 * it is needed to initialize the MEMORY module without initializing all the 68 * others. e.g. for unit tests. 69 */ 70 void portMemInitialize(void); 71 /** 72 * @brief Destroys global Memory tracking structures, and checks for leaks 73 * 74 * This function is called by @ref portShutdown. It is available here in case 75 * it is needed to initialize the MEMORY module without initializing all the 76 * others. e.g. for unit tests. 77 * 78 * @param bForceSilent - Will not print the report, even if 79 * @ref PORT_MEM_TRACK_PRINT_LEVEL isn't PORT_MEM_TRACK_PRINT_LEVEL_SILENT 80 */ 81 void portMemShutdown(NvBool bForceSilent); 82 83 84 /** 85 * @brief Allocates pageable virtual memory of given size. 86 * 87 * Will allocate at least lengthBytes bytes and return a pointer to the 88 * allocated virtual memory. The caller will be able to both read and write 89 * the returned memory via standard pointer accesses. 90 * 91 * The memory is not guaranteed to be initialized before being returned to the 92 * caller. 93 * 94 * An allocation request of size 0 will result in a return value of NULL. 95 * 96 * @par Checked builds only: 97 * Requests of size 0 will breakpoint/assert. 98 * 99 * @par Undefined: 100 * It is possible this function will consume more than lengthBytes of virtual 101 * address space. However behavior is undefined if the caller attempts to read 102 * or write addresses beyond lengthBytes. 103 * 104 * @return Pointer to requested memory, NULL if allocation fails. 105 * 106 * @note Calling this function is identical to calling 107 * @ref PORT_ALLOC ( @ref portMemAllocatorGetGlobalPaged() , lengthBytes) 108 * 109 * @pre Windows: IRQL <= APC_LEVEL 110 * @pre Unix: Non-interrupt context 111 * @note Will not put the thread to sleep. 112 */ 113 void *portMemAllocPaged(NvLength lengthBytes); 114 115 /** 116 * @brief Allocates non-paged (i.e. pinned) memory. 117 * 118 * This function is essentially the same to @ref portMemAllocPaged except that 119 * the virtual memory once returned will always be resident in CPU memory. 120 * 121 * @return Pointer to requested memory, NULL if allocation fails. 122 * 123 * @note Calling this function is identical to calling 124 * @ref PORT_ALLOC ( @ref portMemAllocatorGetGlobalNonPaged() , lengthBytes) 125 * 126 * @pre Windows: IRQL <= DISPATCH_LEVEL 127 * @pre Unix: Non-interrupt context 128 * @note Will not put the thread to sleep. 129 */ 130 void *portMemAllocNonPaged(NvLength lengthBytes); 131 132 /** 133 * @brief Allocates non-paged (i.e. pinned) memory on the stack or the heap 134 * 135 * USE ONLY FOR MEMORY THAT ALLOCATED AND FREED IN THE SAME FUNCTION! 136 * 137 * This function allocates memory on the stack for platforms with a large stack. 138 * Otherwise it is defined to @ref portMemAllocNonPaged and @ref portMemFree. 139 */ 140 #define portMemExAllocStack(lengthBytes) __builtin_alloca(lengthBytes) 141 #define portMemExAllocStack_SUPPORTED PORT_COMPILER_IS_GCC 142 143 #if portMemExAllocStack_SUPPORTED && NVOS_IS_LIBOS 144 #define portMemAllocStackOrHeap(lengthBytes) portMemExAllocStack(lengthBytes) 145 #define portMemFreeStackOrHeap(pData) 146 #else 147 #define portMemAllocStackOrHeap(size) portMemAllocNonPaged(size) 148 #define portMemFreeStackOrHeap(pData) portMemFree(pData) 149 #endif 150 151 /** 152 * @brief Frees memory allocated by @ref portMemAllocPaged or @ref portMemAllocNonPaged. 153 * 154 * Frees either paged or non-paged virtual memory. The pointer passed in must 155 * have been the exact value returned by the allocation routine. 156 * 157 * Calling with NULL has no effect. 158 * 159 * @par Checked builds only: 160 * Will fill the memory with a pattern to help detect use after free. <br> 161 * Will assert/breakpoint if the memory fenceposts have been corrupted 162 * 163 * @par Undefined: 164 * Freeing the same address multiple times results in undefined behavior. <br> 165 * Accessing memory in the region freed by this function results in undefined 166 * behavior. It may generate a page fault, or if the memory has been 167 * reallocated (or kept around to optimize subsequent allocation requests) then 168 * the access may unexpectedly work. 169 * 170 * @pre Windows: IRQL <= APC_LEVEL (DISPATCH_LEVEL if freeing NonPaged memory) 171 * @pre Unix: Non-interrupt context 172 * @note Will not put the thread to sleep. 173 */ 174 void portMemFree(void *pData); 175 176 /** 177 * @brief Copies data from one address to another. 178 * 179 * Copies srcSize bytes from pSource to pDestination, returning pDestination. 180 * pDestination should be at least destSize bytes, pSource at least srcSize. 181 * destSize should be equal or greater to srcSize. 182 * 183 * If destSize is 0, it is guaranteed to not access either buffer. 184 * 185 * @par Undefined: 186 * Behavior is undefined if memory regions referred to by pSource and 187 * pDestination overlap. 188 * 189 * @par Checked builds only: 190 * Will assert/breakpoint if the regions overlap. <br> 191 * Will assert/breakpoint if destSize < srcSize <br> 192 * Will assert/breakpoint if either pointer is NULL 193 * 194 * @return pDestination on success, NULL if the operation failed. 195 * 196 */ 197 void *portMemCopy(void *pDestination, NvLength destSize, const void *pSource, NvLength srcSize); 198 199 /** 200 * @brief Moves data from one address to another. 201 * 202 * Copies memory from pSource to pDestination, returning pDestination. 203 * pDestination should be at least destSize bytes, pSource at least srcSize. 204 * srcSize should be equal or greater to destSize. 205 * 206 * If destSize is 0, it is guaranteed to not access either buffer. 207 * 208 * Unlike @ref portMemCopy this function allows the regions to overlap. 209 * 210 * @par Checked builds only: 211 * Will assert/breakpoint if destSize < srcSize <br> 212 * Will assert/breakpoint if either pointer is NULL 213 * 214 * @return pDestination on success, NULL if the operation failed. 215 * 216 */ 217 void *portMemMove(void *pDestination, NvLength destSize, const void *pSource, NvLength srcSize); 218 219 /** 220 * @brief Sets given memory to specified value. 221 * 222 * Writes lengthBytes bytes of data starting at pData with value. 223 * The buffer is assumed to have the size of at least lengthBytes. 224 * 225 * if lengthBytes is 0 it is guaranteed to not access pData. 226 * 227 * @return pData 228 */ 229 void *portMemSet(void *pData, NvU8 value, NvLength lengthBytes); 230 231 /** 232 * @brief Sets given memory to specified pattern 233 * 234 * Fills lengthBytes of pData repeating the pPattern pattern. 235 * The pData buffer is assumed to have the size of at least lengthBytes. 236 * The pPattern buffer is assumed to have the size of at least patternBytes. 237 * 238 * If lengthBytes is 0 it is guaranteed to not access pData. 239 * @par Undefined: 240 * Behavior is undefined if patternBytes is zero. <br> 241 * Behavior is undefined if pPattern and pData overlap. 242 * 243 * @return pData 244 */ 245 void *portMemSetPattern(void *pData, NvLength lengthBytes, const NvU8 *pPattern, NvLength patternBytes); 246 247 /** 248 * @brief Compares two memory regions. 249 * 250 * This function does a byte by byte comparison of the 2 memory regions provided. 251 * 252 * It simultaneously scans pData0 and pData1 starting from byte 0 and going 253 * until lengthBytes bytes have been scanned or the bytes in pData0 and pData1 254 * are not equal. 255 * 256 * The return value will be 257 * - 0 if all lengthBytes bytes are equal. 258 * - <0 if pData0 is less than pData1 for the first unequal byte. 259 * - >0 if pData0 is greater than pData1 for the first unequal byte. 260 * 261 * Both buffers are assumed to have the size of at least lengthBytes. 262 * 263 * @par Undefined: 264 * Behavior is undefined if memory regions referred to by pData0 and pData1 265 * overlap. <br> 266 * Behavior is undefined if lengthBytes is 0. 267 * 268 * @par Checked builds only: 269 * The function will return 0 and breakpoint/assert if there is overlap. <br> 270 * The function will return 0 and breakpoint/assert if the length is 0. 271 */ 272 NvS32 portMemCmp(const void *pData0, const void *pData1, NvLength lengthBytes); 273 274 275 typedef struct PORT_MEM_ALLOCATOR PORT_MEM_ALLOCATOR; 276 277 /** 278 * @brief Function signature for PORT_MEM_ALLOCATOR::alloc. 279 * 280 * Basic behavior is similar to @ref portMemAllocPaged. What type of memory 281 * is returned depends on the type of allocator that was created. 282 * 283 * Must be given the same instance of @ref PORT_MEM_ALLOCATOR as that which 284 * contains the calling function pointer. A different copy returned by the 285 * same function is not sufficient. Behavior is undefined if this is not done. 286 */ 287 typedef void *PortMemAllocatorAlloc(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 288 289 /** 290 * @brief Function signature for PORT_MEM_ALLOCATOR::free. 291 * 292 * See @ref portMemFree for details. 293 * 294 * Must be given the same instance of @ref PORT_MEM_ALLOCATOR as that which 295 * contains the calling function pointer. A different copy returned by the 296 * same function is not sufficient. Behavior is undefined if this is not done. 297 * 298 * @par Checked builds only: 299 * Will assert if given a different pointer than the one the memory 300 * was allocated with. 301 */ 302 typedef void PortMemAllocatorFree(PORT_MEM_ALLOCATOR *pAlloc, void *pMemory); 303 304 /** 305 * @brief Function signature for PORT_MEM_ALLOCATOR::release. 306 * 307 * This function is called by @ref portMemAllocatorRelease when the allocator is 308 * released. This is only needed when implementing custom allocators, to be able 309 * to clean up as necessary. 310 */ 311 typedef void PortMemAllocatorRelease(PORT_MEM_ALLOCATOR *pAlloc); 312 313 314 /** 315 * @brief Platform specific allocator implementation. 316 */ 317 typedef struct PORT_MEM_ALLOCATOR_IMPL PORT_MEM_ALLOCATOR_IMPL; 318 319 /** 320 * @brief Opaque structure to hold all memory tracking information. 321 */ 322 typedef struct PORT_MEM_ALLOCATOR_TRACKING PORT_MEM_ALLOCATOR_TRACKING; 323 324 /** 325 * @brief Initializes an allocator tracking structures. 326 * 327 * You only need to call this when creating a custom allocator. The functions 328 * declared in this file call this internally. 329 * 330 * @param pTracking - Pointer to an already allocated tracking structure. 331 */ 332 void portMemInitializeAllocatorTracking(PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_ALLOCATOR_TRACKING *pTracking); 333 334 /** 335 * @brief A set of functions that can be used to manage a specific type of memory. 336 * 337 * The intent of the allocator paradigm is to allow for generic code to be 338 * given an instance of PORT_MEM_ALLOCATOR for use to create memory so it does 339 * not have to embed a policy decision in its implementation. It can also 340 * allow for the implementation of specialized allocators that can be leveraged 341 * through a generic interface. 342 * 343 * Don't call these functions directly, use @ref PORT_ALLOC and @ref PORT_FREE 344 * This is done to provide full tracking support for these calls. 345 */ 346 struct PORT_MEM_ALLOCATOR { 347 /** 348 * @brief see @ref PortMemAllocatorAlloc for documentation 349 */ 350 PortMemAllocatorAlloc *_portAlloc; 351 /** 352 * @brief see @ref PortMemAllocatorFree for documentation 353 */ 354 PortMemAllocatorFree *_portFree; 355 /** 356 * @brief see @ref PortMemAllocatorRelease for documentation 357 */ 358 PortMemAllocatorRelease *_portRelease; 359 /** 360 * @brief Pointer to tracking structure. 361 */ 362 PORT_MEM_ALLOCATOR_TRACKING *pTracking; 363 /** 364 * @brief Pointer to the platform specific implementation. 365 */ 366 PORT_MEM_ALLOCATOR_IMPL *pImpl; 367 }; 368 369 /** 370 * @brief Macro for calling the alloc method of an allocator object. 371 * 372 * Please use this instead of calling the methods directly, to ensure proper 373 * memory tracking in all cases. 374 * 375 * @pre Windows: IRQL <= APC_LEVEL(DISPATCH_LEVEL if allocating NonPaged memory) 376 * @pre Unix: Non-interrupt context 377 * @note Will not put the thread to sleep. 378 */ 379 #define PORT_ALLOC(pAlloc, length) _portMemAllocatorAlloc(pAlloc, length) 380 /** 381 * @brief Macro for calling the free method of an allocator object 382 * 383 * Please use this instead of calling the methods directly, to ensure proper 384 * memory tracking in all cases. 385 * 386 * @pre Windows: IRQL <= APC_LEVEL (DISPATCH_LEVEL if freeing NonPaged memory) 387 * @pre Unix: Non-interrupt context 388 * @note Will not put the thread to sleep. 389 */ 390 #define PORT_FREE(pAlloc, pMem) _portMemAllocatorFree(pAlloc, pMem) 391 392 /** 393 * @brief Creates an allocator for paged memory. 394 * 395 * Returns an allocator instance where @ref PORT_ALLOC will behave 396 * like @ref portMemAllocPaged. Note the memory holding the PORT_MEM_ALLOCATOR 397 * instance may also be paged. 398 * 399 * @return NULL if creation failed. 400 * 401 * @pre Windows: IRQL <= APC_LEVEL 402 * @pre Unix: Non-interrupt context 403 * @note Will not put the thread to sleep. 404 */ 405 PORT_MEM_ALLOCATOR *portMemAllocatorCreatePaged(void); 406 407 /** 408 * @brief Creates an allocator for non-paged memory. 409 * 410 * Returns an allocator instance where @ref PORT_ALLOC will 411 * behave like @ref portMemAllocNonPaged. Note the memory holding the 412 * PORT_MEM_ALLOCATOR instance will also be non-paged. 413 * 414 * @return NULL if creation failed. 415 * 416 * @pre Windows: IRQL <= DISPATCH_LEVEL 417 * @pre Unix: Non-interrupt context 418 * @note Will not put the thread to sleep. 419 */ 420 PORT_MEM_ALLOCATOR *portMemAllocatorCreateNonPaged(void); 421 422 /** 423 * @brief Creates an allocator over an existing block of memory. 424 * 425 * Adds allocator bookkeeping information to an existing memory block, so that 426 * it can be used with the standard allocator interface. Some of the space of 427 * the preallocated block will be consumed for bookkeeping, so not all of the 428 * memory will be allocatable. 429 * 430 * Use this to create an allocator object on an ISR stack, so memory allocations 431 * can be done at DIQRL. 432 * 433 * @par Implementation details: 434 * The allocator allocates in chunks of 16 bytes, and uses a 2bit-vector to keep 435 * track of free chunks. Thus, the bookkeeping structures for a block of size N 436 * will take about N/64+sizeof(PORT_MEM_ALLOCATOR) bytes. 437 * Use @ref PORT_MEM_PREALLOCATED_BLOCK if you want to specify useful(allocable) 438 * size instead of total size. 439 * 440 * The allocator is only valid while the memory it was created on is valid. 441 * @ref portMemAllocatorRelease must be called on the allocator before the 442 * memory lifecycle ends. 443 * 444 * @return NULL if creation failed. 445 * 446 * @pre Usable at any IRQL/interrupt context 447 * @note Will not put the thread to sleep. 448 * @note This allocator is not thread safe. 449 */ 450 PORT_MEM_ALLOCATOR *portMemAllocatorCreateOnExistingBlock(void *pPreallocatedBlock, NvLength blockSizeBytes); 451 452 /** 453 * @brief Extends the given size to fit the required bookkeeping information 454 * 455 * To be used when preallocating blocks that will be used to create an allocator 456 * Consider these two preallocated memory blocks: 457 * ~~~{.c} 458 * NvU8 xxx[1024]; 459 * NvU8 yyy[PORT_MEM_PREALLOCATED_BLOCK(1024)]; 460 * ~~~ 461 * Block @c xxx has a size of 1024, but only ~950 of that can be allocated. 462 * Block @c yyy has a size of ~1100, and exactly 1024 bytes can be allocated. 463 */ 464 #define PORT_MEM_PREALLOCATED_BLOCK(size) \ 465 (size + PORT_MEM_PREALLOCATED_BLOCK_EXTRA_SIZE(size)) 466 467 /** 468 * @brief releases an allocator instance. 469 * 470 * This must be called to release any resources associated with the allocator. 471 * 472 * @par Checked builds only: 473 * Will assert if pAllocator has unfreed allocations 474 * 475 * @par Undefined: 476 * pAllocator must be an instance of PORT_MEM_ALLOCATOR that was provided by one 477 * of the portMemAllocatorCreate* functions. 478 * 479 * These limitations don't apply to allocators created using @ref portMemAllocatorCreateOnExistingBlock and 480 * @ref portMemExAllocatorCreateLockedOnExistingBlock. 481 * 482 * @pre Windows: IRQL <= APC_LEVEL 483 * @pre Unix: Non-interrupt context 484 * @note Will not put the thread to sleep. 485 */ 486 void portMemAllocatorRelease(PORT_MEM_ALLOCATOR *pAllocator); 487 488 /** 489 * @brief Returns the pointer to the global nonpaged allocator. 490 * 491 * This allocator is always initialized does not need to be released. 492 * 493 * Allocations performed using this allocator are identical to the ones done 494 * by @ref portMemAllocNonPaged 495 */ 496 PORT_MEM_ALLOCATOR *portMemAllocatorGetGlobalNonPaged(void); 497 /** 498 * @brief Returns the pointer to the global paged allocator. 499 * 500 * This allocator is always initialized does not need to be released. 501 * 502 * Allocations performed using this allocator are identical to the ones done 503 * by @ref portMemAllocPaged 504 */ 505 PORT_MEM_ALLOCATOR *portMemAllocatorGetGlobalPaged(void); 506 /** 507 * @brief Prints the memory details gathered by whatever tracking mechanism is 508 * enabled. If pTracking is NULL, aggregate tracking information from all 509 * allocators will be printed. 510 * 511 * @note Printing is done using portDbgPrintf, which prints regardless of 512 * build type and debug levels. 513 */ 514 void portMemPrintTrackingInfo(const PORT_MEM_ALLOCATOR_TRACKING *pTracking); 515 /** 516 * @brief Calls @ref portMemPrintTrackingInfo for all current allocator trackers. 517 */ 518 void portMemPrintAllTrackingInfo(void); 519 520 // @} End core functions 521 522 523 /** 524 * @name Extended Functions 525 * @{ 526 */ 527 528 /** 529 * @brief Returns true if it is safe to allocate paged memory. 530 */ 531 NvBool portMemExSafeForPagedAlloc(void); 532 #define portMemExSafeForPagedAlloc_SUPPORTED PORT_IS_KERNEL_BUILD 533 534 /** 535 * @brief Returns true if it is safe to allocate non-paged memory. 536 */ 537 NvBool portMemExSafeForNonPagedAlloc(void); 538 #define portMemExSafeForNonPagedAlloc_SUPPORTED PORT_IS_KERNEL_BUILD 539 540 /** 541 * @brief Public allocator tracking information 542 */ 543 typedef struct PORT_MEM_TRACK_ALLOCATOR_STATS 544 { 545 /** @brief Total number of allocations */ 546 NvU32 numAllocations; 547 /** @brief Total allocated bytes, including all staging */ 548 NvLength allocatedSize; 549 /** @brief Useful size of allocations - What was actually requested */ 550 NvLength usefulSize; 551 /** @brief Extra size allocated for tracking/debugging purposes */ 552 NvLength metaSize; 553 } PORT_MEM_TRACK_ALLOCATOR_STATS; 554 555 /** 556 * @brief Returns the statistics of currently active allocations for the given 557 * allocator. 558 * 559 * If pAllocator is NULL, it returns stats for all allocators, as well as the 560 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 561 */ 562 NV_STATUS portMemExTrackingGetActiveStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 563 564 /** 565 * @brief Returns the statistics of all allocations made with the given 566 * allocator since it was created. 567 * 568 * If pAllocator is NULL, it returns stats for all allocators, as well as the 569 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 570 */ 571 NV_STATUS portMemExTrackingGetTotalStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 572 573 /** 574 * @brief Returns the statistics of peak allocations made with the given 575 * allocator since it was created. 576 * 577 * Peak data reports the high-water mark based on the maximum size (the peak 578 * allocations doesn't report the largest number of allocations, it reports 579 * the number of allocations at the time the peak size was achieved). This is 580 * done so that the other peak stats, which are derived from peak size and 581 * peak allocations, are consistent with each other. 582 * 583 * If pAllocator is NULL, it returns stats for all allocators, as well as the 584 * memory allocated with @ref portMemAllocPaged and @ref portMemAllocNonPaged 585 */ 586 NV_STATUS portMemExTrackingGetPeakStats(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOCATOR_STATS *pStats); 587 588 /** 589 * @brief Cycles through the tracking infos for allocations by pAllocator 590 * If pAllocator is NULL, it will cycle through all allocations. 591 * 592 * @param [out] pInfo The info will be written to this buffer. 593 * @param [in, out] pIterator 594 * Should point to NULL the first time it is called. 595 * Every next call should pass the value returned by previous. 596 * To reset the loop, set the iterator to NULL. 597 * Upon writing the last range, the iterator will be set to NULL. 598 * The iterator is only valid until the next alloc/free from this allocator. 599 * There is no need to release the iterator in any way. 600 * 601 * @return NV_ERR_OBJECT_NOT_FOUND if no allocations exist. 602 */ 603 NV_STATUS portMemExTrackingGetNext(const PORT_MEM_ALLOCATOR *pAllocator, PORT_MEM_TRACK_ALLOC_INFO *pInfo, void **pIterator); 604 605 /** 606 * @brief Gets the total size of the underlying heap, in bytes. 607 */ 608 NvLength portMemExTrackingGetHeapSize(void); 609 610 /** 611 * @brief Gets the usable size in bytes (sans metadata/padding) of the given allocation. 612 */ 613 NvLength portMemExTrackingGetAllocUsableSize(void *pMem); 614 615 /** 616 * @brief Copies from user memory to kernel memory. 617 * 618 * When accepting data as input from user space it is necessary to take 619 * additional precautions to access it safely and securely. This means copy 620 * the user data into a kernel buffer and then using that kernel buffer for all 621 * needed accesses. 622 * 623 * The function will fail if pUser is an invalid user space pointer or if the 624 * memory it refers to is less than length bytes long. A valid kernel pointer 625 * is interpreted as an invalid user pointer. 626 * @par Checked builds only: 627 * Will trigger a breakpoint if pUser is invalid userspace pointer 628 * 629 * The function will fail if pKernel is NULL. 630 * 631 * The function will fail if lengthBytes is 0. 632 * 633 * @return 634 * - NV_OK if successful 635 * - NV_ERR_INVALID_POINTER if pUser is invalid or pKernel is NULL 636 * - NV_ERR_INVALID_ARGUMENT if lengthBytes is 0 637 */ 638 NV_STATUS portMemExCopyFromUser(const NvP64 pUser, void *pKernel, NvLength lengthBytes); 639 #define portMemExCopyFromUser_SUPPORTED PORT_IS_KERNEL_BUILD 640 641 642 /** 643 * @brief Copies from kernel memory to user memory. 644 * 645 * This is the reverse of @ref portMemExCopyFromUser. The copy in this case is 646 * from pKernel to pUser. 647 * 648 * See @ref portMemExCopyFromUser for more details. 649 * 650 */ 651 NV_STATUS portMemExCopyToUser(const void *pKernel, NvP64 pUser, NvLength lengthBytes); 652 #define portMemExCopyToUser_SUPPORTED PORT_IS_KERNEL_BUILD 653 654 /** 655 * @brief Returns the size (in bytes) of a single memory page. 656 */ 657 NvLength portMemExGetPageSize(void); 658 #define portMemExGetPageSize_SUPPORTED PORT_IS_KERNEL_BUILD 659 660 /** 661 * @brief Opaque container holding an allocation of physical system memory. 662 */ 663 typedef struct PORT_PHYSICAL_MEMDESC PORT_PHYSICAL_MEMDESC; 664 665 /** 666 * @brief Creates a handle used to manage and manipulate a physical memory 667 * allocation. 668 * 669 * @param pAllocator the allocator to use the create the allocation's tracking 670 * structures. This allocator is *not* used to allocate physical memory. 671 * 672 * @return NULL if the allocation failed. 673 */ 674 PORT_PHYSICAL_MEMDESC *portMemExPhysicalDescCreate(PORT_MEM_ALLOCATOR *pAllocator); 675 #define portMemExPhysicalDescCreate_SUPPORTED PORT_IS_KERNEL_BUILD 676 677 /** 678 * @brief Types of caching for physical memory mappings. 679 * 680 * In case a target architecture does not support a specific caching mode, 681 * the mapping call will fail. 682 * Specifying PORT_MEM_ANYCACHE lets the implementation pick a caching mode that 683 * is present on the target architecture. This way the mapping will not fail. 684 */ 685 typedef enum 686 { 687 PORT_MEM_UNCACHED, 688 PORT_MEM_CACHED, 689 PORT_MEM_WRITECOMBINED, 690 PORT_MEM_ANYCACHE 691 } PortMemCacheMode; 692 693 /** 694 * @brief Types of access protections for physical memory mappings. 695 */ 696 typedef enum 697 { 698 PORT_MEM_PROT_NO_ACCESS = 0, 699 PORT_MEM_PROT_READ = 1, 700 PORT_MEM_PROT_WRITE = 2, 701 PORT_MEM_PROT_READ_WRITE = 3, 702 PORT_MEM_PROT_EXEC = 4, 703 PORT_MEM_PROT_READ_EXEC = 5, 704 PORT_MEM_PROT_WRITE_EXEC = 6, 705 PORT_MEM_PROT_READ_WRITE_EXEC = 7 706 } PortMemProtectMode; 707 708 /** 709 * @brief Populates a physical memory descriptor with backing pages. 710 * 711 * Populates a descriptor with physical pages. Pages will be zeroed. 712 */ 713 NV_STATUS portMemExPhysicalDescPopulate(PORT_PHYSICAL_MEMDESC *pPmd, NvLength sizeBytes, NvBool bContiguous); 714 #define portMemExPhysicalDescPopulate_SUPPORTED PORT_IS_KERNEL_BUILD 715 716 717 /** 718 * @brief allocates a PMD and populates it with memory 719 * 720 * This is a combination of @ref portMemExPhysicalDescCreate and @ref 721 * portMemExPhysicalDescPopulate. It should be the preferred method to allocate 722 * physical memory when it is possible to do it as a single step. Not only 723 * does the caller require less code and error handling but it allows the 724 * implementation the option to combine the tracking data into fewer 725 * allocations since it knows the size up front. 726 * 727 * @param [out] ppPmd - Pointer to the allocated PMD. 728 * @param pAllocator - Allocator to use when allocating the PMD 729 */ 730 NV_STATUS portMemExPhysicalDescCreateAndPopulate(PORT_MEM_ALLOCATOR *pAllocator, 731 PORT_PHYSICAL_MEMDESC **ppPmd, NvLength sizeBytes, NvBool bContiguous); 732 #define portMemExPhysicalDescCreateAndPopulate_SUPPORTED PORT_IS_KERNEL_BUILD 733 734 735 /** 736 * @brief Adds a contiguous memory range to the physical memory descriptor 737 * 738 * To describe a non-contiguous memory range, call this function once for every 739 * contiguous range. Range order will be determined by function call order, 740 * not the range addresses. 741 */ 742 NV_STATUS portMemExPhysicalDescribeRange(PORT_PHYSICAL_MEMDESC *pPmd, NvU64 start, NvLength length); 743 #define portMemExPhysicalDescribeRange_SUPPORTED PORT_IS_KERNEL_BUILD 744 745 /** 746 * @brief Hands back the next contiguous memory range in the memory descriptor 747 * 748 * @param [out] pStart - Physical address of the range 749 * @param [out] pLength - Length of the range 750 * @param [in, out] pIterator 751 * Should point to NULL the first time it is called. 752 * Every next call should pass the value returned by previous. 753 * To reset the loop, set the iterator to NULL. 754 * Upon writing the last range, the iterator will be set to NULL. 755 * The iterator is valid until pPmd is destroyed. 756 * There is no need to release the iterator in any way. 757 * 758 * @return NV_ERR_OBJECT_NOT_FOUND if no ranges exist. 759 */ 760 NV_STATUS portMemExPhysicalGetNextRange(PORT_PHYSICAL_MEMDESC *pPmd, 761 NvU64 *pStart, NvLength *pLength, void **pIterator); 762 #define portMemExPhysicalGetNextRange_SUPPORTED PORT_IS_KERNEL_BUILD 763 764 /** 765 * @brief Frees the memory descriptor and all tracking data. The descriptor must 766 * have been allocated with @ref portMemExPhysicalDescCreate or 767 * @ref portMemExPhysicalDescCreateAndPopulate 768 * 769 * Freed memory is not automatically unmapped. 770 * 771 * It is guaranteed that after memory has been freed, the original data can no 772 * longer be read in any way. 773 * @par Undefined: 774 * Accessing a mapping that has been freed results in undefined behavior. 775 */ 776 void portMemExPhysicalDescFree(PORT_PHYSICAL_MEMDESC *pPmd); 777 #define portMemExPhysicalDescFree_SUPPORTED PORT_IS_KERNEL_BUILD 778 779 780 /** 781 * @brief Frees physical memory allocated with @ref portMemExPhysicalDescPopulate 782 */ 783 void portMemExPhysicalFree(PORT_PHYSICAL_MEMDESC *pPmd); 784 #define portMemExPhysicalFree_SUPPORTED PORT_IS_KERNEL_BUILD 785 786 787 /** 788 * @brief Maps a region of a @ref PORT_PHYSICAL_MEMDESC 789 * 790 * @param [out] ppMapping - Virtual address where the physical memory is mapped 791 * @param offset - Offset of the physical memory where the region starts. 792 * The region must start on a page boundary. 793 * @param length - Length of the physical memory region. 794 * Needs to be a multiple of page size. 795 * @param protect - Mapping protections 796 * @param cacheMode - Mapping cache mode. 797 * Only PORT_MEM_ANYCACHE is guaranteed to be supported. 798 * 799 * @return NV_ERR_NOT_SUPPORTED if the specified cache mode is not supported by 800 * the current architecture. 801 */ 802 NV_STATUS portMemExPhysicalMap(PORT_PHYSICAL_MEMDESC *pPmd, 803 void **ppMapping, NvU64 offset, NvU64 length, 804 PortMemProtectMode protect, PortMemCacheMode cacheMode); 805 #define portMemExPhysicalMap_SUPPORTED PORT_IS_KERNEL_BUILD 806 807 /** 808 * @brief Unmaps a region created with @ref portMemExPhysicalMap. 809 * 810 * @par Undefined: 811 * Accessing an unmapped memory is undefined, but it is guaranteed that the 812 * actual data can't be read/overwritten. 813 */ 814 NV_STATUS portMemExPhysicalUnmap(PORT_PHYSICAL_MEMDESC *pPmd, void *pMapping); 815 #define portMemExPhysicalUnmap_SUPPORTED PORT_IS_KERNEL_BUILD 816 817 /** 818 * @brief Creates a thread safe allocator over an existing block of memory. 819 * 820 * @note See @ref portMemAllocatorCreateOnExistingBlock for other limitations. 821 * @note User should initialize @p pSpinlock and destroy it after it 822 * has finished using this allocator. 823 */ 824 PORT_MEM_ALLOCATOR *portMemExAllocatorCreateLockedOnExistingBlock(void *pPreallocatedBlock, NvLength blockSizeBytes, void *pSpinlock); 825 #define portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED \ 826 (PORT_IS_MODULE_SUPPORTED(sync)) 827 828 829 /** 830 * @brief Maps the given physical address range to nonpaged system space. 831 * 832 * @param[in] start Specifies the starting physical address of the I/O 833 * range to be mapped. 834 * @param[in] byteSize Specifies the number of bytes to be mapped. 835 * 836 * @return The base virtual address that maps the base physical address for 837 * the range 838 */ 839 void *portMemExMapIOSpace(NvU64 start, NvU64 byteSize); 840 #define portMemExMapIOSpace_SUPPORTED (NVOS_IS_WINDOWS && !PORT_IS_MODS) 841 842 /** 843 * @brief Unmaps a specified range of physical addresses previously mapped by 844 * portMapIOSpace 845 * 846 * @param[in] addr Pointer to the base virtual address to which the 847 * physical pages were mapped. 848 * @param[in] byteSize Specifies the number of bytes that were mapped. 849 */ 850 void portMemExUnmapIOSpace(void *addr, NvU64 byteSize); 851 #define portMemExUnmapIOSpace_SUPPORTED (NVOS_IS_WINDOWS && !PORT_IS_MODS) 852 853 // @} End extended functions 854 855 856 /** 857 * @note Memory tracking is controlled through the following compile-time flags. 858 * The PORT_MEM_TRACK_USE_* constants should be defined to 0 or 1. 859 * If nothing is defined, the default values are assigned here. 860 */ 861 #if !defined(PORT_MEM_TRACK_USE_COUNTER) 862 /** 863 * @brief Use allocations counter for all allocators 864 * 865 * Allocation counter is lightweight and can detect if a leak is present. 866 * Default is always on. 867 */ 868 #define PORT_MEM_TRACK_USE_COUNTER 1 869 #endif 870 #if !defined(PORT_MEM_TRACK_USE_FENCEPOSTS) 871 /** 872 * @brief Use fenceposts around all allocated blocks 873 * 874 * Fenceposts can detect out of bounds writes and improper free calls 875 * Default is on for checked builds (where it will assert if an error occurs) 876 */ 877 #define PORT_MEM_TRACK_USE_FENCEPOSTS PORT_IS_CHECKED_BUILD 878 #endif 879 #if !defined(PORT_MEM_TRACK_USE_ALLOCLIST) 880 /** 881 * @brief Keep a list of all allocations. 882 * 883 * Allocation lists can give more details about detected leaks, and allow 884 * cycling through all allocations. 885 * Default is off. 886 * @todo Perhaps enable for checked builds? 887 */ 888 #define PORT_MEM_TRACK_USE_ALLOCLIST 0 889 #endif 890 #if !defined(PORT_MEM_TRACK_USE_CALLERINFO) 891 /** 892 * @brief Track file:line information for all allocations 893 * 894 * On release builds the filename hash is passed instead of the string. This 895 * requires NvLog to be enabled. 896 * Default is off. 897 */ 898 #define PORT_MEM_TRACK_USE_CALLERINFO 0 899 #endif 900 /** 901 * @brief Track instruction pointer instead of function/file/line information 902 * for all allocations 903 * 904 * Has no effect unless PORT_MEM_TRACK_USE_CALLERINFO is also set. 905 */ 906 #if !defined(PORT_MEM_TRACK_USE_CALLERINFO_IP) 907 #if NVCPU_IS_RISCV64 908 #define PORT_MEM_TRACK_USE_CALLERINFO_IP 1 909 #else 910 #define PORT_MEM_TRACK_USE_CALLERINFO_IP 0 911 #endif 912 #endif 913 #if !defined(PORT_MEM_TRACK_USE_LOGGING) 914 /** 915 * @brief Log all alloc and free calls to a binary NvLog buffer 916 * Requires NvLog to be enabled. 917 * 918 * Default is off. 919 */ 920 #define PORT_MEM_TRACK_USE_LOGGING 0 921 #endif 922 #if !defined(PORT_MEM_TRACK_USE_LIMIT) 923 /** 924 * @brief Track and enforce a heap memory usage limit on processes 925 * running in GSP-RM. 926 * 927 * Default is on in GSP-RM only. 928 */ 929 #ifndef GSP_PLUGIN_BUILD 930 #define PORT_MEM_TRACK_USE_LIMIT (NVOS_IS_LIBOS) 931 #else 932 #define PORT_MEM_TRACK_USE_LIMIT 0 933 #endif 934 #endif // !defined(PORT_MEM_TRACK_USE_LIMIT) 935 936 // Memory tracking header can redefine some functions declared here. 937 #include "nvport/inline/memory_tracking.h" 938 939 /** @brief Nothing is printed unless @ref portMemPrintTrackingInfo is called */ 940 #define PORT_MEM_TRACK_PRINT_LEVEL_SILENT 0 941 /** @brief Print when an error occurs and at shutdown */ 942 #define PORT_MEM_TRACK_PRINT_LEVEL_BASIC 1 943 /** @brief Print at every alloc and free, and at any abnormal situation */ 944 #define PORT_MEM_TRACK_PRINT_LEVEL_VERBOSE 2 945 946 #if !defined(PORT_MEM_TRACK_PRINT_LEVEL) 947 #if PORT_IS_CHECKED_BUILD || PORT_MEM_TRACK_ALLOC_SIZE 948 #define PORT_MEM_TRACK_PRINT_LEVEL PORT_MEM_TRACK_PRINT_LEVEL_BASIC 949 #else 950 #define PORT_MEM_TRACK_PRINT_LEVEL PORT_MEM_TRACK_PRINT_LEVEL_SILENT 951 #endif // PORT_IS_CHECKED_BUILD 952 #endif // !defined(PORT_MEM_TRACK_PRINT_LEVEL) 953 954 /** 955 * @brief Single allocation description. 956 * 957 * Must be defined after memory_tracking.h is included for PORT_MEM_CALLERINFO. 958 */ 959 struct PORT_MEM_TRACK_ALLOC_INFO 960 { 961 #if PORT_MEM_TRACK_USE_CALLERINFO 962 /** 963 * @brief Function / file / line or instruction pointer. 964 */ 965 PORT_MEM_CALLERINFO callerInfo; 966 #endif 967 /** 968 * @brief pointer to the allocated memory block. 969 */ 970 void *pMemory; 971 /** 972 * @brief Size of the allocated memory block 973 */ 974 NvLength size; 975 /** 976 * @brief Pointer to the allocator that allocated the memory. 977 * If the memory was allocated globally, this will be NULL 978 */ 979 PORT_MEM_ALLOCATOR *pAllocator; 980 /** 981 * @brief Timestamp of the allocation. Will be 0 if it wasn't logged. 982 */ 983 NvU64 timestamp; 984 }; 985 986 /** 987 * @} 988 */ 989 990 #endif // _NVPORT_MEMORY_H_ 991