1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /** 25 * @file 26 * @brief MEMORY module tracking functions implementation 27 * 28 */ 29 30 #include "nvport/nvport.h" 31 #include <nvport/safe.h> 32 33 #if !PORT_IS_MODULE_SUPPORTED(debug) 34 #error "DEBUG module must be present for memory tracking" 35 #endif 36 37 #if PORT_MEM_TRACK_USE_LIMIT 38 #include "os/os.h" 39 #define PORT_MEM_LIMIT_MAX_PIDS 32 40 #endif 41 42 #if NVOS_IS_LIBOS 43 #define PORT_MEM_THREAD_SAFE_ALLOCATIONS 0 44 #else 45 #define PORT_MEM_THREAD_SAFE_ALLOCATIONS 1 46 #endif 47 48 #if PORT_MEM_THREAD_SAFE_ALLOCATIONS && !PORT_IS_MODULE_SUPPORTED(atomic) 49 #error "ATOMIC module must be present for memory tracking" 50 #endif 51 52 #if PORT_MEM_THREAD_SAFE_ALLOCATIONS 53 #define PORT_MEM_ATOMIC_ADD_SIZE portAtomicAddSize 54 #define PORT_MEM_ATOMIC_SUB_SIZE portAtomicSubSize 55 #define PORT_MEM_ATOMIC_DEC_U32 portAtomicDecrementU32 56 #define PORT_MEM_ATOMIC_INC_U32 portAtomicIncrementU32 57 #define PORT_MEM_ATOMIC_SET_U32 portAtomicSetU32 58 #define PORT_MEM_ATOMIC_CAS_SIZE portAtomicCompareAndSwapSize 59 #define PORT_MEM_ATOMIC_CAS_U32 portAtomicCompareAndSwapU32 60 #else 61 // 62 // We can just stub out the atomic operations for non-atomic ones and not waste 63 // cycles on synchronization 64 // 65 #define PORT_MEM_ATOMIC_ADD_SIZE(pVal, val) (*((volatile NvSPtr *)pVal) += val) 66 #define PORT_MEM_ATOMIC_SUB_SIZE(pVal, val) (*((volatile NvSPtr *)pVal) -= val) 67 #define PORT_MEM_ATOMIC_DEC_U32(pVal) (--(*((volatile NvU32 *)pVal))) 68 #define PORT_MEM_ATOMIC_INC_U32(pVal) (++(*((volatile NvU32 *)pVal))) 69 #define PORT_MEM_ATOMIC_SET_U32(pVal, val) (*((volatile NvU32 *)pVal) = val) 70 #define PORT_MEM_ATOMIC_CAS_SIZE(pVal, newVal, oldVal) \ 71 ((*pVal == oldVal) ? ((*((volatile NvSPtr *)pVal) = newVal), NV_TRUE) : NV_FALSE) 72 #define PORT_MEM_ATOMIC_CAS_U32(pVal, newVal, oldVal) \ 73 ((*pVal == oldVal) ? ((*((volatile NvU32 *)pVal) = newVal), NV_TRUE) : NV_FALSE) 74 #endif // !PORT_MEM_THREAD_SAFE_ALLOCATIONS 75 76 struct PORT_MEM_ALLOCATOR_IMPL 77 { 78 PORT_MEM_ALLOCATOR_TRACKING tracking; 79 }; 80 81 // 82 // Debug print macros 83 // 84 #if PORT_MEM_TRACK_PRINT_LEVEL == PORT_MEM_TRACK_PRINT_LEVEL_SILENT 85 #define PORT_MEM_PRINT_ERROR(...) 86 #define PORT_MEM_PRINT_INFO(...) 87 #elif PORT_MEM_TRACK_PRINT_LEVEL == PORT_MEM_TRACK_PRINT_LEVEL_BASIC 88 #define PORT_MEM_PRINT_ERROR(...) portDbgPrintf(__VA_ARGS__) 89 #define PORT_MEM_PRINT_INFO(...) 90 #else 91 #define PORT_MEM_PRINT_ERROR(...) portDbgPrintf(__VA_ARGS__) 92 #define PORT_MEM_PRINT_INFO(...) portDbgPrintf(__VA_ARGS__) 93 #endif 94 95 // Simple implementation of a spinlock that is going to be used where sync module is not included. 96 #if !PORT_IS_MODULE_SUPPORTED(sync) 97 98 #if NVCPU_IS_RISCV64 99 #error "Sync module should be enabled for RISC-V builds" 100 #endif 101 102 typedef volatile NvU32 PORT_SPINLOCK; 103 static NvLength portSyncSpinlockSize = sizeof(PORT_SPINLOCK); 104 static NV_STATUS portSyncSpinlockInitialize(PORT_SPINLOCK *pSpinlock) 105 { 106 *pSpinlock = 0; 107 return NV_OK; 108 } 109 static void portSyncSpinlockAcquire(PORT_SPINLOCK *pSpinlock) 110 { 111 while (!PORT_MEM_ATOMIC_CAS_U32(pSpinlock, 1, 0)); 112 } 113 static void portSyncSpinlockRelease(PORT_SPINLOCK *pSpinlock) 114 { 115 PORT_MEM_ATOMIC_SET_U32(pSpinlock, 0); 116 } 117 static void portSyncSpinlockDestroy(PORT_SPINLOCK *pSpinlock) 118 { 119 PORT_UNREFERENCED_VARIABLE(pSpinlock); 120 } 121 #endif 122 123 #define PORT_MEM_LOCK_INIT(lock) \ 124 do { \ 125 lock = _portMemAllocNonPagedUntracked(portSyncSpinlockSize); \ 126 portSyncSpinlockInitialize(lock); \ 127 } while (0) 128 #define PORT_MEM_LOCK_DESTROY(lock) \ 129 do { \ 130 portSyncSpinlockDestroy(lock); \ 131 _portMemFreeUntracked(lock); \ 132 } while(0) 133 #define PORT_MEM_LOCK_ACQUIRE(lock) portSyncSpinlockAcquire(lock) 134 #define PORT_MEM_LOCK_RELEASE(lock) portSyncSpinlockRelease(lock) 135 136 137 // 138 // List link operation that operates on structures that have pNext and pPrev 139 // fields. Assumes the root always exists. 140 // 141 #define PORT_LOCKED_LIST_LINK(pRoot, pNode, lock) \ 142 do { \ 143 PORT_MEM_LOCK_ACQUIRE(lock); \ 144 (pNode)->pNext = (pRoot); \ 145 (pNode)->pPrev = (pRoot)->pPrev; \ 146 (pRoot)->pPrev = (pNode); \ 147 (pNode)->pPrev->pNext = (pNode); \ 148 PORT_MEM_LOCK_RELEASE(lock); \ 149 } while(0) 150 151 #define PORT_LOCKED_LIST_UNLINK(pRoot, pNode, lock) \ 152 do { \ 153 PORT_MEM_LOCK_ACQUIRE(lock); \ 154 (pNode)->pNext->pPrev = (pNode)->pPrev; \ 155 (pNode)->pPrev->pNext = (pNode)->pNext; \ 156 PORT_MEM_LOCK_RELEASE(lock); \ 157 } while (0) 158 159 // 160 // All memory tracking globals are contained in this structure 161 // 162 static struct PORT_MEM_GLOBALS 163 { 164 PORT_MEM_ALLOCATOR_TRACKING mainTracking; 165 void *trackingLock; 166 struct 167 { 168 PORT_MEM_ALLOCATOR paged; 169 PORT_MEM_ALLOCATOR nonPaged; 170 PORT_MEM_ALLOCATOR_IMPL pagedImpl; 171 PORT_MEM_ALLOCATOR_IMPL nonPagedImpl; 172 } alloc; 173 NvU32 initCount; 174 NvU32 totalAllocators; 175 #if PORT_MEM_TRACK_USE_LIMIT 176 NvBool bLimitEnabled; 177 NvLength limitPid[PORT_MEM_LIMIT_MAX_PIDS]; 178 NvLength counterPid[PORT_MEM_LIMIT_MAX_PIDS]; 179 #endif 180 } portMemGlobals; 181 182 // 183 // Memory counter implementation 184 // 185 #if PORT_MEM_TRACK_USE_COUNTER 186 #if PORT_MEM_TRACK_ALLOC_SIZE 187 static NV_INLINE NvLength 188 _portMemExTrackingGetAllocUsableSizeWrapper 189 ( 190 void *pMem 191 ) 192 { 193 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetAllocUsableSize) 194 // 195 // blockSize in PORT_MEM_HEADER does not include the size of the header/ 196 // footer, but the underlying malloc implementation doesn't know about 197 // those. Account for them here. 198 // 199 NvLength allocSize = portMemExTrackingGetAllocUsableSize(PORT_MEM_SUB_HEADER_PTR(pMem)); 200 PORT_ASSERT_CHECKED(allocSize > PORT_MEM_STAGING_SIZE); 201 return allocSize - PORT_MEM_STAGING_SIZE; 202 #elif PORT_MEM_HEADER_HAS_BLOCK_SIZE 203 return PORT_MEM_SUB_HEADER_PTR(pMem)->blockSize; 204 #endif 205 } 206 static NV_INLINE void 207 _portMemExTrackingSetOrGetAllocUsableSize 208 ( 209 void *pMem, 210 NvLength *pSize 211 ) 212 { 213 #if PORT_MEM_HEADER_HAS_BLOCK_SIZE 214 PORT_MEM_SUB_HEADER_PTR(pMem)->blockSize = *pSize; 215 #else 216 *pSize = _portMemExTrackingGetAllocUsableSizeWrapper(pMem); 217 #endif 218 } 219 #endif // PORT_MEM_TRACK_ALLOC_SIZE 220 static NV_INLINE void 221 _portMemCounterInit 222 ( 223 PORT_MEM_COUNTER *pCounter 224 ) 225 { 226 portMemSet(pCounter, 0, sizeof(*pCounter)); 227 } 228 static NV_INLINE void 229 _portMemCounterInc 230 ( 231 PORT_MEM_COUNTER *pCounter, 232 NvLength size 233 ) 234 { 235 NvU32 activeAllocs; 236 NvLength activeSize = 0; 237 238 activeAllocs = PORT_MEM_ATOMIC_INC_U32(&pCounter->activeAllocs); 239 PORT_MEM_ATOMIC_INC_U32(&pCounter->totalAllocs); 240 #if PORT_MEM_TRACK_ALLOC_SIZE 241 // 242 // activeSize is only tracked on configurations where we can retrieve the 243 // allocation size from allocation metadata in _portMemCounterDec. 244 // 245 activeSize = PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->activeSize, size); 246 #endif 247 248 // 249 // Note: this can overflow on 32-bit platforms if we exceed 4GB cumulative 250 // allocations. It's not trivial to fix, since NvPort doesn't emulate 64-bit 251 // atomics on 32-bit platforms, so just assume this doesn't happen (or 252 // doesn't matter too much if it does, since it's only for reporting). 253 // 254 PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->totalSize, size); 255 256 // Update the peak stats, if we're updating the peakSize 257 { 258 NvU32 peakAllocs; 259 NvLength peakSize = pCounter->peakSize; 260 while (activeSize > peakSize) 261 { 262 PORT_MEM_ATOMIC_CAS_SIZE(&pCounter->peakSize, activeSize, peakSize); 263 peakSize = pCounter->peakSize; 264 } 265 266 // 267 // Ensure peakAllocs stays (approximately) in sync with peakSize, rather 268 // than always taking the greatest peakAllocs, so that the peak stats 269 // report is consistent. 270 // 271 do 272 { 273 peakAllocs = pCounter->peakAllocs; 274 275 // 276 // Only attempt to update the peakAllocs if activeSize is still the 277 // peakSize. 278 // 279 if (activeSize != pCounter->peakSize) 280 break; 281 } while (!PORT_MEM_ATOMIC_CAS_U32(&pCounter->peakAllocs, activeAllocs, peakAllocs)); 282 } 283 } 284 static NV_INLINE void 285 _portMemCounterDec 286 ( 287 PORT_MEM_COUNTER *pCounter, 288 NvLength size 289 ) 290 { 291 PORT_MEM_ATOMIC_DEC_U32(&pCounter->activeAllocs); 292 #if PORT_MEM_TRACK_ALLOC_SIZE 293 PORT_MEM_ATOMIC_SUB_SIZE(&pCounter->activeSize, size); 294 #else 295 PORT_UNREFERENCED_VARIABLE(size); 296 #endif 297 } 298 299 #define PORT_MEM_COUNTER_INIT(pCounter) _portMemCounterInit(pCounter) 300 #define PORT_MEM_COUNTER_INC(pCounter, size) _portMemCounterInc(pCounter, size) 301 #define PORT_MEM_COUNTER_DEC(pCounter, size) _portMemCounterDec(pCounter, size) 302 #else 303 #define PORT_MEM_COUNTER_INIT(x) 304 #define PORT_MEM_COUNTER_INC(x, y) PORT_UNREFERENCED_VARIABLE(y) 305 #define PORT_MEM_COUNTER_DEC(x, y) PORT_UNREFERENCED_VARIABLE(y) 306 #endif // COUNTER 307 308 309 // 310 // Memory fenceposts implementation 311 // 312 #if PORT_MEM_TRACK_USE_FENCEPOSTS 313 #define PORT_MEM_FENCE_HEAD_MAGIC 0x68656164 // 'head' 314 #define PORT_MEM_FENCE_TAIL_MAGIC 0x7461696c // 'tail' 315 316 static NV_INLINE void 317 _portMemFenceInit 318 ( 319 PORT_MEM_ALLOCATOR *pAlloc, 320 void *pMem, 321 NvLength size 322 ) 323 { 324 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 325 PORT_MEM_FOOTER *pTail = (PORT_MEM_FOOTER*)((NvU8*)pMem + size); 326 327 pHead->fence.pAllocator = pAlloc; 328 pHead->fence.magic = PORT_MEM_FENCE_HEAD_MAGIC; 329 pTail->fence.magic = PORT_MEM_FENCE_TAIL_MAGIC; 330 } 331 332 static NV_INLINE void 333 _portMemFenceCheck 334 ( 335 PORT_MEM_ALLOCATOR *pAlloc, 336 void *pMem, 337 NvLength size 338 ) 339 { 340 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 341 PORT_MEM_FOOTER *pTail = (PORT_MEM_FOOTER*)((NvU8*)pMem + size); 342 343 if (pHead->fence.magic != PORT_MEM_FENCE_HEAD_MAGIC || 344 pTail->fence.magic != PORT_MEM_FENCE_TAIL_MAGIC) 345 { 346 PORT_MEM_PRINT_ERROR("Memory corruption detected on block %p\n", pMem); 347 PORT_ASSERT_CHECKED(pHead->fence.magic == PORT_MEM_FENCE_HEAD_MAGIC); 348 PORT_ASSERT_CHECKED(pTail->fence.magic == PORT_MEM_FENCE_TAIL_MAGIC); 349 } 350 if (pHead->fence.pAllocator != pAlloc) 351 { 352 PORT_MEM_PRINT_ERROR("Freeing block %p using a wrong allocator (%p instead of %p)\n", 353 pMem, pAlloc, pHead->fence.pAllocator); 354 PORT_ASSERT_CHECKED(pHead->fence.pAllocator == pAlloc); 355 356 } 357 } 358 359 #define PORT_MEM_FENCE_CHECK(pAlloc, pMem, size) _portMemFenceCheck(pAlloc, pMem, size) 360 #define PORT_MEM_FENCE_INIT(pAlloc, pMem, size) _portMemFenceInit(pAlloc, pMem, size) 361 #else 362 #define PORT_MEM_FENCE_INIT(x, y, z) 363 #define PORT_MEM_FENCE_CHECK(x, y, z) 364 #endif // FENCEPOSTS 365 366 367 // 368 // Memory allocation lists implementation 369 // 370 #if PORT_MEM_TRACK_USE_ALLOCLIST 371 static NV_INLINE void 372 _portMemListAdd 373 ( 374 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 375 void *pMem 376 ) 377 { 378 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 379 PORT_MEM_LIST *pList = &pHead->list; 380 pList->pNext = pList; 381 pList->pPrev = pList; 382 if (!PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList, NULL)) 383 { 384 PORT_LOCKED_LIST_LINK(pTracking->pFirstAlloc, pList, pTracking->listLock); 385 } 386 } 387 static NV_INLINE void 388 _portMemListRemove 389 ( 390 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 391 void *pMem 392 ) 393 { 394 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 395 PORT_MEM_LIST *pList = &pHead->list; 396 397 if (!PORT_MEM_ATOMIC_CAS_SIZE(&pList->pNext, NULL, pList)) 398 { 399 PORT_LOCKED_LIST_UNLINK(pTracking->pFirstAlloc, pList, pTracking->listLock); 400 } 401 PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList->pNext, pList); 402 } 403 404 static NV_INLINE PORT_MEM_HEADER * 405 _portMemListGetHeader 406 ( 407 PORT_MEM_LIST *pList 408 ) 409 { 410 return (PORT_MEM_HEADER*)((NvU8*)pList - (NvUPtr)(&((PORT_MEM_HEADER*)NULL)->list)); 411 } 412 #define PORT_MEM_LIST_INIT(pTracking) \ 413 do { \ 414 (pTracking)->pFirstAlloc = NULL; \ 415 PORT_MEM_LOCK_INIT((pTracking)->listLock); \ 416 } while (0) 417 #define PORT_MEM_LIST_DESTROY(pTracking) PORT_MEM_LOCK_DESTROY((pTracking)->listLock) 418 #define PORT_MEM_LIST_ADD(pTracking, pMem) _portMemListAdd(pTracking, pMem) 419 #define PORT_MEM_LIST_REMOVE(Tracking, pMem) _portMemListRemove(pTracking, pMem) 420 #else 421 #define PORT_MEM_LIST_INIT(x) 422 #define PORT_MEM_LIST_DESTROY(x) 423 #define PORT_MEM_LIST_ADD(x, y) 424 #define PORT_MEM_LIST_REMOVE(x, y) 425 #endif // ALLOCLIST 426 427 428 429 // 430 // Memory allocation-caller info implementation 431 // 432 #if PORT_MEM_TRACK_USE_CALLERINFO 433 434 static NV_INLINE void 435 _portMemCallerInfoInitMem 436 ( 437 void *pMem, 438 PORT_MEM_CALLERINFO callerInfo 439 ) 440 { 441 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 442 portMemCopy(&pHead->callerInfo, sizeof(callerInfo), 443 &callerInfo, sizeof(callerInfo)); 444 } 445 static NV_INLINE void 446 _portMemCallerInfoInitTracking 447 ( 448 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 449 PORT_MEM_CALLERINFO callerInfo 450 ) 451 { 452 portMemCopy(&pTracking->callerInfo, sizeof(callerInfo), 453 &callerInfo, sizeof(callerInfo)); 454 } 455 456 #define PORT_MEM_CALLERINFO_INIT_TRACKING(pTracking) \ 457 _portMemCallerInfoInitTracking(pTracking, PORT_MEM_CALLERINFO_PARAM) 458 #define PORT_MEM_CALLERINFO_INIT_MEM(pMem) \ 459 _portMemCallerInfoInitMem(pMem, PORT_MEM_CALLERINFO_PARAM) 460 461 #if PORT_MEM_TRACK_USE_CALLERINFO_IP 462 #if NVCPU_IS_RISCV64 463 // 464 // Libos has custom %a format specifier that decodes an instruction pointer into 465 // a function / file / line reference when the binary output is decoded. 466 // 467 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ %a\n", x 468 #else 469 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ 0x%016x\n", x 470 #endif // NVCPU_IS_RISCV64 471 #else 472 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ %s:%u (%s)\n", x.file, x.line, x.func 473 #endif // PORT_MEM_TRACK_USE_CALLERINFO_IP 474 475 #else // PORT_MEM_TRACK_USE_CALLERINFO 476 #define PORT_MEM_CALLERINFO_INIT_TRACKING(x) 477 #define PORT_MEM_CALLERINFO_INIT_MEM(x) 478 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "\n" 479 #endif // PORT_MEM_TRACK_USE_CALLERINFO 480 481 482 #if PORT_MEM_TRACK_USE_LOGGING 483 #include "nvlog/nvlog.h" 484 /** @brief Single log entry. Uses 64bit values even on 32bit systems. */ 485 typedef struct PORT_MEM_LOG_ENTRY 486 { 487 NvP64 address; 488 NvP64 allocator; 489 NvLength size; // if size is 0, it is a free() call, not alloc() 490 } PORT_MEM_LOG_ENTRY; 491 492 #define PORT_MEM_TRACK_LOG_TAG 0x70726d74 493 #define PORT_MEM_LOG_ENTRIES 4096 494 495 static void 496 _portMemLogInit(void) 497 { 498 NVLOG_BUFFER_HANDLE hBuffer; 499 nvlogAllocBuffer(PORT_MEM_LOG_ENTRIES * sizeof(PORT_MEM_LOG_ENTRY), 500 DRF_DEF(LOG, _BUFFER_FLAGS, _FORMAT, _MEMTRACK), 501 PORT_MEM_TRACK_LOG_TAG, &hBuffer); 502 } 503 504 static void 505 _portMemLogDestroy(void) 506 { 507 NVLOG_BUFFER_HANDLE hBuffer; 508 nvlogGetBufferHandleFromTag(PORT_MEM_TRACK_LOG_TAG, &hBuffer); 509 nvlogDeallocBuffer(hBuffer); 510 } 511 512 static void 513 _portMemLogAdd 514 ( 515 PORT_MEM_ALLOCATOR *pAllocator, 516 void *pMem, 517 NvLength lengthBytes 518 ) 519 { 520 NVLOG_BUFFER_HANDLE hBuffer; 521 PORT_MEM_LOG_ENTRY entry = {0}; 522 entry.address = NV_PTR_TO_NvP64(pMem); 523 entry.address = NV_PTR_TO_NvP64(pAllocator); 524 entry.size = lengthBytes; 525 nvlogGetBufferHandleFromTag(PORT_MEM_TRACK_LOG_TAG, &hBuffer); 526 nvlogWriteToBuffer(hBuffer, &entry, sizeof(entry)); 527 } 528 529 #define PORT_MEM_LOG_INIT() _portMemLogInit() 530 #define PORT_MEM_LOG_DESTROY() _portMemLogDestroy() 531 #define PORT_MEM_LOG_ALLOC(pAlloc, pMem, size) \ 532 _portMemLogAdd(pAlloc, pMem, size) 533 #define PORT_MEM_LOG_FREE(pAlloc, pMem) \ 534 _portMemLogAdd(pAlloc, pMem, 0) 535 #else 536 #define PORT_MEM_LOG_INIT() 537 #define PORT_MEM_LOG_DESTROY() 538 #define PORT_MEM_LOG_ALLOC(x, y, z) 539 #define PORT_MEM_LOG_FREE(x, y) 540 #endif // LOGGING 541 542 543 //////////////////////////////////////////////////////////////////////////////// 544 // 545 // Main memory tracking implementation 546 // 547 //////////////////////////////////////////////////////////////////////////////// 548 549 // 550 // All static functions declarations. Definitions at the end of file. 551 // 552 static void *_portMemAllocatorAllocPagedWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 553 static void *_portMemAllocatorAllocNonPagedWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 554 static void _portMemAllocatorFreeWrapper(PORT_MEM_ALLOCATOR *pAlloc, void *pMem); 555 static void _portMemAllocatorReleaseWrapper(PORT_MEM_ALLOCATOR *pAlloc); 556 557 static PORT_MEM_ALLOCATOR *_portMemAllocatorCreateOnExistingBlock(void *pAlloc, NvLength blockSizeBytes, void *pSpinlock PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM); 558 static void *_portMemAllocatorAllocExistingWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 559 static void _portMemAllocatorFreeExistingWrapper(PORT_MEM_ALLOCATOR *pAlloc, void *pMem); 560 561 static void _portMemTrackingRelease(PORT_MEM_ALLOCATOR_TRACKING *pTracking, NvBool bReportLeaks); 562 static void _portMemTrackAlloc(PORT_MEM_ALLOCATOR_TRACKING *pTracking, void *pMem, NvLength size, NvU32 pid PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM); 563 static void _portMemTrackFree(PORT_MEM_ALLOCATOR_TRACKING *pTracking, void *pMem); 564 565 566 567 #if PORT_MEM_TRACK_USE_CALLERINFO 568 #undef portMemAllocPaged 569 #undef portMemAllocNonPaged 570 #undef portMemAllocatorCreatePaged 571 #undef portMemAllocatorCreateNonPaged 572 #undef portMemInitializeAllocatorTracking 573 #undef _portMemAllocatorAlloc 574 #undef portMemAllocatorCreateOnExistingBlock 575 #undef portMemExAllocatorCreateLockedOnExistingBlock 576 // These functions have different names if CallerInfo is enabled. 577 #define portMemAllocPaged portMemAllocPaged_CallerInfo 578 #define portMemAllocNonPaged portMemAllocNonPaged_CallerInfo 579 #define portMemAllocatorCreatePaged portMemAllocatorCreatePaged_CallerInfo 580 #define portMemAllocatorCreateNonPaged portMemAllocatorCreateNonPaged_CallerInfo 581 #define portMemInitializeAllocatorTracking portMemInitializeAllocatorTracking_CallerInfo 582 #define _portMemAllocatorAlloc _portMemAllocatorAlloc_CallerInfo 583 #define portMemAllocatorCreateOnExistingBlock portMemAllocatorCreateOnExistingBlock_CallerInfo 584 #define portMemExAllocatorCreateLockedOnExistingBlock portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo 585 #endif 586 587 // 588 // Per-process heap limiting implementation 589 // 590 #if PORT_MEM_TRACK_USE_LIMIT 591 static NV_INLINE void 592 _portMemLimitInc(NvU32 pid, void *pMem, NvLength size) 593 { 594 PORT_MEM_HEADER *pMemHeader = PORT_MEM_SUB_HEADER_PTR(pMem); 595 pMemHeader->pid = pid; 596 if (portMemGlobals.bLimitEnabled) 597 { 598 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS)) 599 { 600 NvU32 pidIdx = pid - 1; 601 PORT_MEM_ATOMIC_ADD_SIZE(&portMemGlobals.counterPid[pidIdx], size); 602 } 603 } 604 } 605 606 static NV_INLINE void 607 _portMemLimitDec(void *pMem, NvLength size) 608 { 609 if (portMemGlobals.bLimitEnabled) 610 { 611 PORT_MEM_HEADER *pMemHeader = PORT_MEM_SUB_HEADER_PTR(pMem); 612 NvU32 pid = pMemHeader->pid; 613 614 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS)) 615 { 616 NvU32 pidIdx = pid - 1; 617 if (portMemGlobals.counterPid[pidIdx] < size) 618 { 619 PORT_MEM_PRINT_ERROR("memory free error: counter underflow\n"); 620 PORT_BREAKPOINT_CHECKED(); 621 } 622 else 623 { 624 PORT_MEM_ATOMIC_SUB_SIZE(&portMemGlobals.counterPid[pidIdx], size); 625 } 626 } 627 } 628 } 629 630 static NV_INLINE NvBool 631 _portMemLimitExceeded(NvU32 pid, NvLength size) 632 { 633 NvBool bExceeded = NV_FALSE; 634 635 if (portMemGlobals.bLimitEnabled) 636 { 637 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS)) 638 { 639 NvU32 pidIdx = pid - 1; 640 if ((size + portMemGlobals.counterPid[pidIdx]) > portMemGlobals.limitPid[pidIdx]) 641 { 642 PORT_MEM_PRINT_ERROR("memory allocation denied; PID %d exceeded per-process heap limit of %"NvUPtr_fmtu"\n", 643 pid, portMemGlobals.limitPid[pidIdx]); 644 bExceeded = NV_TRUE; 645 } 646 } 647 } 648 return bExceeded; 649 } 650 651 #define PORT_MEM_LIMIT_INC(pid, pMem, size) _portMemLimitInc(pid, pMem, size) 652 #define PORT_MEM_LIMIT_DEC(pMem, size) _portMemLimitDec(pMem, size) 653 #define PORT_MEM_LIMIT_EXCEEDED(pid, size) _portMemLimitExceeded(pid, size) 654 #else 655 #define PORT_MEM_LIMIT_INC(pid, pMem, size) \ 656 do { \ 657 PORT_UNREFERENCED_VARIABLE(pid); \ 658 PORT_UNREFERENCED_VARIABLE(pMem); \ 659 PORT_UNREFERENCED_VARIABLE(size); \ 660 } while (0) 661 #define PORT_MEM_LIMIT_DEC(pMem, size) \ 662 do { \ 663 PORT_UNREFERENCED_VARIABLE(pMem); \ 664 PORT_UNREFERENCED_VARIABLE(size); \ 665 } while (0) 666 #define PORT_MEM_LIMIT_EXCEEDED(pid, size) (NV_FALSE) 667 #endif // PORT_MEM_TRACK_USE_LIMIT 668 669 static NV_INLINE PORT_MEM_ALLOCATOR_TRACKING * 670 _portMemGetTracking 671 ( 672 const PORT_MEM_ALLOCATOR *pAlloc 673 ) 674 { 675 if (pAlloc == NULL) 676 return &portMemGlobals.mainTracking; 677 else 678 return pAlloc->pTracking; 679 } 680 681 682 void 683 portMemInitialize(void) 684 { 685 #if PORT_MEM_TRACK_USE_CALLERINFO 686 PORT_MEM_CALLERINFO_TYPE_PARAM = PORT_MEM_CALLERINFO_MAKE; 687 #endif 688 if (PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.initCount) != 1) 689 return; 690 691 portMemGlobals.mainTracking.pAllocator = NULL; 692 portMemGlobals.mainTracking.pNext = &portMemGlobals.mainTracking; 693 portMemGlobals.mainTracking.pPrev = &portMemGlobals.mainTracking; 694 PORT_MEM_COUNTER_INIT(&portMemGlobals.mainTracking.counter); 695 PORT_MEM_LIST_INIT(&portMemGlobals.mainTracking); 696 PORT_MEM_LOCK_INIT(portMemGlobals.trackingLock); 697 698 #if PORT_MEM_TRACK_USE_LIMIT 699 // Initialize process heap limit to max int (i.e. no limit) 700 portMemGlobals.bLimitEnabled = NV_FALSE; 701 portMemSet(&portMemGlobals.limitPid, NV_U8_MAX, sizeof(portMemGlobals.limitPid)); 702 portMemSet(&portMemGlobals.counterPid, 0, sizeof(portMemGlobals.counterPid)); 703 #endif 704 705 portMemGlobals.alloc.paged._portAlloc = _portMemAllocatorAllocPagedWrapper; 706 portMemGlobals.alloc.nonPaged._portAlloc = _portMemAllocatorAllocNonPagedWrapper; 707 portMemGlobals.alloc.paged._portFree = _portMemAllocatorFreeWrapper; 708 portMemGlobals.alloc.nonPaged._portFree = _portMemAllocatorFreeWrapper; 709 portMemGlobals.alloc.paged._portRelease = NULL; 710 portMemGlobals.alloc.nonPaged._portRelease = NULL; 711 712 if (PORT_MEM_TRACK_USE_FENCEPOSTS) 713 { 714 // 715 // Distinct paged and non-paged allocators require PORT_MEM_TRACK_USE_FENCEPOSTS 716 // so that the correct allocator can be looked up from the fenceposts in the 717 // portMemFree path. 718 // 719 portMemGlobals.alloc.paged.pImpl = &portMemGlobals.alloc.pagedImpl; 720 portMemGlobals.alloc.nonPaged.pImpl = &portMemGlobals.alloc.nonPagedImpl; 721 722 portMemInitializeAllocatorTracking(&portMemGlobals.alloc.paged, 723 &portMemGlobals.alloc.paged.pImpl->tracking 724 PORT_MEM_CALLERINFO_COMMA_PARAM); 725 portMemInitializeAllocatorTracking(&portMemGlobals.alloc.nonPaged, 726 &portMemGlobals.alloc.nonPaged.pImpl->tracking 727 PORT_MEM_CALLERINFO_COMMA_PARAM); 728 } 729 else 730 { 731 // Use the same impl for both paged and nonpaged. 732 portMemGlobals.alloc.paged.pImpl = &portMemGlobals.alloc.pagedImpl; 733 portMemGlobals.alloc.nonPaged.pImpl = &portMemGlobals.alloc.pagedImpl; 734 portMemInitializeAllocatorTracking(NULL, 735 &portMemGlobals.alloc.pagedImpl.tracking 736 PORT_MEM_CALLERINFO_COMMA_PARAM); 737 portMemGlobals.alloc.paged.pTracking = &portMemGlobals.alloc.pagedImpl.tracking; 738 portMemGlobals.alloc.nonPaged.pTracking = &portMemGlobals.alloc.pagedImpl.tracking; 739 } 740 PORT_MEM_LOG_INIT(); 741 } 742 void 743 portMemShutdown(NvBool bForceSilent) 744 { 745 PORT_UNREFERENCED_VARIABLE(bForceSilent); 746 if (PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.initCount) != 0) 747 return; 748 749 #if (PORT_MEM_TRACK_PRINT_LEVEL > PORT_MEM_TRACK_PRINT_LEVEL_SILENT) 750 if (!bForceSilent) 751 { 752 portMemPrintAllTrackingInfo(); 753 } 754 #endif 755 PORT_MEM_LOG_DESTROY(); 756 757 if (PORT_MEM_TRACK_USE_FENCEPOSTS) 758 { 759 _portMemTrackingRelease(&portMemGlobals.alloc.nonPaged.pImpl->tracking, NV_FALSE); 760 _portMemTrackingRelease(&portMemGlobals.alloc.paged.pImpl->tracking, NV_FALSE); 761 } 762 else 763 { 764 _portMemTrackingRelease(&portMemGlobals.alloc.pagedImpl.tracking, NV_FALSE); 765 } 766 767 PORT_MEM_LOCK_DESTROY(portMemGlobals.trackingLock); 768 PORT_MEM_LIST_DESTROY(&portMemGlobals.mainTracking); 769 portMemSet(&portMemGlobals, 0, sizeof(portMemGlobals)); 770 } 771 772 void * 773 portMemAllocPaged 774 ( 775 NvLength length 776 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 777 ) 778 { 779 #if defined(__COVERITY__) 780 return __coverity_alloc__(length); 781 #endif 782 PORT_MEM_ALLOCATOR *pAlloc = portMemAllocatorGetGlobalPaged(); 783 return _portMemAllocatorAlloc(pAlloc, length PORT_MEM_CALLERINFO_COMMA_PARAM); 784 } 785 786 void * 787 portMemAllocNonPaged 788 ( 789 NvLength length 790 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 791 ) 792 { 793 #if defined(__COVERITY__) 794 return __coverity_alloc__(length); 795 #endif 796 PORT_MEM_ALLOCATOR *pAlloc = portMemAllocatorGetGlobalNonPaged(); 797 return _portMemAllocatorAlloc(pAlloc, length PORT_MEM_CALLERINFO_COMMA_PARAM); 798 } 799 800 void 801 portMemFree 802 ( 803 void *pMem 804 ) 805 { 806 if (pMem != NULL) 807 { 808 #if PORT_MEM_TRACK_USE_FENCEPOSTS 809 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1; 810 PORT_FREE(pHead->fence.pAllocator, pMem); 811 #else 812 // Paged/nonpaged are logged together if we don't have fenceposts. 813 PORT_FREE(portMemAllocatorGetGlobalPaged(), pMem); 814 #endif 815 } 816 817 #if defined(__COVERITY__) 818 __coverity_free__(pMem); 819 #endif 820 } 821 822 PORT_MEM_ALLOCATOR * 823 portMemAllocatorCreatePaged(PORT_MEM_CALLERINFO_TYPE_PARAM) 824 { 825 PORT_MEM_ALLOCATOR *pAllocator; 826 827 pAllocator = portMemAllocPaged(PORT_MEM_ALLOCATOR_SIZE 828 PORT_MEM_CALLERINFO_COMMA_PARAM); 829 if (pAllocator == NULL) 830 return NULL; 831 832 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1); 833 pAllocator->_portAlloc = _portMemAllocatorAllocPagedWrapper; 834 pAllocator->_portFree = _portMemAllocatorFreeWrapper; 835 pAllocator->_portRelease = _portMemAllocatorReleaseWrapper; 836 portMemInitializeAllocatorTracking(pAllocator, &pAllocator->pImpl->tracking 837 PORT_MEM_CALLERINFO_COMMA_PARAM); 838 839 PORT_MEM_PRINT_INFO("Acquired paged allocator %p ", pAllocator); 840 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM)); 841 842 return pAllocator; 843 } 844 845 PORT_MEM_ALLOCATOR * 846 portMemAllocatorCreateNonPaged(PORT_MEM_CALLERINFO_TYPE_PARAM) 847 { 848 PORT_MEM_ALLOCATOR *pAllocator; 849 850 pAllocator = portMemAllocNonPaged(PORT_MEM_ALLOCATOR_SIZE 851 PORT_MEM_CALLERINFO_COMMA_PARAM); 852 if (pAllocator == NULL) 853 return NULL; 854 855 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1); 856 pAllocator->_portAlloc = _portMemAllocatorAllocNonPagedWrapper; 857 pAllocator->_portFree = _portMemAllocatorFreeWrapper; 858 pAllocator->_portRelease = _portMemAllocatorReleaseWrapper; 859 portMemInitializeAllocatorTracking(pAllocator, &pAllocator->pImpl->tracking 860 PORT_MEM_CALLERINFO_COMMA_PARAM); 861 862 PORT_MEM_PRINT_INFO("Acquired nonpaged allocator %p ", pAllocator); 863 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM)); 864 return pAllocator; 865 } 866 867 868 PORT_MEM_ALLOCATOR * 869 portMemAllocatorCreateOnExistingBlock 870 ( 871 void *pPreallocatedBlock, 872 NvLength blockSizeBytes 873 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 874 ) 875 { 876 return _portMemAllocatorCreateOnExistingBlock(pPreallocatedBlock, blockSizeBytes, 877 NULL PORT_MEM_CALLERINFO_COMMA_PARAM); 878 } 879 880 PORT_MEM_ALLOCATOR * 881 portMemExAllocatorCreateLockedOnExistingBlock 882 ( 883 void *pPreallocatedBlock, 884 NvLength blockSizeBytes, 885 void *pSpinlock 886 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 887 ) 888 { 889 return _portMemAllocatorCreateOnExistingBlock(pPreallocatedBlock, blockSizeBytes, 890 pSpinlock PORT_MEM_CALLERINFO_COMMA_PARAM); 891 } 892 893 void 894 portMemAllocatorRelease 895 ( 896 PORT_MEM_ALLOCATOR *pAllocator 897 ) 898 { 899 if (pAllocator == NULL) 900 { 901 PORT_BREAKPOINT_CHECKED(); 902 return; 903 } 904 _portMemTrackingRelease(pAllocator->pTracking, NV_TRUE); 905 PORT_MEM_PRINT_INFO("Released allocator %p\n", pAllocator); 906 907 if (pAllocator->_portRelease != NULL) 908 pAllocator->_portRelease(pAllocator); 909 } 910 911 912 PORT_MEM_ALLOCATOR * 913 portMemAllocatorGetGlobalNonPaged(void) 914 { 915 return &portMemGlobals.alloc.nonPaged; 916 } 917 PORT_MEM_ALLOCATOR * 918 portMemAllocatorGetGlobalPaged(void) 919 { 920 return &portMemGlobals.alloc.paged; 921 } 922 923 void 924 portMemInitializeAllocatorTracking 925 ( 926 PORT_MEM_ALLOCATOR *pAlloc, 927 PORT_MEM_ALLOCATOR_TRACKING *pTracking 928 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 929 ) 930 { 931 if (portMemGlobals.initCount == 0) 932 { 933 portMemSet(pTracking, 0, sizeof(*pTracking)); 934 if (pAlloc != NULL) 935 pAlloc->pTracking = NULL; 936 return; 937 } 938 939 pTracking->pAllocator = pAlloc; 940 if (pAlloc != NULL) 941 pAlloc->pTracking = pTracking; 942 PORT_LOCKED_LIST_LINK(&portMemGlobals.mainTracking, pTracking, portMemGlobals.trackingLock); 943 PORT_MEM_COUNTER_INIT(&pTracking->counter); 944 PORT_MEM_LIST_INIT(pTracking); 945 PORT_MEM_CALLERINFO_INIT_TRACKING(pTracking); 946 PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.totalAllocators); 947 } 948 949 #if PORT_MEM_TRACK_USE_LIMIT 950 void 951 portMemInitializeAllocatorTrackingLimit(NvU32 pid, NvLength limit, NvBool bLimitEnabled) 952 { 953 NvU32 pidIdx = pid - 1; 954 portMemGlobals.limitPid[pidIdx] = limit; 955 portMemGlobals.bLimitEnabled = bLimitEnabled; 956 } 957 #endif 958 959 void * 960 _portMemAllocatorAlloc 961 ( 962 PORT_MEM_ALLOCATOR *pAlloc, 963 NvLength length 964 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 965 ) 966 { 967 NvU32 pid = 0; 968 void *pMem = NULL; 969 if (pAlloc == NULL) 970 { 971 PORT_BREAKPOINT_CHECKED(); 972 return NULL; 973 } 974 975 #if PORT_MEM_TRACK_USE_LIMIT 976 if (portMemGlobals.bLimitEnabled) 977 { 978 if (osGetCurrentProcessGfid(&pid) != NV_OK) 979 { 980 PORT_BREAKPOINT_CHECKED(); 981 return NULL; 982 } 983 } 984 #endif 985 986 // Check if per-process memory limit will be exhausted by this allocation 987 if (PORT_MEM_LIMIT_EXCEEDED(pid, length)) 988 return NULL; 989 990 if (length > 0) 991 { 992 NvLength paddedLength; 993 // RISCV64 requires 64-bit alignment of structures, and length indicates the alignment of the footer 994 #if defined(__riscv) 995 if (PORT_MEM_STAGING_SIZE > 0 && (length & 7)) 996 { 997 if (!portSafeAddLength(length & ~7, 8, &length)) 998 { 999 return NULL; 1000 } 1001 } 1002 #endif 1003 if (!portSafeAddLength(length, PORT_MEM_STAGING_SIZE, &paddedLength)) 1004 { 1005 return NULL; 1006 } 1007 pMem = pAlloc->_portAlloc(pAlloc, paddedLength); 1008 } 1009 if (pMem != NULL) 1010 { 1011 pMem = PORT_MEM_ADD_HEADER_PTR(pMem); 1012 _portMemTrackAlloc(_portMemGetTracking(pAlloc), pMem, length, pid 1013 PORT_MEM_CALLERINFO_COMMA_PARAM); 1014 } 1015 return pMem; 1016 } 1017 void 1018 _portMemAllocatorFree 1019 ( 1020 PORT_MEM_ALLOCATOR *pAlloc, 1021 void *pMem 1022 ) 1023 { 1024 if (pAlloc == NULL) 1025 { 1026 PORT_BREAKPOINT_CHECKED(); 1027 return; 1028 } 1029 if (pMem != NULL) 1030 { 1031 _portMemTrackFree(_portMemGetTracking(pAlloc), pMem); 1032 pMem = PORT_MEM_SUB_HEADER_PTR(pMem); 1033 pAlloc->_portFree(pAlloc, pMem); 1034 } 1035 } 1036 1037 void 1038 portMemPrintTrackingInfo 1039 ( 1040 const PORT_MEM_ALLOCATOR_TRACKING *pTracking 1041 ) 1042 { 1043 if (pTracking == NULL) 1044 pTracking = &portMemGlobals.mainTracking; 1045 1046 if (pTracking == &portMemGlobals.mainTracking) 1047 portDbgPrintf("[NvPort] ******** Aggregate Memory Tracking ********\n"); 1048 else if ((pTracking == portMemGlobals.alloc.nonPaged.pTracking) && 1049 (pTracking == portMemGlobals.alloc.paged.pTracking)) 1050 portDbgPrintf("[NvPort] ******** Global Allocator Tracking ********\n"); 1051 else if (pTracking == portMemGlobals.alloc.nonPaged.pTracking) 1052 portDbgPrintf("[NvPort] ******** Global Non-Paged Memory Allocator Tracking ********\n"); 1053 else if (pTracking == portMemGlobals.alloc.paged.pTracking) 1054 portDbgPrintf("[NvPort] ******** Global Paged Memory Allocator Tracking ********\n"); 1055 else 1056 portDbgPrintf("[NvPort] ******** Memory Allocator %p Tracking ******** \n", pTracking->pAllocator); 1057 1058 if (pTracking->counter.activeAllocs != 0) 1059 portDbgPrintf(" !!! MEMORY LEAK DETECTED (%u blocks) !!!\n", 1060 pTracking->counter.activeAllocs); 1061 1062 #if PORT_MEM_TRACK_USE_CALLERINFO 1063 { 1064 portDbgPrintf(" Allocator acquired " 1065 PORT_MEM_CALLERINFO_PRINT_ARGS(pTracking->callerInfo)); 1066 } 1067 #endif 1068 1069 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetHeapSize) 1070 // 1071 // Heap is shared across all allocators, so only print it with the 1072 // aggregate stats. 1073 // 1074 if (pTracking == _portMemGetTracking(NULL)) 1075 portDbgPrintf(" HEAP: %"NvUPtr_fmtu" bytes\n", portMemExTrackingGetHeapSize()); 1076 #endif 1077 1078 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetActiveStats) 1079 { 1080 PORT_MEM_TRACK_ALLOCATOR_STATS stats; 1081 1082 portMemSet(&stats, 0, sizeof(stats)); 1083 1084 portMemExTrackingGetActiveStats(pTracking->pAllocator, &stats); 1085 1086 // 1087 // rmtest_gsp test script (dvs_gsp_sanity.sh) depends on this print, so do not change 1088 // format without updating script! 1089 // 1090 portDbgPrintf(" ACTIVE: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n", 1091 stats.numAllocations, 1092 stats.allocatedSize, 1093 stats.usefulSize, 1094 stats.metaSize); 1095 } 1096 #endif 1097 1098 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetTotalStats) 1099 { 1100 PORT_MEM_TRACK_ALLOCATOR_STATS stats; 1101 1102 portMemSet(&stats, 0, sizeof(stats)); 1103 1104 portMemExTrackingGetTotalStats(pTracking->pAllocator, &stats); 1105 portDbgPrintf(" TOTAL: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n", 1106 stats.numAllocations, 1107 stats.allocatedSize, 1108 stats.usefulSize, 1109 stats.metaSize); 1110 } 1111 #endif 1112 1113 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetPeakStats) 1114 { 1115 PORT_MEM_TRACK_ALLOCATOR_STATS stats; 1116 1117 portMemSet(&stats, 0, sizeof(stats)); 1118 1119 portMemExTrackingGetPeakStats(pTracking->pAllocator, &stats); 1120 portDbgPrintf(" PEAK: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n", 1121 stats.numAllocations, 1122 stats.allocatedSize, 1123 stats.usefulSize, 1124 stats.metaSize); 1125 } 1126 #endif 1127 1128 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetNext) 1129 { 1130 PORT_MEM_TRACK_ALLOC_INFO info; 1131 NvBool bPrinted = NV_FALSE; 1132 void *iterator = NULL; 1133 1134 do 1135 { 1136 if (portMemExTrackingGetNext(pTracking->pAllocator, &info, &iterator) != NV_OK) 1137 { 1138 portDbgPrintf(" (no active allocations)\n"); 1139 break; 1140 } 1141 else if (!bPrinted) 1142 { 1143 portDbgPrintf(" Currently active allocations:\n"); 1144 bPrinted = NV_TRUE; 1145 } 1146 portDbgPrintf(" - A:%p - 0x%p [%8"NvUPtr_fmtu" bytes] T=%llu ", 1147 info.pAllocator, 1148 info.pMemory, 1149 info.size, 1150 info.timestamp); 1151 portDbgPrintf(PORT_MEM_CALLERINFO_PRINT_ARGS(info.callerInfo)); 1152 } while (iterator != NULL); 1153 } 1154 #endif 1155 } 1156 1157 void 1158 portMemPrintAllTrackingInfo(void) 1159 { 1160 const PORT_MEM_ALLOCATOR_TRACKING *pTracking = &portMemGlobals.mainTracking; 1161 PORT_MEM_LOCK_ACQUIRE(portMemGlobals.trackingLock); 1162 do 1163 { 1164 portMemPrintTrackingInfo(pTracking); 1165 } while ((pTracking = pTracking->pNext) != &portMemGlobals.mainTracking); 1166 PORT_MEM_LOCK_RELEASE(portMemGlobals.trackingLock); 1167 } 1168 1169 #if portMemExTrackingGetActiveStats_SUPPORTED 1170 NV_STATUS 1171 portMemExTrackingGetActiveStats 1172 ( 1173 const PORT_MEM_ALLOCATOR *pAllocator, 1174 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 1175 ) 1176 { 1177 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator); 1178 if (pTracking == NULL) 1179 { 1180 return NV_ERR_OBJECT_NOT_FOUND; 1181 } 1182 pStats->numAllocations = pTracking->counter.activeAllocs; 1183 pStats->usefulSize = pTracking->counter.activeSize; 1184 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE; 1185 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize; 1186 return NV_OK; 1187 } 1188 #endif 1189 1190 #if portMemExTrackingGetTotalStats_SUPPORTED 1191 NV_STATUS 1192 portMemExTrackingGetTotalStats 1193 ( 1194 const PORT_MEM_ALLOCATOR *pAllocator, 1195 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 1196 ) 1197 { 1198 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator); 1199 if (pTracking == NULL) 1200 { 1201 return NV_ERR_OBJECT_NOT_FOUND; 1202 } 1203 pStats->numAllocations = pTracking->counter.totalAllocs; 1204 pStats->usefulSize = pTracking->counter.totalSize; 1205 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE; 1206 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize; 1207 return NV_OK; 1208 } 1209 #endif 1210 1211 #if portMemExTrackingGetPeakStats_SUPPORTED 1212 NV_STATUS 1213 portMemExTrackingGetPeakStats 1214 ( 1215 const PORT_MEM_ALLOCATOR *pAllocator, 1216 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats 1217 ) 1218 { 1219 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator); 1220 if (pTracking == NULL) 1221 { 1222 return NV_ERR_OBJECT_NOT_FOUND; 1223 } 1224 pStats->numAllocations = pTracking->counter.peakAllocs; 1225 pStats->usefulSize = pTracking->counter.peakSize; 1226 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE; 1227 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize; 1228 return NV_OK; 1229 } 1230 #endif 1231 1232 #if portMemExTrackingGetNext_SUPPORTED 1233 NV_STATUS 1234 portMemExTrackingGetNext 1235 ( 1236 const PORT_MEM_ALLOCATOR *pAllocator, 1237 PORT_MEM_TRACK_ALLOC_INFO *pInfo, 1238 void **pIterator 1239 ) 1240 { 1241 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator); 1242 PORT_MEM_LIST *pList; 1243 PORT_MEM_HEADER *pHead; 1244 1245 if (pTracking == NULL) 1246 { 1247 return NV_ERR_OBJECT_NOT_FOUND; 1248 } 1249 1250 if (pTracking->pFirstAlloc == NULL) 1251 return NV_ERR_OBJECT_NOT_FOUND; 1252 1253 if (*pIterator == NULL) 1254 pList = pTracking->pFirstAlloc; 1255 else 1256 pList = (PORT_MEM_LIST*)(*pIterator); 1257 1258 pHead = _portMemListGetHeader(pList); 1259 1260 // Advance iterator 1261 if (pList->pNext == pTracking->pFirstAlloc) 1262 *pIterator = NULL; 1263 else 1264 *pIterator = pList->pNext; 1265 1266 // Populate pInfo 1267 pInfo->pMemory = pHead + 1; 1268 pInfo->size = _portMemExTrackingGetAllocUsableSizeWrapper(pInfo->pMemory); 1269 pInfo->pAllocator = pHead->fence.pAllocator; 1270 pInfo->timestamp = 0; 1271 1272 #if PORT_MEM_TRACK_USE_CALLERINFO 1273 pInfo->callerInfo = pHead->callerInfo; 1274 #endif 1275 1276 return NV_OK; 1277 } 1278 #endif 1279 1280 static void 1281 _portMemTrackingRelease 1282 ( 1283 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 1284 NvBool bReportLeaks 1285 ) 1286 { 1287 if (pTracking == NULL) return; 1288 1289 #if (PORT_MEM_TRACK_PRINT_LEVEL > PORT_MEM_TRACK_PRINT_LEVEL_SILENT) 1290 if (bReportLeaks && (pTracking->counter.activeAllocs != 0)) 1291 portMemPrintTrackingInfo(pTracking); 1292 #else 1293 PORT_UNREFERENCED_VARIABLE(bReportLeaks); 1294 #endif 1295 1296 PORT_LOCKED_LIST_UNLINK(&portMemGlobals.mainTracking, pTracking, portMemGlobals.trackingLock); 1297 PORT_MEM_LIST_DESTROY(pTracking); 1298 PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.totalAllocators); 1299 } 1300 1301 static void 1302 _portMemTrackAlloc 1303 ( 1304 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 1305 void *pMem, 1306 NvLength size, 1307 NvU32 pid 1308 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 1309 ) 1310 { 1311 PORT_UNREFERENCED_VARIABLE(pMem); 1312 if (pTracking == NULL) return; 1313 1314 #if PORT_MEM_TRACK_ALLOC_SIZE 1315 // 1316 // Either set the block size in the header, or override it with the value 1317 // from the underlying allocator (which may be bigger than what was 1318 // requested). This keeps the counters consistent with the free path. 1319 // 1320 _portMemExTrackingSetOrGetAllocUsableSize(pMem, &size); 1321 #endif 1322 1323 PORT_MEM_PRINT_INFO("Allocated %"NvUPtr_fmtu" bytes at address %p", size, pMem); 1324 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM)); 1325 1326 PORT_MEM_COUNTER_INC(&pTracking->counter, size); 1327 PORT_MEM_COUNTER_INC(&portMemGlobals.mainTracking.counter, size); 1328 PORT_MEM_LIMIT_INC(pid, pMem, size); 1329 1330 PORT_MEM_FENCE_INIT(pTracking->pAllocator, pMem, size); 1331 PORT_MEM_LIST_ADD(pTracking, pMem); 1332 PORT_MEM_CALLERINFO_INIT_MEM(pMem); 1333 PORT_MEM_LOG_ALLOC(pTracking->pAllocator, pMem, size); 1334 } 1335 1336 static void 1337 _portMemTrackFree 1338 ( 1339 PORT_MEM_ALLOCATOR_TRACKING *pTracking, 1340 void *pMem 1341 ) 1342 { 1343 NvLength size = 0; 1344 1345 if (pTracking == NULL) return; 1346 1347 #if PORT_MEM_TRACK_ALLOC_SIZE 1348 size = _portMemExTrackingGetAllocUsableSizeWrapper(pMem); 1349 PORT_MEM_PRINT_INFO("Freeing %"NvUPtr_fmtu"-byte block at address %p\n", size, pMem); 1350 #else 1351 PORT_MEM_PRINT_INFO("Freeing block at address %p\n", pMem); 1352 #endif 1353 1354 PORT_MEM_COUNTER_DEC(&pTracking->counter, size); 1355 PORT_MEM_COUNTER_DEC(&portMemGlobals.mainTracking.counter, size); 1356 PORT_MEM_LIMIT_DEC(pMem, size); 1357 1358 PORT_MEM_FENCE_CHECK(pTracking->pAllocator, pMem, size); 1359 PORT_MEM_LIST_REMOVE(pTracking, pMem); 1360 PORT_MEM_LOG_FREE(pTracking->pAllocator, pMem); 1361 } 1362 1363 1364 static void * 1365 _portMemAllocatorAllocPagedWrapper 1366 ( 1367 PORT_MEM_ALLOCATOR *pAlloc, 1368 NvLength length 1369 ) 1370 { 1371 PORT_UNREFERENCED_VARIABLE(pAlloc); 1372 return _portMemAllocPagedUntracked(length); 1373 } 1374 1375 static void * 1376 _portMemAllocatorAllocNonPagedWrapper 1377 ( 1378 PORT_MEM_ALLOCATOR *pAlloc, 1379 NvLength length 1380 ) 1381 { 1382 PORT_UNREFERENCED_VARIABLE(pAlloc); 1383 return _portMemAllocNonPagedUntracked(length); 1384 } 1385 1386 static void 1387 _portMemAllocatorFreeWrapper 1388 ( 1389 PORT_MEM_ALLOCATOR *pAlloc, 1390 void *pMem 1391 ) 1392 { 1393 PORT_UNREFERENCED_VARIABLE(pAlloc); 1394 _portMemFreeUntracked(pMem); 1395 } 1396 1397 static void 1398 _portMemAllocatorReleaseWrapper 1399 ( 1400 PORT_MEM_ALLOCATOR *pAllocator 1401 ) 1402 { 1403 portMemFree(pAllocator); 1404 } 1405 1406 /// @todo Add these as intrinsics to UTIL module 1407 static NV_INLINE NvBool _isBitSet(NvU32 *vect, NvU32 bit) 1408 { 1409 return !!(vect[bit/32] & NVBIT32(bit%32)); 1410 } 1411 static NV_INLINE void _setBit(NvU32 *vect, NvU32 bit) 1412 { 1413 vect[bit/32] |= NVBIT32(bit%32); 1414 } 1415 static NV_INLINE void _clearBit(NvU32 *vect, NvU32 bit) 1416 { 1417 vect[bit/32] &= ~NVBIT32(bit%32); 1418 } 1419 1420 static PORT_MEM_ALLOCATOR * 1421 _portMemAllocatorCreateOnExistingBlock 1422 ( 1423 void *pPreallocatedBlock, 1424 NvLength blockSizeBytes, 1425 void *pSpinlock 1426 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 1427 ) 1428 { 1429 PORT_MEM_ALLOCATOR *pAllocator = (PORT_MEM_ALLOCATOR *)pPreallocatedBlock; 1430 PORT_MEM_BITVECTOR *pBitVector; 1431 PORT_MEM_BITVECTOR_CHUNK *pLastChunkInBlock; 1432 NvU32 bitVectorSize; 1433 1434 if ((pPreallocatedBlock == NULL) || 1435 (blockSizeBytes < PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE) || 1436 (blockSizeBytes > NV_S32_MAX)) 1437 { 1438 return NULL; 1439 } 1440 1441 pAllocator->_portAlloc = _portMemAllocatorAllocExistingWrapper; 1442 pAllocator->_portFree = _portMemAllocatorFreeExistingWrapper; 1443 pAllocator->_portRelease = NULL; 1444 pAllocator->pTracking = NULL; // No tracking for this allocator 1445 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1); 1446 1447 pBitVector = (PORT_MEM_BITVECTOR*)(pAllocator->pImpl); 1448 pBitVector->pSpinlock = pSpinlock; 1449 1450 // Calculate total number of chunks available 1451 pBitVector->pChunks = (PORT_MEM_BITVECTOR_CHUNK *)(pBitVector + 1); 1452 pBitVector->pChunks = (void*)NV_ALIGN_UP((NvUPtr)pBitVector->pChunks, 1453 (NvUPtr)PORT_MEM_BITVECTOR_CHUNK_SIZE); 1454 1455 pLastChunkInBlock = (void*)NV_ALIGN_DOWN((NvUPtr)pPreallocatedBlock + 1456 blockSizeBytes - 1457 PORT_MEM_BITVECTOR_CHUNK_SIZE, 1458 (NvUPtr)PORT_MEM_BITVECTOR_CHUNK_SIZE); 1459 if (pLastChunkInBlock < pBitVector->pChunks) 1460 { 1461 pBitVector->numChunks = 0; 1462 } 1463 else 1464 { 1465 pBitVector->numChunks = (NvU32)(pLastChunkInBlock - pBitVector->pChunks) + 1; 1466 } 1467 bitVectorSize = (NvU32)((NvU8*)pBitVector->pChunks - (NvU8*)pBitVector->bits); 1468 1469 while (bitVectorSize*8 < pBitVector->numChunks*2) 1470 { 1471 // If too many chunks to track in current bit vector, increase bitvector by one chunk 1472 pBitVector->pChunks++; 1473 pBitVector->numChunks--; 1474 bitVectorSize = (NvU32)((NvU8*)pBitVector->pChunks - (NvU8*)pBitVector->bits); 1475 } 1476 portMemSet(pBitVector->bits, 0, bitVectorSize); 1477 1478 PORT_MEM_PRINT_INFO("Acquired preallocated block allocator %p (%"NvUPtr_fmtu" bytes) ", pAllocator, blockSizeBytes); 1479 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM)); 1480 return pAllocator; 1481 } 1482 1483 static void * 1484 _portMemAllocatorAllocExistingWrapper 1485 ( 1486 PORT_MEM_ALLOCATOR *pAlloc, 1487 NvLength length 1488 ) 1489 { 1490 NvU32 chunksNeeded = (NvU32)NV_DIV_AND_CEIL(length, PORT_MEM_BITVECTOR_CHUNK_SIZE); 1491 void *pMem = NULL; 1492 NvU32 chunksFound = 0; 1493 NvU32 i; 1494 PORT_MEM_BITVECTOR *pBitVector = (PORT_MEM_BITVECTOR*)(pAlloc->pImpl); 1495 PORT_SPINLOCK *pSpinlock = (PORT_SPINLOCK*)(pBitVector->pSpinlock); 1496 1497 if (chunksNeeded > pBitVector->numChunks) 1498 { 1499 return NULL; 1500 } 1501 if (pSpinlock != NULL) 1502 { 1503 portSyncSpinlockAcquire(pSpinlock); 1504 } 1505 for (i = 0; i < pBitVector->numChunks; i++) 1506 { 1507 NvBool bWholeWordSet; 1508 bWholeWordSet = pBitVector->bits[i/32] == ~0U; 1509 if (bWholeWordSet || (_isBitSet(pBitVector->bits, i))) 1510 { 1511 // Chunk not available as whole. 1512 chunksFound = 0; 1513 // Skip fully set words 1514 if (bWholeWordSet) 1515 { 1516 i += 31; 1517 } 1518 if (chunksNeeded > (pBitVector->numChunks - i - (bWholeWordSet ? 1 : 0))) 1519 { 1520 break; 1521 } 1522 } 1523 else 1524 { 1525 chunksFound++; 1526 if (chunksFound == chunksNeeded) 1527 { 1528 NvU32 j; 1529 NvU32 firstAllocatedChunk = i - chunksFound + 1; 1530 1531 pMem = pBitVector->pChunks[firstAllocatedChunk]; 1532 // Mark all acquired chunks as occupied 1533 for (j = firstAllocatedChunk; j <= i; j++) 1534 { 1535 _setBit(pBitVector->bits, j); 1536 } 1537 // Mark last chunk of allocation 1538 _setBit(pBitVector->bits, pBitVector->numChunks + i); 1539 break; 1540 } 1541 } 1542 } 1543 if (pSpinlock != NULL) 1544 { 1545 portSyncSpinlockRelease(pSpinlock); 1546 } 1547 return pMem; 1548 } 1549 1550 static void 1551 _portMemAllocatorFreeExistingWrapper 1552 ( 1553 PORT_MEM_ALLOCATOR *pAlloc, 1554 void *pMem 1555 ) 1556 { 1557 PORT_MEM_BITVECTOR_CHUNK *pChunk = (PORT_MEM_BITVECTOR_CHUNK *)pMem; 1558 NvU32 i; 1559 PORT_MEM_BITVECTOR *pBitVector = (PORT_MEM_BITVECTOR*)(pAlloc->pImpl); 1560 PORT_SPINLOCK *pSpinlock = (PORT_SPINLOCK*)(pBitVector->pSpinlock); 1561 1562 if (((NvUPtr)pMem < (NvUPtr)pBitVector->pChunks) || 1563 ((NvUPtr)pMem > (NvUPtr)(pBitVector->pChunks + pBitVector->numChunks))) 1564 { 1565 // pMem not inside this allocator. 1566 PORT_BREAKPOINT_CHECKED(); 1567 return; 1568 } 1569 1570 if (pSpinlock != NULL) 1571 { 1572 portSyncSpinlockAcquire(pSpinlock); 1573 } 1574 for (i = (NvU32)(pChunk - pBitVector->pChunks); i < pBitVector->numChunks; i++) 1575 { 1576 // Mark chunk as free 1577 _clearBit(pBitVector->bits, i); 1578 if (_isBitSet(pBitVector->bits, pBitVector->numChunks + i)) 1579 { 1580 // Clear last-allocation-bit and bail 1581 _clearBit(pBitVector->bits, pBitVector->numChunks + i); 1582 break; 1583 } 1584 } 1585 if (pSpinlock != NULL) 1586 { 1587 portSyncSpinlockRelease(pSpinlock); 1588 } 1589 } 1590