1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2015-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /**
25 * @file
26 * @brief MEMORY module tracking functions implementation
27 *
28 */
29
30 #include "nvport/nvport.h"
31 #include <nvport/safe.h>
32
33 #if !PORT_IS_MODULE_SUPPORTED(debug)
34 #error "DEBUG module must be present for memory tracking"
35 #endif
36
37 #if PORT_MEM_TRACK_USE_LIMIT
38 #include "os/os.h"
39 #define PORT_MEM_LIMIT_MAX_PIDS 32
40 #endif
41
42 #if NVOS_IS_LIBOS
43 #define PORT_MEM_THREAD_SAFE_ALLOCATIONS 0
44 #else
45 #define PORT_MEM_THREAD_SAFE_ALLOCATIONS 1
46 #endif
47
48 #if PORT_MEM_THREAD_SAFE_ALLOCATIONS && !PORT_IS_MODULE_SUPPORTED(atomic)
49 #error "ATOMIC module must be present for memory tracking"
50 #endif
51
52 #if PORT_MEM_THREAD_SAFE_ALLOCATIONS
53 #define PORT_MEM_ATOMIC_ADD_SIZE portAtomicAddSize
54 #define PORT_MEM_ATOMIC_SUB_SIZE portAtomicSubSize
55 #define PORT_MEM_ATOMIC_DEC_U32 portAtomicDecrementU32
56 #define PORT_MEM_ATOMIC_INC_U32 portAtomicIncrementU32
57 #define PORT_MEM_ATOMIC_SET_U32 portAtomicSetU32
58 #define PORT_MEM_ATOMIC_CAS_SIZE portAtomicCompareAndSwapSize
59 #define PORT_MEM_ATOMIC_CAS_U32 portAtomicCompareAndSwapU32
60 #else
61 //
62 // We can just stub out the atomic operations for non-atomic ones and not waste
63 // cycles on synchronization
64 //
65 #define PORT_MEM_ATOMIC_ADD_SIZE(pVal, val) (*((volatile NvSPtr *)pVal) += val)
66 #define PORT_MEM_ATOMIC_SUB_SIZE(pVal, val) (*((volatile NvSPtr *)pVal) -= val)
67 #define PORT_MEM_ATOMIC_DEC_U32(pVal) (--(*((volatile NvU32 *)pVal)))
68 #define PORT_MEM_ATOMIC_INC_U32(pVal) (++(*((volatile NvU32 *)pVal)))
69 #define PORT_MEM_ATOMIC_SET_U32(pVal, val) (*((volatile NvU32 *)pVal) = val)
70 #define PORT_MEM_ATOMIC_CAS_SIZE(pVal, newVal, oldVal) \
71 ((*pVal == oldVal) ? ((*((volatile NvSPtr *)pVal) = newVal), NV_TRUE) : NV_FALSE)
72 #define PORT_MEM_ATOMIC_CAS_U32(pVal, newVal, oldVal) \
73 ((*pVal == oldVal) ? ((*((volatile NvU32 *)pVal) = newVal), NV_TRUE) : NV_FALSE)
74 #endif // !PORT_MEM_THREAD_SAFE_ALLOCATIONS
75
76 struct PORT_MEM_ALLOCATOR_IMPL
77 {
78 PORT_MEM_ALLOCATOR_TRACKING tracking;
79 };
80
81 //
82 // Debug print macros
83 //
84 #if PORT_MEM_TRACK_PRINT_LEVEL == PORT_MEM_TRACK_PRINT_LEVEL_SILENT
85 #define PORT_MEM_PRINT_ERROR(...)
86 #define PORT_MEM_PRINT_INFO(...)
87 #elif PORT_MEM_TRACK_PRINT_LEVEL == PORT_MEM_TRACK_PRINT_LEVEL_BASIC
88 #define PORT_MEM_PRINT_ERROR(...) portDbgPrintf(__VA_ARGS__)
89 #define PORT_MEM_PRINT_INFO(...)
90 #else
91 #define PORT_MEM_PRINT_ERROR(...) portDbgPrintf(__VA_ARGS__)
92 #define PORT_MEM_PRINT_INFO(...) portDbgPrintf(__VA_ARGS__)
93 #endif
94
95 // Simple implementation of a spinlock that is going to be used where sync module is not included.
96 #if !PORT_IS_MODULE_SUPPORTED(sync)
97
98 #if NVCPU_IS_RISCV64
99 #error "Sync module should be enabled for RISC-V builds"
100 #endif
101
102 typedef volatile NvU32 PORT_SPINLOCK;
103 static NvLength portSyncSpinlockSize = sizeof(PORT_SPINLOCK);
portSyncSpinlockInitialize(PORT_SPINLOCK * pSpinlock)104 static NV_STATUS portSyncSpinlockInitialize(PORT_SPINLOCK *pSpinlock)
105 {
106 *pSpinlock = 0;
107 return NV_OK;
108 }
portSyncSpinlockAcquire(PORT_SPINLOCK * pSpinlock)109 static void portSyncSpinlockAcquire(PORT_SPINLOCK *pSpinlock)
110 {
111 while (!PORT_MEM_ATOMIC_CAS_U32(pSpinlock, 1, 0));
112 }
portSyncSpinlockRelease(PORT_SPINLOCK * pSpinlock)113 static void portSyncSpinlockRelease(PORT_SPINLOCK *pSpinlock)
114 {
115 PORT_MEM_ATOMIC_SET_U32(pSpinlock, 0);
116 }
portSyncSpinlockDestroy(PORT_SPINLOCK * pSpinlock)117 static void portSyncSpinlockDestroy(PORT_SPINLOCK *pSpinlock)
118 {
119 PORT_UNREFERENCED_VARIABLE(pSpinlock);
120 }
121 #endif
122
123 #define PORT_MEM_LOCK_INIT(lock) \
124 do { \
125 lock = _portMemAllocNonPagedUntracked(portSyncSpinlockSize); \
126 portSyncSpinlockInitialize(lock); \
127 } while (0)
128 #define PORT_MEM_LOCK_DESTROY(lock) \
129 do { \
130 portSyncSpinlockDestroy(lock); \
131 _portMemFreeUntracked(lock); \
132 } while(0)
133 #define PORT_MEM_LOCK_ACQUIRE(lock) portSyncSpinlockAcquire(lock)
134 #define PORT_MEM_LOCK_RELEASE(lock) portSyncSpinlockRelease(lock)
135
136
137 //
138 // List link operation that operates on structures that have pNext and pPrev
139 // fields. Assumes the root always exists.
140 //
141 #define PORT_LOCKED_LIST_LINK(pRoot, pNode, lock) \
142 do { \
143 PORT_MEM_LOCK_ACQUIRE(lock); \
144 (pNode)->pNext = (pRoot); \
145 (pNode)->pPrev = (pRoot)->pPrev; \
146 (pRoot)->pPrev = (pNode); \
147 (pNode)->pPrev->pNext = (pNode); \
148 PORT_MEM_LOCK_RELEASE(lock); \
149 } while(0)
150
151 #define PORT_LOCKED_LIST_UNLINK(pRoot, pNode, lock) \
152 do { \
153 PORT_MEM_LOCK_ACQUIRE(lock); \
154 (pNode)->pNext->pPrev = (pNode)->pPrev; \
155 (pNode)->pPrev->pNext = (pNode)->pNext; \
156 PORT_MEM_LOCK_RELEASE(lock); \
157 } while (0)
158
159 //
160 // All memory tracking globals are contained in this structure
161 //
162 static struct PORT_MEM_GLOBALS
163 {
164 PORT_MEM_ALLOCATOR_TRACKING mainTracking;
165 void *trackingLock;
166 struct
167 {
168 PORT_MEM_ALLOCATOR paged;
169 PORT_MEM_ALLOCATOR nonPaged;
170 PORT_MEM_ALLOCATOR_IMPL pagedImpl;
171 PORT_MEM_ALLOCATOR_IMPL nonPagedImpl;
172 } alloc;
173 NvU32 initCount;
174 NvU32 totalAllocators;
175 #if PORT_MEM_TRACK_USE_LIMIT
176 NvBool bLimitEnabled;
177 NvLength limitPid[PORT_MEM_LIMIT_MAX_PIDS];
178 NvLength counterPid[PORT_MEM_LIMIT_MAX_PIDS];
179 #endif
180 } portMemGlobals;
181
182 //
183 // Memory counter implementation
184 //
185 #if PORT_MEM_TRACK_USE_COUNTER
186 #if PORT_MEM_TRACK_ALLOC_SIZE
187 static NV_INLINE NvLength
_portMemExTrackingGetAllocUsableSizeWrapper(void * pMem)188 _portMemExTrackingGetAllocUsableSizeWrapper
189 (
190 void *pMem
191 )
192 {
193 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetAllocUsableSize)
194 //
195 // blockSize in PORT_MEM_HEADER does not include the size of the header/
196 // footer, but the underlying malloc implementation doesn't know about
197 // those. Account for them here.
198 //
199 NvLength allocSize = portMemExTrackingGetAllocUsableSize(PORT_MEM_SUB_HEADER_PTR(pMem));
200 PORT_ASSERT_CHECKED(allocSize > PORT_MEM_STAGING_SIZE);
201 return allocSize - PORT_MEM_STAGING_SIZE;
202 #elif PORT_MEM_HEADER_HAS_BLOCK_SIZE
203 return PORT_MEM_SUB_HEADER_PTR(pMem)->blockSize;
204 #endif
205 }
206 static NV_INLINE void
_portMemExTrackingSetOrGetAllocUsableSize(void * pMem,NvLength * pSize)207 _portMemExTrackingSetOrGetAllocUsableSize
208 (
209 void *pMem,
210 NvLength *pSize
211 )
212 {
213 #if PORT_MEM_HEADER_HAS_BLOCK_SIZE
214 PORT_MEM_SUB_HEADER_PTR(pMem)->blockSize = *pSize;
215 #else
216 *pSize = _portMemExTrackingGetAllocUsableSizeWrapper(pMem);
217 #endif
218 }
219 #endif // PORT_MEM_TRACK_ALLOC_SIZE
220 static NV_INLINE void
_portMemCounterInit(PORT_MEM_COUNTER * pCounter)221 _portMemCounterInit
222 (
223 PORT_MEM_COUNTER *pCounter
224 )
225 {
226 portMemSet(pCounter, 0, sizeof(*pCounter));
227 }
228 static NV_INLINE void
_portMemCounterInc(PORT_MEM_COUNTER * pCounter,NvLength size)229 _portMemCounterInc
230 (
231 PORT_MEM_COUNTER *pCounter,
232 NvLength size
233 )
234 {
235 NvU32 activeAllocs;
236 NvLength activeSize = 0;
237
238 activeAllocs = PORT_MEM_ATOMIC_INC_U32(&pCounter->activeAllocs);
239 PORT_MEM_ATOMIC_INC_U32(&pCounter->totalAllocs);
240 #if PORT_MEM_TRACK_ALLOC_SIZE
241 //
242 // activeSize is only tracked on configurations where we can retrieve the
243 // allocation size from allocation metadata in _portMemCounterDec.
244 //
245 activeSize = PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->activeSize, size);
246 #endif
247
248 //
249 // Note: this can overflow on 32-bit platforms if we exceed 4GB cumulative
250 // allocations. It's not trivial to fix, since NvPort doesn't emulate 64-bit
251 // atomics on 32-bit platforms, so just assume this doesn't happen (or
252 // doesn't matter too much if it does, since it's only for reporting).
253 //
254 PORT_MEM_ATOMIC_ADD_SIZE(&pCounter->totalSize, size);
255
256 // Update the peak stats, if we're updating the peakSize
257 {
258 NvU32 peakAllocs;
259 NvLength peakSize = pCounter->peakSize;
260 while (activeSize > peakSize)
261 {
262 PORT_MEM_ATOMIC_CAS_SIZE(&pCounter->peakSize, activeSize, peakSize);
263 peakSize = pCounter->peakSize;
264 }
265
266 //
267 // Ensure peakAllocs stays (approximately) in sync with peakSize, rather
268 // than always taking the greatest peakAllocs, so that the peak stats
269 // report is consistent.
270 //
271 do
272 {
273 peakAllocs = pCounter->peakAllocs;
274
275 //
276 // Only attempt to update the peakAllocs if activeSize is still the
277 // peakSize.
278 //
279 if (activeSize != pCounter->peakSize)
280 break;
281 } while (!PORT_MEM_ATOMIC_CAS_U32(&pCounter->peakAllocs, activeAllocs, peakAllocs));
282 }
283 }
284 static NV_INLINE void
_portMemCounterDec(PORT_MEM_COUNTER * pCounter,NvLength size)285 _portMemCounterDec
286 (
287 PORT_MEM_COUNTER *pCounter,
288 NvLength size
289 )
290 {
291 PORT_MEM_ATOMIC_DEC_U32(&pCounter->activeAllocs);
292 #if PORT_MEM_TRACK_ALLOC_SIZE
293 PORT_MEM_ATOMIC_SUB_SIZE(&pCounter->activeSize, size);
294 #else
295 PORT_UNREFERENCED_VARIABLE(size);
296 #endif
297 }
298
299 #define PORT_MEM_COUNTER_INIT(pCounter) _portMemCounterInit(pCounter)
300 #define PORT_MEM_COUNTER_INC(pCounter, size) _portMemCounterInc(pCounter, size)
301 #define PORT_MEM_COUNTER_DEC(pCounter, size) _portMemCounterDec(pCounter, size)
302 #else
303 #define PORT_MEM_COUNTER_INIT(x)
304 #define PORT_MEM_COUNTER_INC(x, y) PORT_UNREFERENCED_VARIABLE(y)
305 #define PORT_MEM_COUNTER_DEC(x, y) PORT_UNREFERENCED_VARIABLE(y)
306 #endif // COUNTER
307
308
309 //
310 // Memory fenceposts implementation
311 //
312 #if PORT_MEM_TRACK_USE_FENCEPOSTS
313 #define PORT_MEM_FENCE_HEAD_MAGIC 0x68656164 // 'head'
314 #define PORT_MEM_FENCE_TAIL_MAGIC 0x7461696c // 'tail'
315
316 static NV_INLINE void
_portMemFenceInit(PORT_MEM_ALLOCATOR * pAlloc,void * pMem,NvLength size)317 _portMemFenceInit
318 (
319 PORT_MEM_ALLOCATOR *pAlloc,
320 void *pMem,
321 NvLength size
322 )
323 {
324 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
325 PORT_MEM_FOOTER *pTail = (PORT_MEM_FOOTER*)((NvU8*)pMem + size);
326
327 pHead->fence.pAllocator = pAlloc;
328 pHead->fence.magic = PORT_MEM_FENCE_HEAD_MAGIC;
329 pTail->fence.magic = PORT_MEM_FENCE_TAIL_MAGIC;
330 }
331
332 static NV_INLINE void
_portMemFenceCheck(PORT_MEM_ALLOCATOR * pAlloc,void * pMem,NvLength size)333 _portMemFenceCheck
334 (
335 PORT_MEM_ALLOCATOR *pAlloc,
336 void *pMem,
337 NvLength size
338 )
339 {
340 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
341 PORT_MEM_FOOTER *pTail = (PORT_MEM_FOOTER*)((NvU8*)pMem + size);
342
343 if (pHead->fence.magic != PORT_MEM_FENCE_HEAD_MAGIC ||
344 pTail->fence.magic != PORT_MEM_FENCE_TAIL_MAGIC)
345 {
346 PORT_MEM_PRINT_ERROR("Memory corruption detected on block %p\n", pMem);
347 PORT_ASSERT_CHECKED(pHead->fence.magic == PORT_MEM_FENCE_HEAD_MAGIC);
348 PORT_ASSERT_CHECKED(pTail->fence.magic == PORT_MEM_FENCE_TAIL_MAGIC);
349 }
350 if (pHead->fence.pAllocator != pAlloc)
351 {
352 PORT_MEM_PRINT_ERROR("Freeing block %p using a wrong allocator (%p instead of %p)\n",
353 pMem, pAlloc, pHead->fence.pAllocator);
354 PORT_ASSERT_CHECKED(pHead->fence.pAllocator == pAlloc);
355
356 }
357 }
358
359 #define PORT_MEM_FENCE_CHECK(pAlloc, pMem, size) _portMemFenceCheck(pAlloc, pMem, size)
360 #define PORT_MEM_FENCE_INIT(pAlloc, pMem, size) _portMemFenceInit(pAlloc, pMem, size)
361 #else
362 #define PORT_MEM_FENCE_INIT(x, y, z)
363 #define PORT_MEM_FENCE_CHECK(x, y, z)
364 #endif // FENCEPOSTS
365
366
367 //
368 // Memory allocation lists implementation
369 //
370 #if PORT_MEM_TRACK_USE_ALLOCLIST
371 static NV_INLINE void
_portMemListAdd(PORT_MEM_ALLOCATOR_TRACKING * pTracking,void * pMem)372 _portMemListAdd
373 (
374 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
375 void *pMem
376 )
377 {
378 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
379 PORT_MEM_LIST *pList = &pHead->list;
380 pList->pNext = pList;
381 pList->pPrev = pList;
382 if (!PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList, NULL))
383 {
384 PORT_LOCKED_LIST_LINK(pTracking->pFirstAlloc, pList, pTracking->listLock);
385 }
386 }
387 static NV_INLINE void
_portMemListRemove(PORT_MEM_ALLOCATOR_TRACKING * pTracking,void * pMem)388 _portMemListRemove
389 (
390 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
391 void *pMem
392 )
393 {
394 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
395 PORT_MEM_LIST *pList = &pHead->list;
396
397 if (!PORT_MEM_ATOMIC_CAS_SIZE(&pList->pNext, NULL, pList))
398 {
399 PORT_LOCKED_LIST_UNLINK(pTracking->pFirstAlloc, pList, pTracking->listLock);
400 }
401 PORT_MEM_ATOMIC_CAS_SIZE(&pTracking->pFirstAlloc, pList->pNext, pList);
402 }
403
404 static NV_INLINE PORT_MEM_HEADER *
_portMemListGetHeader(PORT_MEM_LIST * pList)405 _portMemListGetHeader
406 (
407 PORT_MEM_LIST *pList
408 )
409 {
410 return (PORT_MEM_HEADER*)((NvU8*)pList - (NvUPtr)(&((PORT_MEM_HEADER*)NULL)->list));
411 }
412 #define PORT_MEM_LIST_INIT(pTracking) \
413 do { \
414 (pTracking)->pFirstAlloc = NULL; \
415 PORT_MEM_LOCK_INIT((pTracking)->listLock); \
416 } while (0)
417 #define PORT_MEM_LIST_DESTROY(pTracking) PORT_MEM_LOCK_DESTROY((pTracking)->listLock)
418 #define PORT_MEM_LIST_ADD(pTracking, pMem) _portMemListAdd(pTracking, pMem)
419 #define PORT_MEM_LIST_REMOVE(Tracking, pMem) _portMemListRemove(pTracking, pMem)
420 #else
421 #define PORT_MEM_LIST_INIT(x)
422 #define PORT_MEM_LIST_DESTROY(x)
423 #define PORT_MEM_LIST_ADD(x, y)
424 #define PORT_MEM_LIST_REMOVE(x, y)
425 #endif // ALLOCLIST
426
427
428
429 //
430 // Memory allocation-caller info implementation
431 //
432 #if PORT_MEM_TRACK_USE_CALLERINFO
433
434 static NV_INLINE void
_portMemCallerInfoInitMem(void * pMem,PORT_MEM_CALLERINFO callerInfo)435 _portMemCallerInfoInitMem
436 (
437 void *pMem,
438 PORT_MEM_CALLERINFO callerInfo
439 )
440 {
441 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
442 portMemCopy(&pHead->callerInfo, sizeof(callerInfo),
443 &callerInfo, sizeof(callerInfo));
444 }
445 static NV_INLINE void
_portMemCallerInfoInitTracking(PORT_MEM_ALLOCATOR_TRACKING * pTracking,PORT_MEM_CALLERINFO callerInfo)446 _portMemCallerInfoInitTracking
447 (
448 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
449 PORT_MEM_CALLERINFO callerInfo
450 )
451 {
452 portMemCopy(&pTracking->callerInfo, sizeof(callerInfo),
453 &callerInfo, sizeof(callerInfo));
454 }
455
456 #define PORT_MEM_CALLERINFO_INIT_TRACKING(pTracking) \
457 _portMemCallerInfoInitTracking(pTracking, PORT_MEM_CALLERINFO_PARAM)
458 #define PORT_MEM_CALLERINFO_INIT_MEM(pMem) \
459 _portMemCallerInfoInitMem(pMem, PORT_MEM_CALLERINFO_PARAM)
460
461 #if PORT_MEM_TRACK_USE_CALLERINFO_IP
462 #if NVOS_IS_LIBOS
463 //
464 // Libos has custom %a format specifier that decodes an instruction pointer into
465 // a function / file / line reference when the binary output is decoded.
466 //
467 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ %a\n", x
468 #else
469 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ 0x%016x\n", x
470 #endif // NVOS_IS_LIBOS
471 #else
472 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "@ %s:%u (%s)\n", x.file, x.line, x.func
473 #endif // PORT_MEM_TRACK_USE_CALLERINFO_IP
474
475 #else // PORT_MEM_TRACK_USE_CALLERINFO
476 #define PORT_MEM_CALLERINFO_INIT_TRACKING(x)
477 #define PORT_MEM_CALLERINFO_INIT_MEM(x)
478 #define PORT_MEM_CALLERINFO_PRINT_ARGS(x) "\n"
479 #endif // PORT_MEM_TRACK_USE_CALLERINFO
480
481
482 #if PORT_MEM_TRACK_USE_LOGGING
483 #include "nvlog/nvlog.h"
484 /** @brief Single log entry. Uses 64bit values even on 32bit systems. */
485 typedef struct PORT_MEM_LOG_ENTRY
486 {
487 NvP64 address;
488 NvP64 allocator;
489 NvLength size; // if size is 0, it is a free() call, not alloc()
490 } PORT_MEM_LOG_ENTRY;
491
492 #define PORT_MEM_TRACK_LOG_TAG 0x70726d74
493 #define PORT_MEM_LOG_ENTRIES 4096
494
495 static void
_portMemLogInit(void)496 _portMemLogInit(void)
497 {
498 NVLOG_BUFFER_HANDLE hBuffer;
499 nvlogAllocBuffer(PORT_MEM_LOG_ENTRIES * sizeof(PORT_MEM_LOG_ENTRY),
500 DRF_DEF(LOG, _BUFFER_FLAGS, _FORMAT, _MEMTRACK),
501 PORT_MEM_TRACK_LOG_TAG, &hBuffer);
502 }
503
504 static void
_portMemLogDestroy(void)505 _portMemLogDestroy(void)
506 {
507 NVLOG_BUFFER_HANDLE hBuffer;
508 nvlogGetBufferHandleFromTag(PORT_MEM_TRACK_LOG_TAG, &hBuffer);
509 nvlogDeallocBuffer(hBuffer);
510 }
511
512 static void
_portMemLogAdd(PORT_MEM_ALLOCATOR * pAllocator,void * pMem,NvLength lengthBytes)513 _portMemLogAdd
514 (
515 PORT_MEM_ALLOCATOR *pAllocator,
516 void *pMem,
517 NvLength lengthBytes
518 )
519 {
520 NVLOG_BUFFER_HANDLE hBuffer;
521 PORT_MEM_LOG_ENTRY entry = {0};
522 entry.address = NV_PTR_TO_NvP64(pMem);
523 entry.address = NV_PTR_TO_NvP64(pAllocator);
524 entry.size = lengthBytes;
525 nvlogGetBufferHandleFromTag(PORT_MEM_TRACK_LOG_TAG, &hBuffer);
526 nvlogWriteToBuffer(hBuffer, &entry, sizeof(entry));
527 }
528
529 #define PORT_MEM_LOG_INIT() _portMemLogInit()
530 #define PORT_MEM_LOG_DESTROY() _portMemLogDestroy()
531 #define PORT_MEM_LOG_ALLOC(pAlloc, pMem, size) \
532 _portMemLogAdd(pAlloc, pMem, size)
533 #define PORT_MEM_LOG_FREE(pAlloc, pMem) \
534 _portMemLogAdd(pAlloc, pMem, 0)
535 #else
536 #define PORT_MEM_LOG_INIT()
537 #define PORT_MEM_LOG_DESTROY()
538 #define PORT_MEM_LOG_ALLOC(x, y, z)
539 #define PORT_MEM_LOG_FREE(x, y)
540 #endif // LOGGING
541
542
543 ////////////////////////////////////////////////////////////////////////////////
544 //
545 // Main memory tracking implementation
546 //
547 ////////////////////////////////////////////////////////////////////////////////
548
549 //
550 // All static functions declarations. Definitions at the end of file.
551 //
552 static void *_portMemAllocatorAllocPagedWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length);
553 static void *_portMemAllocatorAllocNonPagedWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length);
554 static void _portMemAllocatorFreeWrapper(PORT_MEM_ALLOCATOR *pAlloc, void *pMem);
555 static void _portMemAllocatorReleaseWrapper(PORT_MEM_ALLOCATOR *pAlloc);
556
557 static PORT_MEM_ALLOCATOR *_portMemAllocatorCreateOnExistingBlock(void *pAlloc, NvLength blockSizeBytes, void *pSpinlock PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM);
558 static void *_portMemAllocatorAllocExistingWrapper(PORT_MEM_ALLOCATOR *pAlloc, NvLength length);
559 static void _portMemAllocatorFreeExistingWrapper(PORT_MEM_ALLOCATOR *pAlloc, void *pMem);
560
561 static void _portMemTrackingRelease(PORT_MEM_ALLOCATOR_TRACKING *pTracking, NvBool bReportLeaks);
562 static void _portMemTrackAlloc(PORT_MEM_ALLOCATOR_TRACKING *pTracking, void *pMem, NvLength size, NvU32 pid PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM);
563 static void _portMemTrackFree(PORT_MEM_ALLOCATOR_TRACKING *pTracking, void *pMem);
564
565
566
567 #if PORT_MEM_TRACK_USE_CALLERINFO
568 #undef portMemAllocPaged
569 #undef portMemAllocNonPaged
570 #undef portMemAllocatorCreatePaged
571 #undef portMemAllocatorCreateNonPaged
572 #undef portMemInitializeAllocatorTracking
573 #undef _portMemAllocatorAlloc
574 #undef portMemAllocatorCreateOnExistingBlock
575 #undef portMemExAllocatorCreateLockedOnExistingBlock
576 // These functions have different names if CallerInfo is enabled.
577 #define portMemAllocPaged portMemAllocPaged_CallerInfo
578 #define portMemAllocNonPaged portMemAllocNonPaged_CallerInfo
579 #define portMemAllocatorCreatePaged portMemAllocatorCreatePaged_CallerInfo
580 #define portMemAllocatorCreateNonPaged portMemAllocatorCreateNonPaged_CallerInfo
581 #define portMemInitializeAllocatorTracking portMemInitializeAllocatorTracking_CallerInfo
582 #define _portMemAllocatorAlloc _portMemAllocatorAlloc_CallerInfo
583 #define portMemAllocatorCreateOnExistingBlock portMemAllocatorCreateOnExistingBlock_CallerInfo
584 #define portMemExAllocatorCreateLockedOnExistingBlock portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo
585 #endif
586
587 //
588 // Per-process heap limiting implementation
589 //
590 #if PORT_MEM_TRACK_USE_LIMIT
591 static NV_INLINE void
_portMemLimitInc(NvU32 pid,void * pMem,NvLength size)592 _portMemLimitInc(NvU32 pid, void *pMem, NvLength size)
593 {
594 PORT_MEM_HEADER *pMemHeader = PORT_MEM_SUB_HEADER_PTR(pMem);
595 pMemHeader->pid = pid;
596 if (portMemGlobals.bLimitEnabled)
597 {
598 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS))
599 {
600 NvU32 pidIdx = pid - 1;
601 PORT_MEM_ATOMIC_ADD_SIZE(&portMemGlobals.counterPid[pidIdx], size);
602 }
603 }
604 }
605
606 static NV_INLINE void
_portMemLimitDec(void * pMem,NvLength size)607 _portMemLimitDec(void *pMem, NvLength size)
608 {
609 if (portMemGlobals.bLimitEnabled)
610 {
611 PORT_MEM_HEADER *pMemHeader = PORT_MEM_SUB_HEADER_PTR(pMem);
612 NvU32 pid = pMemHeader->pid;
613
614 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS))
615 {
616 NvU32 pidIdx = pid - 1;
617 if (portMemGlobals.counterPid[pidIdx] < size)
618 {
619 PORT_MEM_PRINT_ERROR("memory free error: counter underflow\n");
620 PORT_BREAKPOINT_CHECKED();
621 }
622 else
623 {
624 PORT_MEM_ATOMIC_SUB_SIZE(&portMemGlobals.counterPid[pidIdx], size);
625 }
626 }
627 }
628 }
629
630 static NV_INLINE NvBool
_portMemLimitExceeded(NvU32 pid,NvLength size)631 _portMemLimitExceeded(NvU32 pid, NvLength size)
632 {
633 NvBool bExceeded = NV_FALSE;
634
635 if (portMemGlobals.bLimitEnabled)
636 {
637 if ((pid > 0) && (pid <= PORT_MEM_LIMIT_MAX_PIDS))
638 {
639 NvU32 pidIdx = pid - 1;
640 if ((size + portMemGlobals.counterPid[pidIdx]) > portMemGlobals.limitPid[pidIdx])
641 {
642 PORT_MEM_PRINT_ERROR("memory allocation denied; PID %d exceeded per-process heap limit of %"NvUPtr_fmtu"\n",
643 pid, portMemGlobals.limitPid[pidIdx]);
644 bExceeded = NV_TRUE;
645 }
646 }
647 }
648 return bExceeded;
649 }
650
651 #define PORT_MEM_LIMIT_INC(pid, pMem, size) _portMemLimitInc(pid, pMem, size)
652 #define PORT_MEM_LIMIT_DEC(pMem, size) _portMemLimitDec(pMem, size)
653 #define PORT_MEM_LIMIT_EXCEEDED(pid, size) _portMemLimitExceeded(pid, size)
654 #else
655 #define PORT_MEM_LIMIT_INC(pid, pMem, size) \
656 do { \
657 PORT_UNREFERENCED_VARIABLE(pid); \
658 PORT_UNREFERENCED_VARIABLE(pMem); \
659 PORT_UNREFERENCED_VARIABLE(size); \
660 } while (0)
661 #define PORT_MEM_LIMIT_DEC(pMem, size) \
662 do { \
663 PORT_UNREFERENCED_VARIABLE(pMem); \
664 PORT_UNREFERENCED_VARIABLE(size); \
665 } while (0)
666 #define PORT_MEM_LIMIT_EXCEEDED(pid, size) (NV_FALSE)
667 #endif // PORT_MEM_TRACK_USE_LIMIT
668
669 static NV_INLINE PORT_MEM_ALLOCATOR_TRACKING *
_portMemGetTracking(const PORT_MEM_ALLOCATOR * pAlloc)670 _portMemGetTracking
671 (
672 const PORT_MEM_ALLOCATOR *pAlloc
673 )
674 {
675 if (pAlloc == NULL)
676 return &portMemGlobals.mainTracking;
677 else
678 return pAlloc->pTracking;
679 }
680
681
682 void
portMemInitialize(void)683 portMemInitialize(void)
684 {
685 #if PORT_MEM_TRACK_USE_CALLERINFO
686 PORT_MEM_CALLERINFO_TYPE_PARAM = PORT_MEM_CALLERINFO_MAKE;
687 #endif
688 if (PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.initCount) != 1)
689 return;
690
691 portMemGlobals.mainTracking.pAllocator = NULL;
692 portMemGlobals.mainTracking.pNext = &portMemGlobals.mainTracking;
693 portMemGlobals.mainTracking.pPrev = &portMemGlobals.mainTracking;
694 PORT_MEM_COUNTER_INIT(&portMemGlobals.mainTracking.counter);
695 PORT_MEM_LIST_INIT(&portMemGlobals.mainTracking);
696 PORT_MEM_LOCK_INIT(portMemGlobals.trackingLock);
697
698 #if PORT_MEM_TRACK_USE_LIMIT
699 // Initialize process heap limit to max int (i.e. no limit)
700 portMemGlobals.bLimitEnabled = NV_FALSE;
701 portMemSet(&portMemGlobals.limitPid, NV_U8_MAX, sizeof(portMemGlobals.limitPid));
702 portMemSet(&portMemGlobals.counterPid, 0, sizeof(portMemGlobals.counterPid));
703 #endif
704
705 portMemGlobals.alloc.paged._portAlloc = _portMemAllocatorAllocPagedWrapper;
706 portMemGlobals.alloc.nonPaged._portAlloc = _portMemAllocatorAllocNonPagedWrapper;
707 portMemGlobals.alloc.paged._portFree = _portMemAllocatorFreeWrapper;
708 portMemGlobals.alloc.nonPaged._portFree = _portMemAllocatorFreeWrapper;
709 portMemGlobals.alloc.paged._portRelease = NULL;
710 portMemGlobals.alloc.nonPaged._portRelease = NULL;
711
712 if (PORT_MEM_TRACK_USE_FENCEPOSTS)
713 {
714 //
715 // Distinct paged and non-paged allocators require PORT_MEM_TRACK_USE_FENCEPOSTS
716 // so that the correct allocator can be looked up from the fenceposts in the
717 // portMemFree path.
718 //
719 portMemGlobals.alloc.paged.pImpl = &portMemGlobals.alloc.pagedImpl;
720 portMemGlobals.alloc.nonPaged.pImpl = &portMemGlobals.alloc.nonPagedImpl;
721
722 portMemInitializeAllocatorTracking(&portMemGlobals.alloc.paged,
723 &portMemGlobals.alloc.paged.pImpl->tracking
724 PORT_MEM_CALLERINFO_COMMA_PARAM);
725 portMemInitializeAllocatorTracking(&portMemGlobals.alloc.nonPaged,
726 &portMemGlobals.alloc.nonPaged.pImpl->tracking
727 PORT_MEM_CALLERINFO_COMMA_PARAM);
728 }
729 else
730 {
731 // Use the same impl for both paged and nonpaged.
732 portMemGlobals.alloc.paged.pImpl = &portMemGlobals.alloc.pagedImpl;
733 portMemGlobals.alloc.nonPaged.pImpl = &portMemGlobals.alloc.pagedImpl;
734 portMemInitializeAllocatorTracking(NULL,
735 &portMemGlobals.alloc.pagedImpl.tracking
736 PORT_MEM_CALLERINFO_COMMA_PARAM);
737 portMemGlobals.alloc.paged.pTracking = &portMemGlobals.alloc.pagedImpl.tracking;
738 portMemGlobals.alloc.nonPaged.pTracking = &portMemGlobals.alloc.pagedImpl.tracking;
739 }
740 PORT_MEM_LOG_INIT();
741 }
742 void
portMemShutdown(NvBool bForceSilent)743 portMemShutdown(NvBool bForceSilent)
744 {
745 PORT_UNREFERENCED_VARIABLE(bForceSilent);
746 if (PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.initCount) != 0)
747 return;
748
749 #if (PORT_MEM_TRACK_PRINT_LEVEL > PORT_MEM_TRACK_PRINT_LEVEL_SILENT)
750 if (!bForceSilent)
751 {
752 portMemPrintAllTrackingInfo();
753 }
754 #endif
755 PORT_MEM_LOG_DESTROY();
756
757 if (PORT_MEM_TRACK_USE_FENCEPOSTS)
758 {
759 _portMemTrackingRelease(&portMemGlobals.alloc.nonPaged.pImpl->tracking, NV_FALSE);
760 _portMemTrackingRelease(&portMemGlobals.alloc.paged.pImpl->tracking, NV_FALSE);
761 }
762 else
763 {
764 _portMemTrackingRelease(&portMemGlobals.alloc.pagedImpl.tracking, NV_FALSE);
765 }
766
767 PORT_MEM_LOCK_DESTROY(portMemGlobals.trackingLock);
768 PORT_MEM_LIST_DESTROY(&portMemGlobals.mainTracking);
769 portMemSet(&portMemGlobals, 0, sizeof(portMemGlobals));
770 }
771
772 void *
portMemAllocPaged(NvLength length PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)773 portMemAllocPaged
774 (
775 NvLength length
776 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
777 )
778 {
779 #if defined(__COVERITY__)
780 return __coverity_alloc__(length);
781 #endif
782 PORT_MEM_ALLOCATOR *pAlloc = portMemAllocatorGetGlobalPaged();
783 return _portMemAllocatorAlloc(pAlloc, length PORT_MEM_CALLERINFO_COMMA_PARAM);
784 }
785
786 void *
portMemAllocNonPaged(NvLength length PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)787 portMemAllocNonPaged
788 (
789 NvLength length
790 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
791 )
792 {
793 #if defined(__COVERITY__)
794 return __coverity_alloc__(length);
795 #endif
796 PORT_MEM_ALLOCATOR *pAlloc = portMemAllocatorGetGlobalNonPaged();
797 return _portMemAllocatorAlloc(pAlloc, length PORT_MEM_CALLERINFO_COMMA_PARAM);
798 }
799
800 void
portMemFree(void * pMem)801 portMemFree
802 (
803 void *pMem
804 )
805 {
806 if (pMem != NULL)
807 {
808 #if PORT_MEM_TRACK_USE_FENCEPOSTS
809 PORT_MEM_HEADER *pHead = (PORT_MEM_HEADER*)pMem - 1;
810 PORT_FREE(pHead->fence.pAllocator, pMem);
811 #else
812 // Paged/nonpaged are logged together if we don't have fenceposts.
813 PORT_FREE(portMemAllocatorGetGlobalPaged(), pMem);
814 #endif
815 }
816
817 #if defined(__COVERITY__)
818 __coverity_free__(pMem);
819 #endif
820 }
821
822 PORT_MEM_ALLOCATOR *
portMemAllocatorCreatePaged(PORT_MEM_CALLERINFO_TYPE_PARAM)823 portMemAllocatorCreatePaged(PORT_MEM_CALLERINFO_TYPE_PARAM)
824 {
825 PORT_MEM_ALLOCATOR *pAllocator;
826
827 pAllocator = portMemAllocPaged(PORT_MEM_ALLOCATOR_SIZE
828 PORT_MEM_CALLERINFO_COMMA_PARAM);
829 if (pAllocator == NULL)
830 return NULL;
831
832 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1);
833 pAllocator->_portAlloc = _portMemAllocatorAllocPagedWrapper;
834 pAllocator->_portFree = _portMemAllocatorFreeWrapper;
835 pAllocator->_portRelease = _portMemAllocatorReleaseWrapper;
836 portMemInitializeAllocatorTracking(pAllocator, &pAllocator->pImpl->tracking
837 PORT_MEM_CALLERINFO_COMMA_PARAM);
838
839 PORT_MEM_PRINT_INFO("Acquired paged allocator %p ", pAllocator);
840 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM));
841
842 return pAllocator;
843 }
844
845 PORT_MEM_ALLOCATOR *
portMemAllocatorCreateNonPaged(PORT_MEM_CALLERINFO_TYPE_PARAM)846 portMemAllocatorCreateNonPaged(PORT_MEM_CALLERINFO_TYPE_PARAM)
847 {
848 PORT_MEM_ALLOCATOR *pAllocator;
849
850 pAllocator = portMemAllocNonPaged(PORT_MEM_ALLOCATOR_SIZE
851 PORT_MEM_CALLERINFO_COMMA_PARAM);
852 if (pAllocator == NULL)
853 return NULL;
854
855 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1);
856 pAllocator->_portAlloc = _portMemAllocatorAllocNonPagedWrapper;
857 pAllocator->_portFree = _portMemAllocatorFreeWrapper;
858 pAllocator->_portRelease = _portMemAllocatorReleaseWrapper;
859 portMemInitializeAllocatorTracking(pAllocator, &pAllocator->pImpl->tracking
860 PORT_MEM_CALLERINFO_COMMA_PARAM);
861
862 PORT_MEM_PRINT_INFO("Acquired nonpaged allocator %p ", pAllocator);
863 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM));
864 return pAllocator;
865 }
866
867
868 PORT_MEM_ALLOCATOR *
portMemAllocatorCreateOnExistingBlock(void * pPreallocatedBlock,NvLength blockSizeBytes PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)869 portMemAllocatorCreateOnExistingBlock
870 (
871 void *pPreallocatedBlock,
872 NvLength blockSizeBytes
873 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
874 )
875 {
876 return _portMemAllocatorCreateOnExistingBlock(pPreallocatedBlock, blockSizeBytes,
877 NULL PORT_MEM_CALLERINFO_COMMA_PARAM);
878 }
879
880 PORT_MEM_ALLOCATOR *
portMemExAllocatorCreateLockedOnExistingBlock(void * pPreallocatedBlock,NvLength blockSizeBytes,void * pSpinlock PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)881 portMemExAllocatorCreateLockedOnExistingBlock
882 (
883 void *pPreallocatedBlock,
884 NvLength blockSizeBytes,
885 void *pSpinlock
886 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
887 )
888 {
889 return _portMemAllocatorCreateOnExistingBlock(pPreallocatedBlock, blockSizeBytes,
890 pSpinlock PORT_MEM_CALLERINFO_COMMA_PARAM);
891 }
892
893 void
portMemAllocatorRelease(PORT_MEM_ALLOCATOR * pAllocator)894 portMemAllocatorRelease
895 (
896 PORT_MEM_ALLOCATOR *pAllocator
897 )
898 {
899 if (pAllocator == NULL)
900 {
901 PORT_BREAKPOINT_CHECKED();
902 return;
903 }
904 _portMemTrackingRelease(pAllocator->pTracking, NV_TRUE);
905 PORT_MEM_PRINT_INFO("Released allocator %p\n", pAllocator);
906
907 if (pAllocator->_portRelease != NULL)
908 pAllocator->_portRelease(pAllocator);
909 }
910
911
912 PORT_MEM_ALLOCATOR *
portMemAllocatorGetGlobalNonPaged(void)913 portMemAllocatorGetGlobalNonPaged(void)
914 {
915 return &portMemGlobals.alloc.nonPaged;
916 }
917 PORT_MEM_ALLOCATOR *
portMemAllocatorGetGlobalPaged(void)918 portMemAllocatorGetGlobalPaged(void)
919 {
920 return &portMemGlobals.alloc.paged;
921 }
922
923 void
portMemInitializeAllocatorTracking(PORT_MEM_ALLOCATOR * pAlloc,PORT_MEM_ALLOCATOR_TRACKING * pTracking PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)924 portMemInitializeAllocatorTracking
925 (
926 PORT_MEM_ALLOCATOR *pAlloc,
927 PORT_MEM_ALLOCATOR_TRACKING *pTracking
928 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
929 )
930 {
931 if (portMemGlobals.initCount == 0)
932 {
933 portMemSet(pTracking, 0, sizeof(*pTracking));
934 if (pAlloc != NULL)
935 pAlloc->pTracking = NULL;
936 return;
937 }
938
939 pTracking->pAllocator = pAlloc;
940 if (pAlloc != NULL)
941 pAlloc->pTracking = pTracking;
942 PORT_LOCKED_LIST_LINK(&portMemGlobals.mainTracking, pTracking, portMemGlobals.trackingLock);
943 PORT_MEM_COUNTER_INIT(&pTracking->counter);
944 PORT_MEM_LIST_INIT(pTracking);
945 PORT_MEM_CALLERINFO_INIT_TRACKING(pTracking);
946 PORT_MEM_ATOMIC_INC_U32(&portMemGlobals.totalAllocators);
947 }
948
949 #if PORT_MEM_TRACK_USE_LIMIT
950 void
portMemInitializeAllocatorTrackingLimit(NvU32 pid,NvLength limit,NvBool bLimitEnabled)951 portMemInitializeAllocatorTrackingLimit(NvU32 pid, NvLength limit, NvBool bLimitEnabled)
952 {
953 NvU32 pidIdx = pid - 1;
954 portMemGlobals.limitPid[pidIdx] = limit;
955 portMemGlobals.bLimitEnabled = bLimitEnabled;
956 }
957 #endif
958
959 void *
_portMemAllocatorAlloc(PORT_MEM_ALLOCATOR * pAlloc,NvLength length PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)960 _portMemAllocatorAlloc
961 (
962 PORT_MEM_ALLOCATOR *pAlloc,
963 NvLength length
964 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
965 )
966 {
967 NvU32 pid = 0;
968 void *pMem = NULL;
969 if (pAlloc == NULL)
970 {
971 PORT_BREAKPOINT_CHECKED();
972 return NULL;
973 }
974
975 #if PORT_MEM_TRACK_USE_LIMIT
976 if (portMemGlobals.bLimitEnabled)
977 {
978 if (osGetCurrentProcessGfid(&pid) != NV_OK)
979 {
980 PORT_BREAKPOINT_CHECKED();
981 return NULL;
982 }
983 }
984 #endif
985
986 // Check if per-process memory limit will be exhausted by this allocation
987 if (PORT_MEM_LIMIT_EXCEEDED(pid, length))
988 return NULL;
989
990 if (length > 0)
991 {
992 NvLength paddedLength;
993 // RISCV64 requires 64-bit alignment of structures, and length indicates the alignment of the footer
994 #if defined(__riscv)
995 if (PORT_MEM_STAGING_SIZE > 0 && (length & 7))
996 {
997 if (!portSafeAddLength(length & ~7, 8, &length))
998 {
999 return NULL;
1000 }
1001 }
1002 #endif
1003 if (!portSafeAddLength(length, PORT_MEM_STAGING_SIZE, &paddedLength))
1004 {
1005 return NULL;
1006 }
1007 pMem = pAlloc->_portAlloc(pAlloc, paddedLength);
1008 }
1009 if (pMem != NULL)
1010 {
1011 pMem = PORT_MEM_ADD_HEADER_PTR(pMem);
1012 _portMemTrackAlloc(_portMemGetTracking(pAlloc), pMem, length, pid
1013 PORT_MEM_CALLERINFO_COMMA_PARAM);
1014 }
1015 return pMem;
1016 }
1017 void
_portMemAllocatorFree(PORT_MEM_ALLOCATOR * pAlloc,void * pMem)1018 _portMemAllocatorFree
1019 (
1020 PORT_MEM_ALLOCATOR *pAlloc,
1021 void *pMem
1022 )
1023 {
1024 if (pAlloc == NULL)
1025 {
1026 PORT_BREAKPOINT_CHECKED();
1027 return;
1028 }
1029 if (pMem != NULL)
1030 {
1031 _portMemTrackFree(_portMemGetTracking(pAlloc), pMem);
1032 pMem = PORT_MEM_SUB_HEADER_PTR(pMem);
1033 pAlloc->_portFree(pAlloc, pMem);
1034 }
1035 }
1036
1037 void
portMemPrintTrackingInfo(const PORT_MEM_ALLOCATOR_TRACKING * pTracking)1038 portMemPrintTrackingInfo
1039 (
1040 const PORT_MEM_ALLOCATOR_TRACKING *pTracking
1041 )
1042 {
1043 if (pTracking == NULL)
1044 pTracking = &portMemGlobals.mainTracking;
1045
1046 if (pTracking == &portMemGlobals.mainTracking)
1047 portDbgPrintf("[NvPort] ======== Aggregate Memory Tracking ========\n");
1048 else if ((pTracking == portMemGlobals.alloc.nonPaged.pTracking) &&
1049 (pTracking == portMemGlobals.alloc.paged.pTracking))
1050 portDbgPrintf("[NvPort] ======== Global Allocator Tracking ========\n");
1051 else if (pTracking == portMemGlobals.alloc.nonPaged.pTracking)
1052 portDbgPrintf("[NvPort] ======== Global Non-Paged Memory Allocator Tracking ========\n");
1053 else if (pTracking == portMemGlobals.alloc.paged.pTracking)
1054 portDbgPrintf("[NvPort] ======== Global Paged Memory Allocator Tracking ========\n");
1055 else
1056 portDbgPrintf("[NvPort] ======== Memory Allocator %p Tracking ======== \n", pTracking->pAllocator);
1057
1058 if (pTracking->counter.activeAllocs != 0)
1059 portDbgPrintf(" !!! MEMORY LEAK DETECTED (%u blocks) !!!\n",
1060 pTracking->counter.activeAllocs);
1061
1062 #if PORT_MEM_TRACK_USE_CALLERINFO
1063 {
1064 portDbgPrintf(" Allocator acquired "
1065 PORT_MEM_CALLERINFO_PRINT_ARGS(pTracking->callerInfo));
1066 }
1067 #endif
1068
1069 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetHeapSize)
1070 //
1071 // Heap is shared across all allocators, so only print it with the
1072 // aggregate stats.
1073 //
1074 if (pTracking == _portMemGetTracking(NULL))
1075 portDbgPrintf(" HEAP: %"NvUPtr_fmtu" bytes\n", portMemExTrackingGetHeapSize());
1076 #endif
1077
1078 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetActiveStats)
1079 {
1080 PORT_MEM_TRACK_ALLOCATOR_STATS stats;
1081
1082 portMemSet(&stats, 0, sizeof(stats));
1083
1084 portMemExTrackingGetActiveStats(pTracking->pAllocator, &stats);
1085
1086 //
1087 // rmtest_gsp test script (dvs_gsp_sanity.sh) depends on this print, so do not change
1088 // format without updating script!
1089 //
1090 portDbgPrintf(" ACTIVE: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n",
1091 stats.numAllocations,
1092 stats.allocatedSize,
1093 stats.usefulSize,
1094 stats.metaSize);
1095 }
1096 #endif
1097
1098 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetTotalStats)
1099 {
1100 PORT_MEM_TRACK_ALLOCATOR_STATS stats;
1101
1102 portMemSet(&stats, 0, sizeof(stats));
1103
1104 portMemExTrackingGetTotalStats(pTracking->pAllocator, &stats);
1105 portDbgPrintf(" TOTAL: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n",
1106 stats.numAllocations,
1107 stats.allocatedSize,
1108 stats.usefulSize,
1109 stats.metaSize);
1110 }
1111 #endif
1112
1113 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetPeakStats)
1114 {
1115 PORT_MEM_TRACK_ALLOCATOR_STATS stats;
1116
1117 portMemSet(&stats, 0, sizeof(stats));
1118
1119 portMemExTrackingGetPeakStats(pTracking->pAllocator, &stats);
1120 portDbgPrintf(" PEAK: %u allocations, %"NvUPtr_fmtu" bytes allocated (%"NvUPtr_fmtu" useful, %"NvUPtr_fmtu" meta)\n",
1121 stats.numAllocations,
1122 stats.allocatedSize,
1123 stats.usefulSize,
1124 stats.metaSize);
1125 }
1126 #endif
1127
1128 #if PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetNext)
1129 {
1130 PORT_MEM_TRACK_ALLOC_INFO info;
1131 NvBool bPrinted = NV_FALSE;
1132 void *iterator = NULL;
1133
1134 do
1135 {
1136 if (portMemExTrackingGetNext(pTracking->pAllocator, &info, &iterator) != NV_OK)
1137 {
1138 portDbgPrintf(" (no active allocations)\n");
1139 break;
1140 }
1141 else if (!bPrinted)
1142 {
1143 portDbgPrintf(" Currently active allocations:\n");
1144 bPrinted = NV_TRUE;
1145 }
1146 portDbgPrintf(" - A:%p - 0x%p [%8"NvUPtr_fmtu" bytes] T=%llu ",
1147 info.pAllocator,
1148 info.pMemory,
1149 info.size,
1150 info.timestamp);
1151 portDbgPrintf(PORT_MEM_CALLERINFO_PRINT_ARGS(info.callerInfo));
1152 } while (iterator != NULL);
1153 }
1154 #endif
1155 }
1156
1157 void
portMemPrintAllTrackingInfo(void)1158 portMemPrintAllTrackingInfo(void)
1159 {
1160 const PORT_MEM_ALLOCATOR_TRACKING *pTracking = &portMemGlobals.mainTracking;
1161 PORT_MEM_LOCK_ACQUIRE(portMemGlobals.trackingLock);
1162 do
1163 {
1164 portMemPrintTrackingInfo(pTracking);
1165 } while ((pTracking = pTracking->pNext) != &portMemGlobals.mainTracking);
1166 PORT_MEM_LOCK_RELEASE(portMemGlobals.trackingLock);
1167 }
1168
1169 #if portMemExTrackingGetActiveStats_SUPPORTED
1170 NV_STATUS
portMemExTrackingGetActiveStats(const PORT_MEM_ALLOCATOR * pAllocator,PORT_MEM_TRACK_ALLOCATOR_STATS * pStats)1171 portMemExTrackingGetActiveStats
1172 (
1173 const PORT_MEM_ALLOCATOR *pAllocator,
1174 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats
1175 )
1176 {
1177 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator);
1178 if (pTracking == NULL)
1179 {
1180 return NV_ERR_OBJECT_NOT_FOUND;
1181 }
1182 pStats->numAllocations = pTracking->counter.activeAllocs;
1183 pStats->usefulSize = pTracking->counter.activeSize;
1184 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE;
1185 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize;
1186 return NV_OK;
1187 }
1188 #endif
1189
1190 #if portMemExTrackingGetTotalStats_SUPPORTED
1191 NV_STATUS
portMemExTrackingGetTotalStats(const PORT_MEM_ALLOCATOR * pAllocator,PORT_MEM_TRACK_ALLOCATOR_STATS * pStats)1192 portMemExTrackingGetTotalStats
1193 (
1194 const PORT_MEM_ALLOCATOR *pAllocator,
1195 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats
1196 )
1197 {
1198 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator);
1199 if (pTracking == NULL)
1200 {
1201 return NV_ERR_OBJECT_NOT_FOUND;
1202 }
1203 pStats->numAllocations = pTracking->counter.totalAllocs;
1204 pStats->usefulSize = pTracking->counter.totalSize;
1205 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE;
1206 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize;
1207 return NV_OK;
1208 }
1209 #endif
1210
1211 #if portMemExTrackingGetPeakStats_SUPPORTED
1212 NV_STATUS
portMemExTrackingGetPeakStats(const PORT_MEM_ALLOCATOR * pAllocator,PORT_MEM_TRACK_ALLOCATOR_STATS * pStats)1213 portMemExTrackingGetPeakStats
1214 (
1215 const PORT_MEM_ALLOCATOR *pAllocator,
1216 PORT_MEM_TRACK_ALLOCATOR_STATS *pStats
1217 )
1218 {
1219 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator);
1220 if (pTracking == NULL)
1221 {
1222 return NV_ERR_OBJECT_NOT_FOUND;
1223 }
1224 pStats->numAllocations = pTracking->counter.peakAllocs;
1225 pStats->usefulSize = pTracking->counter.peakSize;
1226 pStats->metaSize = pStats->numAllocations * PORT_MEM_STAGING_SIZE;
1227 pStats->allocatedSize = pStats->usefulSize + pStats->metaSize;
1228 return NV_OK;
1229 }
1230 #endif
1231
1232 #if portMemExTrackingGetNext_SUPPORTED
1233 NV_STATUS
portMemExTrackingGetNext(const PORT_MEM_ALLOCATOR * pAllocator,PORT_MEM_TRACK_ALLOC_INFO * pInfo,void ** pIterator)1234 portMemExTrackingGetNext
1235 (
1236 const PORT_MEM_ALLOCATOR *pAllocator,
1237 PORT_MEM_TRACK_ALLOC_INFO *pInfo,
1238 void **pIterator
1239 )
1240 {
1241 PORT_MEM_ALLOCATOR_TRACKING *pTracking = _portMemGetTracking(pAllocator);
1242 PORT_MEM_LIST *pList;
1243 PORT_MEM_HEADER *pHead;
1244
1245 if (pTracking == NULL)
1246 {
1247 return NV_ERR_OBJECT_NOT_FOUND;
1248 }
1249
1250 if (pTracking->pFirstAlloc == NULL)
1251 return NV_ERR_OBJECT_NOT_FOUND;
1252
1253 if (*pIterator == NULL)
1254 pList = pTracking->pFirstAlloc;
1255 else
1256 pList = (PORT_MEM_LIST*)(*pIterator);
1257
1258 pHead = _portMemListGetHeader(pList);
1259
1260 // Advance iterator
1261 if (pList->pNext == pTracking->pFirstAlloc)
1262 *pIterator = NULL;
1263 else
1264 *pIterator = pList->pNext;
1265
1266 // Populate pInfo
1267 pInfo->pMemory = pHead + 1;
1268 pInfo->size = _portMemExTrackingGetAllocUsableSizeWrapper(pInfo->pMemory);
1269 pInfo->pAllocator = pHead->fence.pAllocator;
1270 pInfo->timestamp = 0;
1271
1272 #if PORT_MEM_TRACK_USE_CALLERINFO
1273 pInfo->callerInfo = pHead->callerInfo;
1274 #endif
1275
1276 return NV_OK;
1277 }
1278 #endif
1279
1280 static void
_portMemTrackingRelease(PORT_MEM_ALLOCATOR_TRACKING * pTracking,NvBool bReportLeaks)1281 _portMemTrackingRelease
1282 (
1283 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
1284 NvBool bReportLeaks
1285 )
1286 {
1287 if (pTracking == NULL) return;
1288
1289 #if (PORT_MEM_TRACK_PRINT_LEVEL > PORT_MEM_TRACK_PRINT_LEVEL_SILENT)
1290 if (bReportLeaks && (pTracking->counter.activeAllocs != 0))
1291 portMemPrintTrackingInfo(pTracking);
1292 #else
1293 PORT_UNREFERENCED_VARIABLE(bReportLeaks);
1294 #endif
1295
1296 PORT_LOCKED_LIST_UNLINK(&portMemGlobals.mainTracking, pTracking, portMemGlobals.trackingLock);
1297 PORT_MEM_LIST_DESTROY(pTracking);
1298 PORT_MEM_ATOMIC_DEC_U32(&portMemGlobals.totalAllocators);
1299 }
1300
1301 static void
_portMemTrackAlloc(PORT_MEM_ALLOCATOR_TRACKING * pTracking,void * pMem,NvLength size,NvU32 pid PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)1302 _portMemTrackAlloc
1303 (
1304 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
1305 void *pMem,
1306 NvLength size,
1307 NvU32 pid
1308 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
1309 )
1310 {
1311 PORT_UNREFERENCED_VARIABLE(pMem);
1312 if (pTracking == NULL) return;
1313
1314 #if PORT_MEM_TRACK_ALLOC_SIZE
1315 //
1316 // Either set the block size in the header, or override it with the value
1317 // from the underlying allocator (which may be bigger than what was
1318 // requested). This keeps the counters consistent with the free path.
1319 //
1320 _portMemExTrackingSetOrGetAllocUsableSize(pMem, &size);
1321 #endif
1322
1323 PORT_MEM_PRINT_INFO("Allocated %"NvUPtr_fmtu" bytes at address %p", size, pMem);
1324 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM));
1325
1326 PORT_MEM_COUNTER_INC(&pTracking->counter, size);
1327 PORT_MEM_COUNTER_INC(&portMemGlobals.mainTracking.counter, size);
1328 PORT_MEM_LIMIT_INC(pid, pMem, size);
1329
1330 PORT_MEM_FENCE_INIT(pTracking->pAllocator, pMem, size);
1331 PORT_MEM_LIST_ADD(pTracking, pMem);
1332 PORT_MEM_CALLERINFO_INIT_MEM(pMem);
1333 PORT_MEM_LOG_ALLOC(pTracking->pAllocator, pMem, size);
1334 }
1335
1336 static void
_portMemTrackFree(PORT_MEM_ALLOCATOR_TRACKING * pTracking,void * pMem)1337 _portMemTrackFree
1338 (
1339 PORT_MEM_ALLOCATOR_TRACKING *pTracking,
1340 void *pMem
1341 )
1342 {
1343 NvLength size = 0;
1344
1345 if (pTracking == NULL) return;
1346
1347 #if PORT_MEM_TRACK_ALLOC_SIZE
1348 size = _portMemExTrackingGetAllocUsableSizeWrapper(pMem);
1349 PORT_MEM_PRINT_INFO("Freeing %"NvUPtr_fmtu"-byte block at address %p\n", size, pMem);
1350 #else
1351 PORT_MEM_PRINT_INFO("Freeing block at address %p\n", pMem);
1352 #endif
1353
1354 PORT_MEM_COUNTER_DEC(&pTracking->counter, size);
1355 PORT_MEM_COUNTER_DEC(&portMemGlobals.mainTracking.counter, size);
1356 PORT_MEM_LIMIT_DEC(pMem, size);
1357
1358 PORT_MEM_FENCE_CHECK(pTracking->pAllocator, pMem, size);
1359 PORT_MEM_LIST_REMOVE(pTracking, pMem);
1360 PORT_MEM_LOG_FREE(pTracking->pAllocator, pMem);
1361 }
1362
1363
1364 static void *
_portMemAllocatorAllocPagedWrapper(PORT_MEM_ALLOCATOR * pAlloc,NvLength length)1365 _portMemAllocatorAllocPagedWrapper
1366 (
1367 PORT_MEM_ALLOCATOR *pAlloc,
1368 NvLength length
1369 )
1370 {
1371 PORT_UNREFERENCED_VARIABLE(pAlloc);
1372 return _portMemAllocPagedUntracked(length);
1373 }
1374
1375 static void *
_portMemAllocatorAllocNonPagedWrapper(PORT_MEM_ALLOCATOR * pAlloc,NvLength length)1376 _portMemAllocatorAllocNonPagedWrapper
1377 (
1378 PORT_MEM_ALLOCATOR *pAlloc,
1379 NvLength length
1380 )
1381 {
1382 PORT_UNREFERENCED_VARIABLE(pAlloc);
1383 return _portMemAllocNonPagedUntracked(length);
1384 }
1385
1386 static void
_portMemAllocatorFreeWrapper(PORT_MEM_ALLOCATOR * pAlloc,void * pMem)1387 _portMemAllocatorFreeWrapper
1388 (
1389 PORT_MEM_ALLOCATOR *pAlloc,
1390 void *pMem
1391 )
1392 {
1393 PORT_UNREFERENCED_VARIABLE(pAlloc);
1394 _portMemFreeUntracked(pMem);
1395 }
1396
1397 static void
_portMemAllocatorReleaseWrapper(PORT_MEM_ALLOCATOR * pAllocator)1398 _portMemAllocatorReleaseWrapper
1399 (
1400 PORT_MEM_ALLOCATOR *pAllocator
1401 )
1402 {
1403 portMemFree(pAllocator);
1404 }
1405
1406 /// @todo Add these as intrinsics to UTIL module
_isBitSet(NvU32 * vect,NvU32 bit)1407 static NV_INLINE NvBool _isBitSet(NvU32 *vect, NvU32 bit)
1408 {
1409 return !!(vect[bit/32] & NVBIT32(bit%32));
1410 }
_setBit(NvU32 * vect,NvU32 bit)1411 static NV_INLINE void _setBit(NvU32 *vect, NvU32 bit)
1412 {
1413 vect[bit/32] |= NVBIT32(bit%32);
1414 }
_clearBit(NvU32 * vect,NvU32 bit)1415 static NV_INLINE void _clearBit(NvU32 *vect, NvU32 bit)
1416 {
1417 vect[bit/32] &= ~NVBIT32(bit%32);
1418 }
1419
1420 static PORT_MEM_ALLOCATOR *
_portMemAllocatorCreateOnExistingBlock(void * pPreallocatedBlock,NvLength blockSizeBytes,void * pSpinlock PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM)1421 _portMemAllocatorCreateOnExistingBlock
1422 (
1423 void *pPreallocatedBlock,
1424 NvLength blockSizeBytes,
1425 void *pSpinlock
1426 PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
1427 )
1428 {
1429 PORT_MEM_ALLOCATOR *pAllocator = (PORT_MEM_ALLOCATOR *)pPreallocatedBlock;
1430 PORT_MEM_BITVECTOR *pBitVector;
1431 PORT_MEM_BITVECTOR_CHUNK *pLastChunkInBlock;
1432 NvU32 bitVectorSize;
1433
1434 if ((pPreallocatedBlock == NULL) ||
1435 (blockSizeBytes < PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE) ||
1436 (blockSizeBytes > NV_S32_MAX))
1437 {
1438 return NULL;
1439 }
1440
1441 pAllocator->_portAlloc = _portMemAllocatorAllocExistingWrapper;
1442 pAllocator->_portFree = _portMemAllocatorFreeExistingWrapper;
1443 pAllocator->_portRelease = NULL;
1444 pAllocator->pTracking = NULL; // No tracking for this allocator
1445 pAllocator->pImpl = (PORT_MEM_ALLOCATOR_IMPL*)(pAllocator + 1);
1446
1447
1448 //
1449 // PORT_MEM_BITVECTOR (pAllocator->pImpl) and PORT_MEM_ALLOCATOR_TRACKING (pAllocator->pImpl->tracking)
1450 // are mutually exclusively used.
1451 // When pAllocator->pTracking = NULL the data in pAllocator->pImpl->tracking is not used and instead
1452 // pBitVector uses the same meory location.
1453 // When pAllocator->pImpl->tracking there is no usage of PORT_MEM_BITVECTOR
1454 //
1455 pBitVector = (PORT_MEM_BITVECTOR*)(pAllocator->pImpl);
1456 pBitVector->pSpinlock = pSpinlock;
1457
1458 // Calculate total number of chunks available
1459 pBitVector->pChunks = (PORT_MEM_BITVECTOR_CHUNK *)(pBitVector + 1);
1460 pBitVector->pChunks = (void*)NV_ALIGN_UP((NvUPtr)pBitVector->pChunks,
1461 (NvUPtr)PORT_MEM_BITVECTOR_CHUNK_SIZE);
1462
1463 pLastChunkInBlock = (void*)NV_ALIGN_DOWN((NvUPtr)pPreallocatedBlock +
1464 blockSizeBytes -
1465 PORT_MEM_BITVECTOR_CHUNK_SIZE,
1466 (NvUPtr)PORT_MEM_BITVECTOR_CHUNK_SIZE);
1467 if (pLastChunkInBlock < pBitVector->pChunks)
1468 {
1469 pBitVector->numChunks = 0;
1470 }
1471 else
1472 {
1473 pBitVector->numChunks = (NvU32)(pLastChunkInBlock - pBitVector->pChunks) + 1;
1474 }
1475 bitVectorSize = (NvU32)((NvU8*)pBitVector->pChunks - (NvU8*)pBitVector->bits);
1476
1477 while (bitVectorSize*8 < pBitVector->numChunks*2)
1478 {
1479 // If too many chunks to track in current bit vector, increase bitvector by one chunk
1480 pBitVector->pChunks++;
1481 pBitVector->numChunks--;
1482 bitVectorSize = (NvU32)((NvU8*)pBitVector->pChunks - (NvU8*)pBitVector->bits);
1483 }
1484 portMemSet(pBitVector->bits, 0, bitVectorSize);
1485
1486 PORT_MEM_PRINT_INFO("Acquired preallocated block allocator %p (%"NvUPtr_fmtu" bytes) ", pAllocator, blockSizeBytes);
1487 PORT_MEM_PRINT_INFO(PORT_MEM_CALLERINFO_PRINT_ARGS(PORT_MEM_CALLERINFO_PARAM));
1488 return pAllocator;
1489 }
1490
1491 static void *
_portMemAllocatorAllocExistingWrapper(PORT_MEM_ALLOCATOR * pAlloc,NvLength length)1492 _portMemAllocatorAllocExistingWrapper
1493 (
1494 PORT_MEM_ALLOCATOR *pAlloc,
1495 NvLength length
1496 )
1497 {
1498 NvU32 chunksNeeded = (NvU32)NV_DIV_AND_CEIL(length, PORT_MEM_BITVECTOR_CHUNK_SIZE);
1499 void *pMem = NULL;
1500 NvU32 chunksFound = 0;
1501 NvU32 i;
1502 PORT_MEM_BITVECTOR *pBitVector = (PORT_MEM_BITVECTOR*)(pAlloc->pImpl);
1503 PORT_SPINLOCK *pSpinlock = (PORT_SPINLOCK*)(pBitVector->pSpinlock);
1504
1505 if (chunksNeeded > pBitVector->numChunks)
1506 {
1507 return NULL;
1508 }
1509 if (pSpinlock != NULL)
1510 {
1511 portSyncSpinlockAcquire(pSpinlock);
1512 }
1513 for (i = 0; i < pBitVector->numChunks; i++)
1514 {
1515 NvBool bWholeWordSet;
1516 bWholeWordSet = pBitVector->bits[i/32] == ~0U;
1517 if (bWholeWordSet || (_isBitSet(pBitVector->bits, i)))
1518 {
1519 // Chunk not available as whole.
1520 chunksFound = 0;
1521 // Skip fully set words
1522 if (bWholeWordSet)
1523 {
1524 i += 31;
1525 }
1526 if (chunksNeeded > (pBitVector->numChunks - i - (bWholeWordSet ? 1 : 0)))
1527 {
1528 break;
1529 }
1530 }
1531 else
1532 {
1533 chunksFound++;
1534 if (chunksFound == chunksNeeded)
1535 {
1536 NvU32 j;
1537 NvU32 firstAllocatedChunk = i - chunksFound + 1;
1538
1539 pMem = pBitVector->pChunks[firstAllocatedChunk];
1540 // Mark all acquired chunks as occupied
1541 for (j = firstAllocatedChunk; j <= i; j++)
1542 {
1543 _setBit(pBitVector->bits, j);
1544 }
1545 // Mark last chunk of allocation
1546 _setBit(pBitVector->bits, pBitVector->numChunks + i);
1547 break;
1548 }
1549 }
1550 }
1551 if (pSpinlock != NULL)
1552 {
1553 portSyncSpinlockRelease(pSpinlock);
1554 }
1555 if (pMem == NULL)
1556 {
1557 PORT_MEM_PRINT_ERROR("Memory allocation failed.\n");
1558 }
1559 return pMem;
1560 }
1561
1562 static void
_portMemAllocatorFreeExistingWrapper(PORT_MEM_ALLOCATOR * pAlloc,void * pMem)1563 _portMemAllocatorFreeExistingWrapper
1564 (
1565 PORT_MEM_ALLOCATOR *pAlloc,
1566 void *pMem
1567 )
1568 {
1569 PORT_MEM_BITVECTOR_CHUNK *pChunk = (PORT_MEM_BITVECTOR_CHUNK *)pMem;
1570 NvU32 i;
1571 PORT_MEM_BITVECTOR *pBitVector = (PORT_MEM_BITVECTOR*)(pAlloc->pImpl);
1572 PORT_SPINLOCK *pSpinlock = (PORT_SPINLOCK*)(pBitVector->pSpinlock);
1573
1574 if (((NvUPtr)pMem < (NvUPtr)pBitVector->pChunks) ||
1575 ((NvUPtr)pMem > (NvUPtr)(pBitVector->pChunks + pBitVector->numChunks)))
1576 {
1577 // pMem not inside this allocator.
1578 PORT_BREAKPOINT_CHECKED();
1579 return;
1580 }
1581
1582 if (pSpinlock != NULL)
1583 {
1584 portSyncSpinlockAcquire(pSpinlock);
1585 }
1586 for (i = (NvU32)(pChunk - pBitVector->pChunks); i < pBitVector->numChunks; i++)
1587 {
1588 // Mark chunk as free
1589 _clearBit(pBitVector->bits, i);
1590 if (_isBitSet(pBitVector->bits, pBitVector->numChunks + i))
1591 {
1592 // Clear last-allocation-bit and bail
1593 _clearBit(pBitVector->bits, pBitVector->numChunks + i);
1594 break;
1595 }
1596 }
1597 if (pSpinlock != NULL)
1598 {
1599 portSyncSpinlockRelease(pSpinlock);
1600 }
1601 }
1602