1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2014-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /**
25 * @file
26 * @brief NvPort MEMORY module extension to track memory allocations
27 *
28 * This file is internal to NvPort MEMORY module.
29 * @cond NVPORT_INTERNAL
30 */
31
32 #ifndef _NVPORT_MEMORY_INTERNAL_H_
33 #define _NVPORT_MEMORY_INTERNAL_H_
34
35 #define portMemExTrackingGetActiveStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER
36 #define portMemExTrackingGetTotalStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER
37 #define portMemExTrackingGetPeakStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER
38 #define portMemExTrackingGetNext_SUPPORTED \
39 (PORT_MEM_TRACK_USE_FENCEPOSTS & PORT_MEM_TRACK_USE_ALLOCLIST)
40 #define portMemExTrackingGetHeapSize_SUPPORTED (NVOS_IS_LIBOS)
41 //
42 // portMemExTrackingGetAllocUsableSize provides higher-fidelity tracking at the
43 // expense of higher runtime overhead, due to additional function calls in the
44 // portMemAlloc path. It is only enabled by default in debug ("checked") builds for
45 // this reason.
46 //
47 #define portMemExTrackingGetAllocUsableSize_SUPPORTED (PORT_IS_CHECKED_BUILD && NVOS_IS_LIBOS)
48
49 #define portMemExValidate_SUPPORTED 0
50 #define portMemExValidateAllocations_SUPPORTED 0
51 #define portMemExFreeAll_SUPPORTED 0
52
53 /** @brief Untracked paged memory allocation, platform specific */
54 void *_portMemAllocPagedUntracked(NvLength lengthBytes);
55 /** @brief Untracked nonpaged memory allocation, platform specific */
56 void *_portMemAllocNonPagedUntracked(NvLength lengthBytes);
57 /** @brief Untracked memory free, platform specific */
58 void _portMemFreeUntracked(void *pMemory);
59 /** @brief Wrapper around pAlloc->_portAlloc() that tracks the allocation */
60 void *_portMemAllocatorAlloc(PORT_MEM_ALLOCATOR *pAlloc, NvLength length);
61 /** @brief Wrapper around pAlloc->_portFree() that tracks the allocation */
62 void _portMemAllocatorFree(PORT_MEM_ALLOCATOR *pAlloc, void *pMem);
63
64 #if PORT_MEM_TRACK_USE_LIMIT
65 /** @brief Initialize per VF tracking limit **/
66 void portMemInitializeAllocatorTrackingLimit(NvU32 pid, NvLength limit, NvBool bLimitEnabled);
67 #endif
68
69 typedef struct PORT_MEM_COUNTER
70 {
71 volatile NvU32 activeAllocs;
72 volatile NvU32 totalAllocs;
73 volatile NvU32 peakAllocs;
74 volatile NvLength activeSize;
75 volatile NvLength totalSize;
76 volatile NvLength peakSize;
77 } PORT_MEM_COUNTER;
78
79 typedef struct PORT_MEM_FENCE_HEAD
80 {
81 PORT_MEM_ALLOCATOR *pAllocator;
82 NvU32 magic;
83 } PORT_MEM_FENCE_HEAD;
84
85 typedef struct PORT_MEM_FENCE_TAIL
86 {
87 NvU32 magic;
88 } PORT_MEM_FENCE_TAIL;
89
90 typedef struct PORT_MEM_LIST
91 {
92 struct PORT_MEM_LIST *pPrev;
93 struct PORT_MEM_LIST *pNext;
94 } PORT_MEM_LIST;
95
96 #if PORT_MEM_TRACK_USE_CALLERINFO
97
98 #if PORT_MEM_TRACK_USE_CALLERINFO_IP
99
100 typedef NvUPtr PORT_MEM_CALLERINFO;
101 #define PORT_MEM_CALLERINFO_MAKE portUtilGetIPAddress()
102
103 #else // PORT_MEM_TRACK_USE_CALLERINFO_IP
104
105 typedef struct PORT_MEM_CALLERINFO
106 {
107 const char *file;
108 const char *func;
109 NvU32 line;
110 } PORT_MEM_CALLERINFO;
111
112 /** @note Needed since not all compilers support automatic struct creation */
113 static NV_INLINE PORT_MEM_CALLERINFO
_portMemCallerInfoMake(const char * file,const char * func,NvU32 line)114 _portMemCallerInfoMake
115 (
116 const char *file,
117 const char *func,
118 NvU32 line
119 )
120 {
121 PORT_MEM_CALLERINFO callerInfo;
122 callerInfo.file = file;
123 callerInfo.func = func;
124 callerInfo.line = line;
125 return callerInfo;
126 }
127
128 #define PORT_MEM_CALLERINFO_MAKE \
129 _portMemCallerInfoMake(__FILE__, __FUNCTION__, __LINE__)
130 #endif // PORT_MEM_TRACK_USE_CALLERINFO_IP
131
132 void *portMemAllocPaged_CallerInfo(NvLength, PORT_MEM_CALLERINFO);
133 void *portMemAllocNonPaged_CallerInfo(NvLength, PORT_MEM_CALLERINFO);
134 PORT_MEM_ALLOCATOR *portMemAllocatorCreatePaged_CallerInfo(PORT_MEM_CALLERINFO);
135 PORT_MEM_ALLOCATOR *portMemAllocatorCreateNonPaged_CallerInfo(PORT_MEM_CALLERINFO);
136 void portMemInitializeAllocatorTracking_CallerInfo(PORT_MEM_ALLOCATOR *, PORT_MEM_ALLOCATOR_TRACKING *, PORT_MEM_CALLERINFO);
137 void *_portMemAllocatorAlloc_CallerInfo(PORT_MEM_ALLOCATOR*, NvLength, PORT_MEM_CALLERINFO);
138 PORT_MEM_ALLOCATOR *portMemAllocatorCreateOnExistingBlock_CallerInfo(void *, NvLength, PORT_MEM_CALLERINFO);
139 #if portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED
140 PORT_MEM_ALLOCATOR *portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo(void *, NvLength, void *, PORT_MEM_CALLERINFO);
141 #endif //portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED
142 #undef PORT_ALLOC
143 #define PORT_ALLOC(pAlloc, length) \
144 _portMemAllocatorAlloc_CallerInfo(pAlloc, length, PORT_MEM_CALLERINFO_MAKE)
145
146 #define portMemAllocPaged(size) \
147 portMemAllocPaged_CallerInfo((size), PORT_MEM_CALLERINFO_MAKE)
148 #define portMemAllocNonPaged(size) \
149 portMemAllocNonPaged_CallerInfo((size), PORT_MEM_CALLERINFO_MAKE)
150 #define portMemAllocatorCreatePaged() \
151 portMemAllocatorCreatePaged_CallerInfo(PORT_MEM_CALLERINFO_MAKE)
152 #define portMemAllocatorCreateNonPaged() \
153 portMemAllocatorCreateNonPaged_CallerInfo(PORT_MEM_CALLERINFO_MAKE)
154
155 #define portMemInitializeAllocatorTracking(pAlloc, pTrack) \
156 portMemInitializeAllocatorTracking_CallerInfo(pAlloc, pTrack, PORT_MEM_CALLERINFO_MAKE)
157
158 #define portMemAllocatorCreateOnExistingBlock(pMem, size) \
159 portMemAllocatorCreateOnExistingBlock_CallerInfo(pMem, size, PORT_MEM_CALLERINFO_MAKE)
160 #if portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED
161 #define portMemExAllocatorCreateLockedOnExistingBlock(pMem, size, pLock) \
162 portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo(pMem, size, pLock, \
163 PORT_MEM_CALLERINFO_MAKE)
164 #endif //portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED
165 #else
166 #define PORT_MEM_CALLERINFO_MAKE
167 #endif // CALLERINFO
168
169
170 #if PORT_MEM_TRACK_USE_FENCEPOSTS || PORT_MEM_TRACK_USE_ALLOCLIST || PORT_MEM_TRACK_USE_CALLERINFO || PORT_MEM_TRACK_USE_LIMIT
171
172 //
173 // The blockSize of the allocation can be tracked in 1 of 2 places,
174 // depending on build configuration. To reduce per-alloc memory waste in
175 // allocating multiple fields for the same data, the following is the
176 // order in which they are preferred at compile time:
177 //
178 // 1. portMemExTrackingGetAllocUsableSize - the underlying malloc
179 // implementation has a function to get the allocation size from allocated
180 // pointer. No memory will be allocated in the PORT_MEM_HEADER to track the
181 // block size.
182 // 2. PORT_MEM_HEADER::blockSize - used when fenceposts or per-GFID limit
183 // tracking is enabled and portMemExTrackingGetAllocUsableSize is not
184 // available.
185 //
186 #define PORT_MEM_HEADER_HAS_BLOCK_SIZE \
187 !PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetAllocUsableSize) && \
188 (PORT_MEM_TRACK_USE_FENCEPOSTS || PORT_MEM_TRACK_USE_LIMIT)
189
190 typedef struct PORT_MEM_HEADER
191 {
192 #if PORT_MEM_HEADER_HAS_BLOCK_SIZE
193 NvLength blockSize;
194 #endif
195 #if PORT_MEM_TRACK_USE_CALLERINFO
196 PORT_MEM_CALLERINFO callerInfo;
197 #endif
198 #if PORT_MEM_TRACK_USE_ALLOCLIST
199 PORT_MEM_LIST list;
200 #endif
201 #if PORT_MEM_TRACK_USE_FENCEPOSTS
202 PORT_MEM_FENCE_HEAD fence;
203 #endif
204 #if PORT_MEM_TRACK_USE_LIMIT
205 NV_DECLARE_ALIGNED(NvU32 pid, 8);
206 #endif
207 } PORT_MEM_HEADER;
208
209 typedef struct PORT_MEM_FOOTER
210 {
211 #if PORT_MEM_TRACK_USE_FENCEPOSTS
212 PORT_MEM_FENCE_TAIL fence;
213 #endif
214 } PORT_MEM_FOOTER;
215
216 #define PORT_MEM_ADD_HEADER_PTR(p) ((PORT_MEM_HEADER*)p + 1)
217 #define PORT_MEM_SUB_HEADER_PTR(p) ((PORT_MEM_HEADER*)p - 1)
218 #define PORT_MEM_STAGING_SIZE (sizeof(PORT_MEM_HEADER)+sizeof(PORT_MEM_FOOTER))
219 #else
220 #define PORT_MEM_ADD_HEADER_PTR(p) p
221 #define PORT_MEM_SUB_HEADER_PTR(p) p
222 #define PORT_MEM_STAGING_SIZE 0
223 #define PORT_MEM_HEADER_HAS_BLOCK_SIZE 0
224 #endif
225
226 #define PORT_MEM_TRACK_ALLOC_SIZE \
227 PORT_MEM_TRACK_USE_COUNTER && \
228 (PORT_IS_FUNC_SUPPORTED(portMemExTrackingGetAllocUsableSize) || \
229 PORT_MEM_HEADER_HAS_BLOCK_SIZE)
230
231 struct PORT_MEM_ALLOCATOR_TRACKING
232 {
233 PORT_MEM_ALLOCATOR *pAllocator;
234 struct PORT_MEM_ALLOCATOR_TRACKING *pPrev;
235 struct PORT_MEM_ALLOCATOR_TRACKING *pNext;
236
237 #if PORT_MEM_TRACK_USE_COUNTER
238 PORT_MEM_COUNTER counter;
239 #endif
240 #if PORT_MEM_TRACK_USE_ALLOCLIST
241 PORT_MEM_LIST *pFirstAlloc;
242 void *listLock;
243 #endif
244 #if PORT_MEM_TRACK_USE_CALLERINFO
245 PORT_MEM_CALLERINFO callerInfo;
246 #endif
247 };
248
249 /// @brief Actual size of an allocator structure, including internals
250 #define PORT_MEM_ALLOCATOR_SIZE \
251 (sizeof(PORT_MEM_ALLOCATOR) + sizeof(PORT_MEM_ALLOCATOR_TRACKING))
252
253 #if defined(BIT)
254 #define NVIDIA_UNDEF_LEGACY_BIT_MACROS
255 #endif
256 #include "nvmisc.h"
257
258 //
259 // Internal bitvector structures for allocators over existing blocks
260 //
261 #define PORT_MEM_BITVECTOR_CHUNK_SIZE 16U
262 typedef NvU8 PORT_MEM_BITVECTOR_CHUNK[PORT_MEM_BITVECTOR_CHUNK_SIZE];
263 typedef struct
264 {
265 //
266 // Points to a PORT_SPINLOCK that make memory thread safe.
267 // If this is not thread safe variant, then it is NULL.
268 //
269 void *pSpinlock;
270 // Points to after the bitvector, aligned to first chunk.
271 PORT_MEM_BITVECTOR_CHUNK *pChunks;
272 NvU32 numChunks;
273 //
274 // What follows are two bitvectors one next to another:
275 // - The first represents availability of chunks: 0=free, 1=allocated
276 // - The second represents allocation sizes: 1=last chunk of an allocation
277 // So the total size of this array is 2*numChunks bits
278 // The second vector continues immediately after the first, no alignment
279 //
280 // Example: numChunks = 8, 2 allocations of 3 chunks each:
281 // bits == |11111100| <- 2*3 chunks allocated, 2 free
282 // |00100100| <- Chunks 2 and 5 are last in allocation
283 //
284 NvU32 bits[NV_ANYSIZE_ARRAY];
285 } PORT_MEM_BITVECTOR;
286
287 /// @note the following can be used as arguments for static array size, so
288 /// they must be fully known at compile time - macros, not inline functions
289
290 /// @brief Total number of chunks in a preallocated block of given size
291 #define PORT_MEM_PREALLOCATED_BLOCK_NUM_CHUNKS(size) \
292 NV_DIV_AND_CEIL(size, PORT_MEM_BITVECTOR_CHUNK_SIZE)
293
294 /// @brief Minimal nonaligned bookkeeping size required for a preallocated block
295 #define PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_NONALIGNED_EXTRA_SIZE \
296 sizeof(PORT_MEM_ALLOCATOR) + sizeof(PORT_MEM_BITVECTOR)
297
298 /// @brief Minimal bookkeeping size required for a preallocated block
299 #define PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE \
300 NV_ALIGN_UP(PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_NONALIGNED_EXTRA_SIZE, \
301 PORT_MEM_BITVECTOR_CHUNK_SIZE)
302
303 /// @brief Number of chunks that can be tracked in the minimal bookkeeping size
304 #define PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS \
305 (( \
306 PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE - \
307 sizeof(PORT_MEM_ALLOCATOR) - \
308 NV_OFFSETOF(PORT_MEM_BITVECTOR, bits) \
309 )*4U)
310
311 // Although we can never execute the underflow branch, the compiler will complain
312 // if any constant expression results in underflow, even in dead code.
313 // Note: Skipping (parens) around a and b on purpose here.
314 #define _PORT_CEIL_NO_UNDERFLOW(a, b) (NV_DIV_AND_CEIL(b + a, b) - 1)
315
316 /// @brief Required additional size for a given number of chunks
317 #define PORT_MEM_PREALLOCATED_BLOCK_SIZE_FOR_NONGRATIS_CHUNKS(num_chunks) \
318 ((num_chunks > PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS) \
319 ? _PORT_CEIL_NO_UNDERFLOW(num_chunks - PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS, \
320 4*PORT_MEM_BITVECTOR_CHUNK_SIZE) \
321 * PORT_MEM_BITVECTOR_CHUNK_SIZE \
322 : 0)
323
324 /// @brief Total required bookkeeping size for a block of given useful size
325 #define PORT_MEM_PREALLOCATED_BLOCK_EXTRA_SIZE(size) \
326 PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE + \
327 PORT_MEM_PREALLOCATED_BLOCK_SIZE_FOR_NONGRATIS_CHUNKS( \
328 PORT_MEM_PREALLOCATED_BLOCK_NUM_CHUNKS(size))
329
330 /**
331 * Macros for defining memory allocation wrappers.
332 *
333 * The function / file / line reference is not useful when portMemAlloc
334 * is called from a generic memory allocator function, such as the memCreate
335 * function in resman.
336 *
337 * These macros can be used to push the function /file / line reference up one
338 * level when defining a memory allocator function. In other words, log who
339 * calls memCreate instead of logging memCreate.
340 *
341 * These macros are also used throughout memory-tracking.c
342 */
343 #if PORT_MEM_TRACK_USE_CALLERINFO
344
345 #define PORT_MEM_CALLERINFO_PARAM _portMemCallerInfo
346 #define PORT_MEM_CALLERINFO_TYPE_PARAM \
347 PORT_MEM_CALLERINFO PORT_MEM_CALLERINFO_PARAM
348 #define PORT_MEM_CALLERINFO_COMMA_PARAM ,PORT_MEM_CALLERINFO_PARAM
349 #define PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM ,PORT_MEM_CALLERINFO_TYPE_PARAM
350 #define PORT_MEM_CALLINFO_FUNC(f) f##_CallerInfo
351
352 #else // PORT_MEM_TRACK_USE_CALLERINFO
353
354 #define PORT_MEM_CALLERINFO_PARAM
355 #define PORT_MEM_CALLERINFO_TYPE_PARAM void
356 #define PORT_MEM_CALLERINFO_COMMA_PARAM
357 #define PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM
358 #define PORT_MEM_CALLINFO_FUNC(f) f
359
360 #endif // PORT_MEM_TRACK_USE_CALLERINFO
361
362 #endif // _NVPORT_MEMORY_INTERNAL_H_
363 /// @endcond
364