1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2014-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /** 25 * @file 26 * @brief NvPort MEMORY module extension to track memory allocations 27 * 28 * This file is internal to NvPort MEMORY module. 29 * @cond NVPORT_INTERNAL 30 */ 31 32 #ifndef _NVPORT_MEMORY_INTERNAL_H_ 33 #define _NVPORT_MEMORY_INTERNAL_H_ 34 35 /** @brief Untracked paged memory allocation, platform specific */ 36 void *_portMemAllocPagedUntracked(NvLength lengthBytes); 37 /** @brief Untracked nonpaged memory allocation, platform specific */ 38 void *_portMemAllocNonPagedUntracked(NvLength lengthBytes); 39 /** @brief Untracked memory free, platform specific */ 40 void _portMemFreeUntracked(void *pMemory); 41 /** @brief Wrapper around pAlloc->_portAlloc() that tracks the allocation */ 42 void *_portMemAllocatorAlloc(PORT_MEM_ALLOCATOR *pAlloc, NvLength length); 43 /** @brief Wrapper around pAlloc->_portFree() that tracks the allocation */ 44 void _portMemAllocatorFree(PORT_MEM_ALLOCATOR *pAlloc, void *pMem); 45 46 #if PORT_MEM_TRACK_USE_LIMIT 47 /** @brief Initialize per VF tracking limit **/ 48 void portMemInitializeAllocatorTrackingLimit(NvU32 pid, NvU64 heapSize, NvBool bLimitEnabled); 49 #endif 50 51 typedef struct PORT_MEM_COUNTER 52 { 53 volatile NvU32 activeAllocs; 54 volatile NvU32 totalAllocs; 55 volatile NvU32 peakAllocs; 56 volatile NvLength activeSize; 57 volatile NvLength totalSize; 58 volatile NvLength peakSize; 59 } PORT_MEM_COUNTER; 60 61 typedef struct PORT_MEM_FENCE_HEAD 62 { 63 PORT_MEM_ALLOCATOR *pAllocator; 64 NvLength blockSize; 65 NvU32 magic; 66 } PORT_MEM_FENCE_HEAD; 67 68 typedef struct PORT_MEM_FENCE_TAIL 69 { 70 NvU32 magic; 71 } PORT_MEM_FENCE_TAIL; 72 73 typedef struct PORT_MEM_LIST 74 { 75 struct PORT_MEM_LIST *pPrev; 76 struct PORT_MEM_LIST *pNext; 77 } PORT_MEM_LIST; 78 79 #if PORT_MEM_TRACK_USE_CALLERINFO 80 81 #if PORT_MEM_TRACK_USE_CALLERINFO_IP 82 83 typedef NvU64 PORT_MEM_CALLERINFO; 84 #define PORT_MEM_CALLERINFO_MAKE ((NvU64)portUtilGetIPAddress()) 85 86 #else // PORT_MEM_TRACK_USE_CALLERINFO_IP 87 88 typedef struct PORT_MEM_CALLERINFO 89 { 90 const char *file; 91 const char *func; 92 NvU32 line; 93 } PORT_MEM_CALLERINFO; 94 95 /** @note Needed since not all compilers support automatic struct creation */ 96 static NV_INLINE PORT_MEM_CALLERINFO 97 _portMemCallerInfoMake 98 ( 99 const char *file, 100 const char *func, 101 NvU32 line 102 ) 103 { 104 PORT_MEM_CALLERINFO callerInfo; 105 callerInfo.file = file; 106 callerInfo.func = func; 107 callerInfo.line = line; 108 return callerInfo; 109 } 110 111 #define PORT_MEM_CALLERINFO_MAKE \ 112 _portMemCallerInfoMake(__FILE__, __FUNCTION__, __LINE__) 113 #endif // PORT_MEM_TRACK_USE_CALLERINFO_IP 114 115 void *portMemAllocPaged_CallerInfo(NvLength, PORT_MEM_CALLERINFO); 116 void *portMemAllocNonPaged_CallerInfo(NvLength, PORT_MEM_CALLERINFO); 117 PORT_MEM_ALLOCATOR *portMemAllocatorCreatePaged_CallerInfo(PORT_MEM_CALLERINFO); 118 PORT_MEM_ALLOCATOR *portMemAllocatorCreateNonPaged_CallerInfo(PORT_MEM_CALLERINFO); 119 void portMemInitializeAllocatorTracking_CallerInfo(PORT_MEM_ALLOCATOR *, PORT_MEM_ALLOCATOR_TRACKING *, PORT_MEM_CALLERINFO); 120 void *_portMemAllocatorAlloc_CallerInfo(PORT_MEM_ALLOCATOR*, NvLength, PORT_MEM_CALLERINFO); 121 PORT_MEM_ALLOCATOR *portMemAllocatorCreateOnExistingBlock_CallerInfo(void *, NvLength, PORT_MEM_CALLERINFO); 122 #if portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED 123 PORT_MEM_ALLOCATOR *portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo(void *, NvLength, void *, PORT_MEM_CALLERINFO); 124 #endif //portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED 125 #undef PORT_ALLOC 126 #define PORT_ALLOC(pAlloc, length) \ 127 _portMemAllocatorAlloc_CallerInfo(pAlloc, length, PORT_MEM_CALLERINFO_MAKE) 128 129 #define portMemAllocPaged(size) \ 130 portMemAllocPaged_CallerInfo((size), PORT_MEM_CALLERINFO_MAKE) 131 #define portMemAllocNonPaged(size) \ 132 portMemAllocNonPaged_CallerInfo((size), PORT_MEM_CALLERINFO_MAKE) 133 #define portMemAllocatorCreatePaged() \ 134 portMemAllocatorCreatePaged_CallerInfo(PORT_MEM_CALLERINFO_MAKE) 135 #define portMemAllocatorCreateNonPaged() \ 136 portMemAllocatorCreateNonPaged_CallerInfo(PORT_MEM_CALLERINFO_MAKE) 137 138 #define portMemInitializeAllocatorTracking(pAlloc, pTrack) \ 139 portMemInitializeAllocatorTracking_CallerInfo(pAlloc, pTrack, PORT_MEM_CALLERINFO_MAKE) 140 141 #define portMemAllocatorCreateOnExistingBlock(pMem, size) \ 142 portMemAllocatorCreateOnExistingBlock_CallerInfo(pMem, size, PORT_MEM_CALLERINFO_MAKE) 143 #if portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED 144 #define portMemExAllocatorCreateLockedOnExistingBlock(pMem, size, pLock) \ 145 portMemExAllocatorCreateLockedOnExistingBlock_CallerInfo(pMem, size, pLock,\ 146 PORT_MEM_CALLERINFO_MAKE) 147 #endif //portMemExAllocatorCreateLockedOnExistingBlock_SUPPORTED 148 #else 149 #define PORT_MEM_CALLERINFO_MAKE 150 #endif // CALLERINFO 151 152 153 #if PORT_MEM_TRACK_USE_FENCEPOSTS || PORT_MEM_TRACK_USE_ALLOCLIST || PORT_MEM_TRACK_USE_CALLERINFO || PORT_MEM_TRACK_USE_LIMIT 154 typedef struct PORT_MEM_HEADER 155 { 156 #if PORT_MEM_TRACK_USE_CALLERINFO 157 PORT_MEM_CALLERINFO callerInfo; 158 #endif 159 #if PORT_MEM_TRACK_USE_ALLOCLIST 160 PORT_MEM_LIST list; 161 #endif 162 #if PORT_MEM_TRACK_USE_FENCEPOSTS 163 PORT_MEM_FENCE_HEAD fence; 164 #endif 165 #if PORT_MEM_TRACK_USE_LIMIT 166 NvU64 blockSize; 167 NvU32 pid; 168 #endif 169 } PORT_MEM_HEADER; 170 171 typedef struct PORT_MEM_FOOTER 172 { 173 #if PORT_MEM_TRACK_USE_FENCEPOSTS 174 PORT_MEM_FENCE_TAIL fence; 175 #endif 176 } PORT_MEM_FOOTER; 177 178 #define PORT_MEM_ADD_HEADER_PTR(p) ((PORT_MEM_HEADER*)p + 1) 179 #define PORT_MEM_SUB_HEADER_PTR(p) ((PORT_MEM_HEADER*)p - 1) 180 #define PORT_MEM_STAGING_SIZE (sizeof(PORT_MEM_HEADER)+sizeof(PORT_MEM_FOOTER)) 181 182 #else 183 #define PORT_MEM_ADD_HEADER_PTR(p) p 184 #define PORT_MEM_SUB_HEADER_PTR(p) p 185 #define PORT_MEM_STAGING_SIZE 0 186 #endif 187 188 struct PORT_MEM_ALLOCATOR_TRACKING 189 { 190 PORT_MEM_ALLOCATOR *pAllocator; 191 struct PORT_MEM_ALLOCATOR_TRACKING *pPrev; 192 struct PORT_MEM_ALLOCATOR_TRACKING *pNext; 193 194 #if PORT_MEM_TRACK_USE_COUNTER 195 PORT_MEM_COUNTER counter; 196 #endif 197 #if PORT_MEM_TRACK_USE_ALLOCLIST 198 PORT_MEM_LIST *pFirstAlloc; 199 void *listLock; 200 #endif 201 #if PORT_MEM_TRACK_USE_CALLERINFO 202 PORT_MEM_CALLERINFO callerInfo; 203 #endif 204 }; 205 206 207 #define portMemExTrackingGetActiveStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER 208 #define portMemExTrackingGetTotalStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER 209 #define portMemExTrackingGetPeakStats_SUPPORTED PORT_MEM_TRACK_USE_COUNTER 210 #define portMemExTrackingGetNext_SUPPORTED \ 211 (PORT_MEM_TRACK_USE_FENCEPOSTS & PORT_MEM_TRACK_USE_ALLOCLIST) 212 213 #define portMemExValidate_SUPPORTED 0 214 #define portMemExValidateAllocations_SUPPORTED 0 215 #define portMemExFreeAll_SUPPORTED 0 216 217 /// @brief Actual size of an allocator structure, including internals 218 #define PORT_MEM_ALLOCATOR_SIZE \ 219 (sizeof(PORT_MEM_ALLOCATOR) + sizeof(PORT_MEM_ALLOCATOR_TRACKING)) 220 221 #if defined(BIT) 222 #define NVIDIA_UNDEF_LEGACY_BIT_MACROS 223 #endif 224 #include "nvmisc.h" 225 226 // 227 // Internal bitvector structures for allocators over existing blocks 228 // 229 #define PORT_MEM_BITVECTOR_CHUNK_SIZE 16U 230 typedef NvU8 PORT_MEM_BITVECTOR_CHUNK[PORT_MEM_BITVECTOR_CHUNK_SIZE]; 231 typedef struct 232 { 233 // 234 // Points to a PORT_SPINLOCK that make memory thread safe. 235 // If this is not thread safe variant, then it is NULL. 236 // 237 void *pSpinlock; 238 // Points to after the bitvector, aligned to first chunk. 239 PORT_MEM_BITVECTOR_CHUNK *pChunks; 240 NvU32 numChunks; 241 // 242 // What follows are two bitvectors one next to another: 243 // - The first represents availability of chunks: 0=free, 1=allocated 244 // - The second represents allocation sizes: 1=last chunk of an allocation 245 // So the total size of this array is 2*numChunks bits 246 // The second vector continues immediately after the first, no alignment 247 // 248 // Example: numChunks = 8, 2 allocations of 3 chunks each: 249 // bits == |11111100| <- 2*3 chunks allocated, 2 free 250 // |00100100| <- Chunks 2 and 5 are last in allocation 251 // 252 NvU32 bits[NV_ANYSIZE_ARRAY]; 253 } PORT_MEM_BITVECTOR; 254 255 /// @note the following can be used as arguments for static array size, so 256 /// they must be fully known at compile time - macros, not inline functions 257 258 /// @brief Total number of chunks in a preallocated block of given size 259 #define PORT_MEM_PREALLOCATED_BLOCK_NUM_CHUNKS(size) \ 260 NV_DIV_AND_CEIL(size, PORT_MEM_BITVECTOR_CHUNK_SIZE) 261 262 /// @brief Minimal nonaligned bookkeeping size required for a preallocated block 263 #define PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_NONALIGNED_EXTRA_SIZE \ 264 sizeof(PORT_MEM_ALLOCATOR) + sizeof(PORT_MEM_BITVECTOR) 265 266 /// @brief Minimal bookkeeping size required for a preallocated block 267 #define PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE \ 268 NV_ALIGN_UP(PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_NONALIGNED_EXTRA_SIZE, \ 269 PORT_MEM_BITVECTOR_CHUNK_SIZE) 270 271 /// @brief Number of chunks that can be tracked in the minimal bookkeeping size 272 #define PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS \ 273 (( \ 274 PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE - \ 275 sizeof(PORT_MEM_ALLOCATOR) - \ 276 NV_OFFSETOF(PORT_MEM_BITVECTOR, bits) \ 277 )*4U) 278 279 // Although we can never execute the underflow branch, the compiler will complain 280 // if any constant expression results in underflow, even in dead code. 281 // Note: Skipping (parens) around a and b on purpose here. 282 #define _PORT_CEIL_NO_UNDERFLOW(a, b) (NV_DIV_AND_CEIL(b + a, b) - 1) 283 284 /// @brief Required additional size for a given number of chunks 285 #define PORT_MEM_PREALLOCATED_BLOCK_SIZE_FOR_NONGRATIS_CHUNKS(num_chunks) \ 286 ((num_chunks > PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS) \ 287 ? _PORT_CEIL_NO_UNDERFLOW(num_chunks - PORT_MEM_PREALLOCATED_BLOCK_CHUNKS_GRATIS,\ 288 4*PORT_MEM_BITVECTOR_CHUNK_SIZE) \ 289 * PORT_MEM_BITVECTOR_CHUNK_SIZE \ 290 : 0) 291 292 /// @brief Total required bookkeeping size for a block of given useful size 293 #define PORT_MEM_PREALLOCATED_BLOCK_EXTRA_SIZE(size) \ 294 PORT_MEM_PREALLOCATED_BLOCK_MINIMAL_EXTRA_SIZE + \ 295 PORT_MEM_PREALLOCATED_BLOCK_SIZE_FOR_NONGRATIS_CHUNKS( \ 296 PORT_MEM_PREALLOCATED_BLOCK_NUM_CHUNKS(size)) 297 298 /** 299 * Macros for defining memory allocation wrappers. 300 * 301 * The function / file / line reference is not useful when portMemAlloc 302 * is called from a generic memory allocator function, such as the memCreate 303 * function in resman. 304 * 305 * These macros can be used to push the function /file / line reference up one 306 * level when defining a memory allocator function. In other words, log who 307 * calls memCreate instead of logging memCreate. 308 * 309 * These macros are also used throughout memory-tracking.c 310 */ 311 #if PORT_MEM_TRACK_USE_CALLERINFO 312 313 #define PORT_MEM_CALLERINFO_PARAM _portMemCallerInfo 314 #define PORT_MEM_CALLERINFO_TYPE_PARAM \ 315 PORT_MEM_CALLERINFO PORT_MEM_CALLERINFO_PARAM 316 #define PORT_MEM_CALLERINFO_COMMA_PARAM ,PORT_MEM_CALLERINFO_PARAM 317 #define PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM ,PORT_MEM_CALLERINFO_TYPE_PARAM 318 #define PORT_MEM_CALLINFO_FUNC(f) f##_CallerInfo 319 320 #else // PORT_MEM_TRACK_USE_CALLERINFO 321 322 #define PORT_MEM_CALLERINFO_PARAM 323 #define PORT_MEM_CALLERINFO_TYPE_PARAM 324 #define PORT_MEM_CALLERINFO_COMMA_PARAM 325 #define PORT_MEM_CALLERINFO_COMMA_TYPE_PARAM 326 #define PORT_MEM_CALLINFO_FUNC(f) f 327 328 #endif // PORT_MEM_TRACK_USE_CALLERINFO 329 330 #endif // _NVPORT_MEMORY_INTERNAL_H_ 331 /// @endcond 332