1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 *************************************************************************** 6 * @file lac_mem_pools.c 7 * 8 * @ingroup LacMemPool 9 * 10 * Memory Pool creation and mgmt function implementations 11 * 12 ***************************************************************************/ 13 14 #include "cpa.h" 15 #include "qat_utils.h" 16 #include "icp_accel_devices.h" 17 #include "icp_adf_init.h" 18 #include "icp_adf_transport.h" 19 #include "icp_adf_debug.h" 20 #include "lac_lock_free_stack.h" 21 #include "lac_mem_pools.h" 22 #include "lac_mem.h" 23 #include "lac_common.h" 24 #include "cpa_dc.h" 25 #include "dc_session.h" 26 #include "dc_datapath.h" 27 #include "icp_qat_fw_comp.h" 28 #include "icp_buffer_desc.h" 29 #include "lac_sym.h" 30 31 #define LAC_MEM_POOLS_NUM_SUPPORTED 32000 32 /**< @ingroup LacMemPool 33 * Number of mem pools supported */ 34 35 #define LAC_MEM_POOLS_NAME_SIZE 17 36 /**< @ingroup LacMemPool 37 * 16 bytes plus '\\0' terminator */ 38 39 /**< @ingroup LacMemPool 40 * This structure is used to manage each pool created using this utility 41 * feature. The client will maintain a pointer (identifier) to the created 42 * structure per pool. 43 */ 44 typedef struct lac_mem_pool_hdr_s { 45 lock_free_stack_t stack; 46 char poolName[LAC_MEM_POOLS_NAME_SIZE]; /*16 bytes of a pool name */ 47 /**< up to 16 bytes of a pool name */ 48 unsigned int numElementsInPool; 49 /**< number of elements in the Pool */ 50 unsigned int blkSizeInBytes; 51 /**< Block size in bytes */ 52 unsigned int blkAlignmentInBytes; 53 /**< block alignment in bytes */ 54 lac_mem_blk_t **trackBlks; 55 /* An array of mem block pointers to track the allocated entries in pool 56 */ 57 volatile size_t availBlks; 58 /* Number of blocks available for allocation in this pool */ 59 } lac_mem_pool_hdr_t; 60 61 static lac_mem_pool_hdr_t *lac_mem_pools[LAC_MEM_POOLS_NUM_SUPPORTED] = { 62 NULL 63 }; 64 /**< @ingroup LacMemPool 65 * Array of pointers to the mem pool header structure 66 */ 67 68 LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t); 69 /**< @ingroup LacMemPool 70 * local constant for quickening computation of additional space allocated 71 * for holding lac_mem_blk_t container-structure 72 */ 73 74 /** 75 ******************************************************************************* 76 * @ingroup LacMemPool 77 * This function cleans up a mem pool. 78 ******************************************************************************/ 79 void Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID); 80 81 static inline Cpa32U 82 Lac_MemPoolGetElementRealSize(Cpa32U blkSizeInBytes, Cpa32U blkAlignmentInBytes) 83 { 84 Cpa32U addSize = (blkAlignmentInBytes >= sizeof(lac_mem_blk_t) ? 85 blkAlignmentInBytes : 86 1 << (highest_bit_of_lac_mem_blk_t + 1)); 87 return blkSizeInBytes + addSize; 88 } 89 90 CpaStatus 91 Lac_MemPoolCreate(lac_memory_pool_id_t *pPoolID, 92 char *poolName, 93 unsigned int numElementsInPool, /*Number of elements*/ 94 unsigned int blkSizeInBytes, /*Block Size in bytes*/ 95 unsigned int blkAlignmentInBytes, /*Block alignment (bytes)*/ 96 CpaBoolean trackMemory, 97 Cpa32U node) 98 { 99 unsigned int poolSearch = 0; 100 unsigned int counter = 0; 101 lac_mem_blk_t *pMemBlkCurrent = NULL; 102 103 void *pMemBlk = NULL; 104 105 if (pPoolID == NULL) { 106 QAT_UTILS_LOG("Invalid Pool ID param\n"); 107 return CPA_STATUS_INVALID_PARAM; /*Error*/ 108 } 109 110 /* Find First available Pool return error otherwise */ 111 while (lac_mem_pools[poolSearch] != NULL) { 112 poolSearch++; 113 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) { 114 QAT_UTILS_LOG( 115 "No more memory pools available for allocation.\n"); 116 return CPA_STATUS_FAIL; 117 } 118 } 119 120 /* Allocate a Pool header */ 121 lac_mem_pools[poolSearch] = LAC_OS_MALLOC(sizeof(lac_mem_pool_hdr_t)); 122 if (NULL == lac_mem_pools[poolSearch]) { 123 QAT_UTILS_LOG( 124 "Unable to allocate memory for creation of the pool.\n"); 125 return CPA_STATUS_RESOURCE; /*Error*/ 126 } 127 memset(lac_mem_pools[poolSearch], 0, sizeof(lac_mem_pool_hdr_t)); 128 129 /* Copy in Pool Name */ 130 if (poolName != NULL) { 131 snprintf(lac_mem_pools[poolSearch]->poolName, 132 LAC_MEM_POOLS_NAME_SIZE, 133 "%s", 134 poolName); 135 } else { 136 LAC_OS_FREE(lac_mem_pools[poolSearch]); 137 lac_mem_pools[poolSearch] = NULL; 138 QAT_UTILS_LOG("Invalid Pool Name pointer\n"); 139 return CPA_STATUS_INVALID_PARAM; /*Error*/ 140 } 141 142 /* Allocate table for tracking memory blocks */ 143 if (CPA_TRUE == trackMemory) { 144 lac_mem_pools[poolSearch]->trackBlks = LAC_OS_MALLOC( 145 (sizeof(lac_mem_blk_t *) * numElementsInPool)); 146 if (NULL == lac_mem_pools[poolSearch]->trackBlks) { 147 LAC_OS_FREE(lac_mem_pools[poolSearch]); 148 lac_mem_pools[poolSearch] = NULL; 149 QAT_UTILS_LOG( 150 "Unable to allocate memory for tracking memory blocks.\n"); 151 return CPA_STATUS_RESOURCE; /*Error*/ 152 } 153 } else { 154 lac_mem_pools[poolSearch]->trackBlks = NULL; 155 } 156 157 lac_mem_pools[poolSearch]->availBlks = 0; 158 lac_mem_pools[poolSearch]->stack = _init_stack(); 159 160 /* Calculate alignment needed for allocation */ 161 for (counter = 0; counter < numElementsInPool; counter++) { 162 CpaPhysicalAddr physAddr = 0; 163 /* realSize is computed for allocation of blkSize bytes + 164 additional 165 capacity for lac_mem_blk_t structure storage due to the some 166 OSes 167 (BSD) limitations for memory alignment to be power of 2; 168 sizeof(lac_mem_blk_t) is being round up to the closest power 169 of 2 - 170 optimised towards the least CPU overhead but at additional 171 memory 172 cost 173 */ 174 Cpa32U realSize = 175 Lac_MemPoolGetElementRealSize(blkSizeInBytes, 176 blkAlignmentInBytes); 177 Cpa32U addSize = realSize - blkSizeInBytes; 178 179 if (CPA_STATUS_SUCCESS != LAC_OS_CAMALLOC(&pMemBlk, 180 realSize, 181 blkAlignmentInBytes, 182 node)) { 183 Lac_MemPoolCleanUpInternal(lac_mem_pools[poolSearch]); 184 lac_mem_pools[poolSearch] = NULL; 185 QAT_UTILS_LOG( 186 "Unable to allocate contiguous chunk of memory.\n"); 187 return CPA_STATUS_RESOURCE; 188 } 189 190 /* Calcaulate various offsets */ 191 physAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 192 (void *)((LAC_ARCH_UINT)pMemBlk + addSize)); 193 194 /* physAddr is now already aligned to the greater power of 2: 195 blkAlignmentInBytes or sizeof(lac_mem_blk_t) round up 196 We safely put the structure right before the blkSize 197 real data block 198 */ 199 pMemBlkCurrent = 200 (lac_mem_blk_t *)(((LAC_ARCH_UINT)(pMemBlk)) + addSize - 201 sizeof(lac_mem_blk_t)); 202 203 pMemBlkCurrent->physDataPtr = physAddr; 204 pMemBlkCurrent->pMemAllocPtr = pMemBlk; 205 pMemBlkCurrent->pPoolID = lac_mem_pools[poolSearch]; 206 pMemBlkCurrent->isInUse = CPA_FALSE; 207 pMemBlkCurrent->pNext = NULL; 208 209 push(&lac_mem_pools[poolSearch]->stack, pMemBlkCurrent); 210 211 /* Store allocated memory pointer */ 212 if (lac_mem_pools[poolSearch]->trackBlks != NULL) { 213 (lac_mem_pools[poolSearch]->trackBlks[counter]) = 214 (lac_mem_blk_t *)pMemBlkCurrent; 215 } 216 __sync_add_and_fetch(&lac_mem_pools[poolSearch]->availBlks, 1); 217 (lac_mem_pools[poolSearch])->numElementsInPool = counter + 1; 218 } 219 220 /* Set Pool details in the header */ 221 (lac_mem_pools[poolSearch])->blkSizeInBytes = blkSizeInBytes; 222 (lac_mem_pools[poolSearch])->blkAlignmentInBytes = blkAlignmentInBytes; 223 /* Set the Pool ID output parameter */ 224 *pPoolID = (LAC_ARCH_UINT)(lac_mem_pools[poolSearch]); 225 /* Success */ 226 return CPA_STATUS_SUCCESS; 227 } 228 229 void * 230 Lac_MemPoolEntryAlloc(lac_memory_pool_id_t poolID) 231 { 232 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 233 lac_mem_blk_t *pMemBlkCurrent = NULL; 234 235 /* Explicitly removing NULL PoolID check for speed */ 236 if (pPoolID == NULL) { 237 QAT_UTILS_LOG("Invalid Pool ID"); 238 return NULL; 239 } 240 241 /* Remove block from pool */ 242 pMemBlkCurrent = pop(&pPoolID->stack); 243 if (NULL == pMemBlkCurrent) { 244 return (void *)CPA_STATUS_RETRY; 245 } 246 __sync_sub_and_fetch(&pPoolID->availBlks, 1); 247 pMemBlkCurrent->isInUse = CPA_TRUE; 248 return (void *)((LAC_ARCH_UINT)(pMemBlkCurrent) + 249 sizeof(lac_mem_blk_t)); 250 } 251 252 void 253 Lac_MemPoolEntryFree(void *pEntry) 254 { 255 lac_mem_blk_t *pMemBlk = NULL; 256 257 /* Explicitly NULL pointer check */ 258 if (pEntry == NULL) { 259 QAT_UTILS_LOG("Memory Handle NULL"); 260 return; 261 } 262 263 pMemBlk = 264 (lac_mem_blk_t *)((LAC_ARCH_UINT)pEntry - sizeof(lac_mem_blk_t)); 265 pMemBlk->isInUse = CPA_FALSE; 266 267 push(&pMemBlk->pPoolID->stack, pMemBlk); 268 __sync_add_and_fetch(&pMemBlk->pPoolID->availBlks, 1); 269 } 270 271 void 272 Lac_MemPoolDestroy(lac_memory_pool_id_t poolID) 273 { 274 unsigned int poolSearch = 0; 275 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 276 277 if (pPoolID != NULL) { 278 /*Remove entry from table*/ 279 while (lac_mem_pools[poolSearch] != pPoolID) { 280 poolSearch++; 281 282 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) { 283 QAT_UTILS_LOG("Invalid Pool ID submitted.\n"); 284 return; 285 } 286 } 287 288 lac_mem_pools[poolSearch] = NULL; /*Remove handle from pool*/ 289 290 Lac_MemPoolCleanUpInternal(pPoolID); 291 } 292 } 293 294 void 295 Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID) 296 { 297 lac_mem_blk_t *pCurrentBlk = NULL; 298 void *pFreePtr = NULL; 299 Cpa32U count = 0; 300 301 if (pPoolID->trackBlks == NULL) { 302 pCurrentBlk = pop(&pPoolID->stack); 303 304 while (pCurrentBlk != NULL) { 305 /* Free Data Blocks */ 306 pFreePtr = pCurrentBlk->pMemAllocPtr; 307 pCurrentBlk = pop(&pPoolID->stack); 308 LAC_OS_CAFREE(pFreePtr); 309 } 310 } else { 311 for (count = 0; count < pPoolID->numElementsInPool; count++) { 312 pFreePtr = (pPoolID->trackBlks[count])->pMemAllocPtr; 313 LAC_OS_CAFREE(pFreePtr); 314 } 315 LAC_OS_FREE(pPoolID->trackBlks); 316 } 317 LAC_OS_FREE(pPoolID); 318 } 319 320 unsigned int 321 Lac_MemPoolAvailableEntries(lac_memory_pool_id_t poolID) 322 { 323 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 324 if (pPoolID == NULL) { 325 QAT_UTILS_LOG("Invalid Pool ID\n"); 326 return 0; 327 } 328 return pPoolID->availBlks; 329 } 330 331 void 332 Lac_MemPoolStatsShow(void) 333 { 334 unsigned int index = 0; 335 QAT_UTILS_LOG(SEPARATOR BORDER 336 " Memory Pools Stats\n" SEPARATOR); 337 338 while (index < LAC_MEM_POOLS_NUM_SUPPORTED) { 339 if (lac_mem_pools[index] != NULL) { 340 QAT_UTILS_LOG( 341 BORDER " Pool Name: %s \n" BORDER 342 " No. Elements in Pool: %10u \n" BORDER 343 " Element Size in Bytes: %10u \n" BORDER 344 " Alignment in Bytes: %10u \n" BORDER 345 " No. Available Blocks: %10zu \n" SEPARATOR, 346 lac_mem_pools[index]->poolName, 347 lac_mem_pools[index]->numElementsInPool, 348 lac_mem_pools[index]->blkSizeInBytes, 349 lac_mem_pools[index]->blkAlignmentInBytes, 350 lac_mem_pools[index]->availBlks); 351 } 352 index++; 353 } 354 } 355 356 static void 357 Lac_MemPoolInitSymCookies(lac_sym_cookie_t *pSymCookie) 358 { 359 pSymCookie->keyContentDescPhyAddr = 360 LAC_OS_VIRT_TO_PHYS_INTERNAL(pSymCookie->u.keyCookie.contentDesc); 361 pSymCookie->keyHashStateBufferPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 362 pSymCookie->u.keyCookie.hashStateBuffer); 363 pSymCookie->keySslKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 364 &(pSymCookie->u.keyCookie.u.sslKeyInput)); 365 pSymCookie->keyTlsKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 366 &(pSymCookie->u.keyCookie.u.tlsKeyInput)); 367 } 368 369 CpaStatus 370 Lac_MemPoolInitSymCookiesPhyAddr(lac_memory_pool_id_t poolID) 371 { 372 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 373 lac_sym_cookie_t *pSymCookie = NULL; 374 lac_mem_blk_t *pCurrentBlk = NULL; 375 376 if (NULL == pPoolID) { 377 QAT_UTILS_LOG("Invalid Pool ID\n"); 378 return CPA_STATUS_FAIL; 379 } 380 381 if (pPoolID->trackBlks == NULL) { 382 pCurrentBlk = top(&pPoolID->stack); 383 384 while (pCurrentBlk != NULL) { 385 pSymCookie = 386 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) + 387 sizeof(lac_mem_blk_t)); 388 pCurrentBlk = pCurrentBlk->pNext; 389 Lac_MemPoolInitSymCookies(pSymCookie); 390 } 391 } else { 392 Cpa32U count = 0; 393 394 for (count = 0; count < pPoolID->numElementsInPool; count++) { 395 pCurrentBlk = pPoolID->trackBlks[count]; 396 pSymCookie = 397 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) + 398 sizeof(lac_mem_blk_t)); 399 Lac_MemPoolInitSymCookies(pSymCookie); 400 } 401 } 402 return CPA_STATUS_SUCCESS; 403 } 404 405 CpaStatus 406 Lac_MemPoolInitDcCookiePhyAddr(lac_memory_pool_id_t poolID) 407 { 408 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 409 lac_mem_blk_t *pCurrentBlk = NULL; 410 411 if (NULL == pPoolID) { 412 QAT_UTILS_LOG("Invalid Pool ID\n"); 413 return CPA_STATUS_FAIL; 414 } 415 416 if (NULL == pPoolID->trackBlks) { 417 pCurrentBlk = top(&pPoolID->stack); 418 419 while (pCurrentBlk != NULL) { 420 pCurrentBlk = pCurrentBlk->pNext; 421 } 422 } else { 423 Cpa32U count = 0; 424 425 for (count = 0; count < pPoolID->numElementsInPool; count++) { 426 pCurrentBlk = pPoolID->trackBlks[count]; 427 } 428 } 429 return CPA_STATUS_SUCCESS; 430 } 431