1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /** 4 *************************************************************************** 5 * @file lac_mem_pools.c 6 * 7 * @ingroup LacMemPool 8 * 9 * Memory Pool creation and mgmt function implementations 10 * 11 ***************************************************************************/ 12 13 #include "cpa.h" 14 #include "qat_utils.h" 15 #include "icp_accel_devices.h" 16 #include "icp_adf_init.h" 17 #include "icp_adf_transport.h" 18 #include "icp_adf_debug.h" 19 #include "lac_lock_free_stack.h" 20 #include "lac_mem_pools.h" 21 #include "lac_mem.h" 22 #include "lac_common.h" 23 #include "cpa_dc.h" 24 #include "dc_session.h" 25 #include "dc_datapath.h" 26 #include "icp_qat_fw_comp.h" 27 #include "icp_buffer_desc.h" 28 #include "lac_sym.h" 29 30 #define LAC_MEM_POOLS_NUM_SUPPORTED 32000 31 /**< @ingroup LacMemPool 32 * Number of mem pools supported */ 33 34 #define LAC_MEM_POOLS_NAME_SIZE 17 35 /**< @ingroup LacMemPool 36 * 16 bytes plus '\\0' terminator */ 37 38 /**< @ingroup LacMemPool 39 * This structure is used to manage each pool created using this utility 40 * feature. The client will maintain a pointer (identifier) to the created 41 * structure per pool. 42 */ 43 typedef struct lac_mem_pool_hdr_s { 44 lock_free_stack_t stack; 45 char poolName[LAC_MEM_POOLS_NAME_SIZE]; /*16 bytes of a pool name */ 46 /**< up to 16 bytes of a pool name */ 47 unsigned int numElementsInPool; 48 /**< number of elements in the Pool */ 49 unsigned int blkSizeInBytes; 50 /**< Block size in bytes */ 51 unsigned int blkAlignmentInBytes; 52 /**< block alignment in bytes */ 53 lac_mem_blk_t **trackBlks; 54 /* An array of mem block pointers to track the allocated entries in pool 55 */ 56 volatile size_t availBlks; 57 /* Number of blocks available for allocation in this pool */ 58 } lac_mem_pool_hdr_t; 59 60 static lac_mem_pool_hdr_t *lac_mem_pools[LAC_MEM_POOLS_NUM_SUPPORTED] = { 61 NULL 62 }; 63 /**< @ingroup LacMemPool 64 * Array of pointers to the mem pool header structure 65 */ 66 67 LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t); 68 /**< @ingroup LacMemPool 69 * local constant for quickening computation of additional space allocated 70 * for holding lac_mem_blk_t container-structure 71 */ 72 73 /** 74 ******************************************************************************* 75 * @ingroup LacMemPool 76 * This function cleans up a mem pool. 77 ******************************************************************************/ 78 void Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID); 79 80 static inline Cpa32U 81 Lac_MemPoolGetElementRealSize(Cpa32U blkSizeInBytes, Cpa32U blkAlignmentInBytes) 82 { 83 Cpa32U addSize = (blkAlignmentInBytes >= sizeof(lac_mem_blk_t) ? 84 blkAlignmentInBytes : 85 1 << (highest_bit_of_lac_mem_blk_t + 1)); 86 return blkSizeInBytes + addSize; 87 } 88 89 CpaStatus 90 Lac_MemPoolCreate(lac_memory_pool_id_t *pPoolID, 91 char *poolName, 92 unsigned int numElementsInPool, /*Number of elements*/ 93 unsigned int blkSizeInBytes, /*Block Size in bytes*/ 94 unsigned int blkAlignmentInBytes, /*Block alignment (bytes)*/ 95 CpaBoolean trackMemory, 96 Cpa32U node) 97 { 98 unsigned int poolSearch = 0; 99 unsigned int counter = 0; 100 lac_mem_blk_t *pMemBlkCurrent = NULL; 101 102 void *pMemBlk = NULL; 103 104 if (pPoolID == NULL) { 105 QAT_UTILS_LOG("Invalid Pool ID param\n"); 106 return CPA_STATUS_INVALID_PARAM; /*Error*/ 107 } 108 109 /* Find First available Pool return error otherwise */ 110 while (lac_mem_pools[poolSearch] != NULL) { 111 poolSearch++; 112 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) { 113 QAT_UTILS_LOG( 114 "No more memory pools available for allocation.\n"); 115 return CPA_STATUS_FAIL; 116 } 117 } 118 119 /* Allocate a Pool header */ 120 lac_mem_pools[poolSearch] = LAC_OS_MALLOC(sizeof(lac_mem_pool_hdr_t)); 121 if (NULL == lac_mem_pools[poolSearch]) { 122 QAT_UTILS_LOG( 123 "Unable to allocate memory for creation of the pool.\n"); 124 return CPA_STATUS_RESOURCE; /*Error*/ 125 } 126 memset(lac_mem_pools[poolSearch], 0, sizeof(lac_mem_pool_hdr_t)); 127 128 /* Copy in Pool Name */ 129 if (poolName != NULL) { 130 snprintf(lac_mem_pools[poolSearch]->poolName, 131 LAC_MEM_POOLS_NAME_SIZE, 132 "%s", 133 poolName); 134 } else { 135 LAC_OS_FREE(lac_mem_pools[poolSearch]); 136 lac_mem_pools[poolSearch] = NULL; 137 QAT_UTILS_LOG("Invalid Pool Name pointer\n"); 138 return CPA_STATUS_INVALID_PARAM; /*Error*/ 139 } 140 141 /* Allocate table for tracking memory blocks */ 142 if (CPA_TRUE == trackMemory) { 143 lac_mem_pools[poolSearch]->trackBlks = LAC_OS_MALLOC( 144 (sizeof(lac_mem_blk_t *) * numElementsInPool)); 145 if (NULL == lac_mem_pools[poolSearch]->trackBlks) { 146 LAC_OS_FREE(lac_mem_pools[poolSearch]); 147 lac_mem_pools[poolSearch] = NULL; 148 QAT_UTILS_LOG( 149 "Unable to allocate memory for tracking memory blocks.\n"); 150 return CPA_STATUS_RESOURCE; /*Error*/ 151 } 152 } else { 153 lac_mem_pools[poolSearch]->trackBlks = NULL; 154 } 155 156 lac_mem_pools[poolSearch]->availBlks = 0; 157 lac_mem_pools[poolSearch]->stack = _init_stack(); 158 159 /* Calculate alignment needed for allocation */ 160 for (counter = 0; counter < numElementsInPool; counter++) { 161 CpaPhysicalAddr physAddr = 0; 162 /* realSize is computed for allocation of blkSize bytes + 163 additional 164 capacity for lac_mem_blk_t structure storage due to the some 165 OSes 166 (BSD) limitations for memory alignment to be power of 2; 167 sizeof(lac_mem_blk_t) is being round up to the closest power 168 of 2 - 169 optimised towards the least CPU overhead but at additional 170 memory 171 cost 172 */ 173 Cpa32U realSize = 174 Lac_MemPoolGetElementRealSize(blkSizeInBytes, 175 blkAlignmentInBytes); 176 Cpa32U addSize = realSize - blkSizeInBytes; 177 178 if (CPA_STATUS_SUCCESS != LAC_OS_CAMALLOC(&pMemBlk, 179 realSize, 180 blkAlignmentInBytes, 181 node)) { 182 Lac_MemPoolCleanUpInternal(lac_mem_pools[poolSearch]); 183 lac_mem_pools[poolSearch] = NULL; 184 QAT_UTILS_LOG( 185 "Unable to allocate contiguous chunk of memory.\n"); 186 return CPA_STATUS_RESOURCE; 187 } 188 189 /* Calcaulate various offsets */ 190 physAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 191 (void *)((LAC_ARCH_UINT)pMemBlk + addSize)); 192 193 /* physAddr is now already aligned to the greater power of 2: 194 blkAlignmentInBytes or sizeof(lac_mem_blk_t) round up 195 We safely put the structure right before the blkSize 196 real data block 197 */ 198 pMemBlkCurrent = 199 (lac_mem_blk_t *)(((LAC_ARCH_UINT)(pMemBlk)) + addSize - 200 sizeof(lac_mem_blk_t)); 201 202 pMemBlkCurrent->physDataPtr = physAddr; 203 pMemBlkCurrent->pMemAllocPtr = pMemBlk; 204 pMemBlkCurrent->pPoolID = lac_mem_pools[poolSearch]; 205 pMemBlkCurrent->isInUse = CPA_FALSE; 206 pMemBlkCurrent->pNext = NULL; 207 208 push(&lac_mem_pools[poolSearch]->stack, pMemBlkCurrent); 209 210 /* Store allocated memory pointer */ 211 if (lac_mem_pools[poolSearch]->trackBlks != NULL) { 212 (lac_mem_pools[poolSearch]->trackBlks[counter]) = 213 (lac_mem_blk_t *)pMemBlkCurrent; 214 } 215 __sync_add_and_fetch(&lac_mem_pools[poolSearch]->availBlks, 1); 216 (lac_mem_pools[poolSearch])->numElementsInPool = counter + 1; 217 } 218 219 /* Set Pool details in the header */ 220 (lac_mem_pools[poolSearch])->blkSizeInBytes = blkSizeInBytes; 221 (lac_mem_pools[poolSearch])->blkAlignmentInBytes = blkAlignmentInBytes; 222 /* Set the Pool ID output parameter */ 223 *pPoolID = (LAC_ARCH_UINT)(lac_mem_pools[poolSearch]); 224 /* Success */ 225 return CPA_STATUS_SUCCESS; 226 } 227 228 void * 229 Lac_MemPoolEntryAlloc(lac_memory_pool_id_t poolID) 230 { 231 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 232 lac_mem_blk_t *pMemBlkCurrent = NULL; 233 234 /* Explicitly removing NULL PoolID check for speed */ 235 if (pPoolID == NULL) { 236 QAT_UTILS_LOG("Invalid Pool ID"); 237 return NULL; 238 } 239 240 /* Remove block from pool */ 241 pMemBlkCurrent = pop(&pPoolID->stack); 242 if (NULL == pMemBlkCurrent) { 243 return (void *)CPA_STATUS_RETRY; 244 } 245 __sync_sub_and_fetch(&pPoolID->availBlks, 1); 246 pMemBlkCurrent->isInUse = CPA_TRUE; 247 return (void *)((LAC_ARCH_UINT)(pMemBlkCurrent) + 248 sizeof(lac_mem_blk_t)); 249 } 250 251 void 252 Lac_MemPoolEntryFree(void *pEntry) 253 { 254 lac_mem_blk_t *pMemBlk = NULL; 255 256 /* Explicitly NULL pointer check */ 257 if (pEntry == NULL) { 258 QAT_UTILS_LOG("Memory Handle NULL"); 259 return; 260 } 261 262 pMemBlk = 263 (lac_mem_blk_t *)((LAC_ARCH_UINT)pEntry - sizeof(lac_mem_blk_t)); 264 pMemBlk->isInUse = CPA_FALSE; 265 266 push(&pMemBlk->pPoolID->stack, pMemBlk); 267 __sync_add_and_fetch(&pMemBlk->pPoolID->availBlks, 1); 268 } 269 270 void 271 Lac_MemPoolDestroy(lac_memory_pool_id_t poolID) 272 { 273 unsigned int poolSearch = 0; 274 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 275 276 if (pPoolID != NULL) { 277 /*Remove entry from table*/ 278 while (lac_mem_pools[poolSearch] != pPoolID) { 279 poolSearch++; 280 281 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) { 282 QAT_UTILS_LOG("Invalid Pool ID submitted.\n"); 283 return; 284 } 285 } 286 287 lac_mem_pools[poolSearch] = NULL; /*Remove handle from pool*/ 288 289 Lac_MemPoolCleanUpInternal(pPoolID); 290 } 291 } 292 293 void 294 Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID) 295 { 296 lac_mem_blk_t *pCurrentBlk = NULL; 297 void *pFreePtr = NULL; 298 Cpa32U count = 0; 299 300 if (pPoolID->trackBlks == NULL) { 301 pCurrentBlk = pop(&pPoolID->stack); 302 303 while (pCurrentBlk != NULL) { 304 /* Free Data Blocks */ 305 pFreePtr = pCurrentBlk->pMemAllocPtr; 306 pCurrentBlk = pop(&pPoolID->stack); 307 LAC_OS_CAFREE(pFreePtr); 308 } 309 } else { 310 for (count = 0; count < pPoolID->numElementsInPool; count++) { 311 pFreePtr = (pPoolID->trackBlks[count])->pMemAllocPtr; 312 LAC_OS_CAFREE(pFreePtr); 313 } 314 LAC_OS_FREE(pPoolID->trackBlks); 315 } 316 LAC_OS_FREE(pPoolID); 317 } 318 319 unsigned int 320 Lac_MemPoolAvailableEntries(lac_memory_pool_id_t poolID) 321 { 322 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 323 if (pPoolID == NULL) { 324 QAT_UTILS_LOG("Invalid Pool ID\n"); 325 return 0; 326 } 327 return pPoolID->availBlks; 328 } 329 330 void 331 Lac_MemPoolStatsShow(void) 332 { 333 unsigned int index = 0; 334 QAT_UTILS_LOG(SEPARATOR BORDER 335 " Memory Pools Stats\n" SEPARATOR); 336 337 while (index < LAC_MEM_POOLS_NUM_SUPPORTED) { 338 if (lac_mem_pools[index] != NULL) { 339 QAT_UTILS_LOG( 340 BORDER " Pool Name: %s \n" BORDER 341 " No. Elements in Pool: %10u \n" BORDER 342 " Element Size in Bytes: %10u \n" BORDER 343 " Alignment in Bytes: %10u \n" BORDER 344 " No. Available Blocks: %10zu \n" SEPARATOR, 345 lac_mem_pools[index]->poolName, 346 lac_mem_pools[index]->numElementsInPool, 347 lac_mem_pools[index]->blkSizeInBytes, 348 lac_mem_pools[index]->blkAlignmentInBytes, 349 lac_mem_pools[index]->availBlks); 350 } 351 index++; 352 } 353 } 354 355 static void 356 Lac_MemPoolInitSymCookies(lac_sym_cookie_t *pSymCookie) 357 { 358 pSymCookie->keyContentDescPhyAddr = 359 LAC_OS_VIRT_TO_PHYS_INTERNAL(pSymCookie->u.keyCookie.contentDesc); 360 pSymCookie->keyHashStateBufferPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 361 pSymCookie->u.keyCookie.hashStateBuffer); 362 pSymCookie->keySslKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 363 &(pSymCookie->u.keyCookie.u.sslKeyInput)); 364 pSymCookie->keyTlsKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL( 365 &(pSymCookie->u.keyCookie.u.tlsKeyInput)); 366 } 367 368 CpaStatus 369 Lac_MemPoolInitSymCookiesPhyAddr(lac_memory_pool_id_t poolID) 370 { 371 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 372 lac_sym_cookie_t *pSymCookie = NULL; 373 lac_mem_blk_t *pCurrentBlk = NULL; 374 375 if (NULL == pPoolID) { 376 QAT_UTILS_LOG("Invalid Pool ID\n"); 377 return CPA_STATUS_FAIL; 378 } 379 380 if (pPoolID->trackBlks == NULL) { 381 pCurrentBlk = top(&pPoolID->stack); 382 383 while (pCurrentBlk != NULL) { 384 pSymCookie = 385 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) + 386 sizeof(lac_mem_blk_t)); 387 pCurrentBlk = pCurrentBlk->pNext; 388 Lac_MemPoolInitSymCookies(pSymCookie); 389 } 390 } else { 391 Cpa32U count = 0; 392 393 for (count = 0; count < pPoolID->numElementsInPool; count++) { 394 pCurrentBlk = pPoolID->trackBlks[count]; 395 pSymCookie = 396 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) + 397 sizeof(lac_mem_blk_t)); 398 Lac_MemPoolInitSymCookies(pSymCookie); 399 } 400 } 401 return CPA_STATUS_SUCCESS; 402 } 403 404 CpaStatus 405 Lac_MemPoolInitDcCookiePhyAddr(lac_memory_pool_id_t poolID) 406 { 407 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID; 408 lac_mem_blk_t *pCurrentBlk = NULL; 409 410 if (NULL == pPoolID) { 411 QAT_UTILS_LOG("Invalid Pool ID\n"); 412 return CPA_STATUS_FAIL; 413 } 414 415 if (NULL == pPoolID->trackBlks) { 416 pCurrentBlk = top(&pPoolID->stack); 417 418 while (pCurrentBlk != NULL) { 419 pCurrentBlk = pCurrentBlk->pNext; 420 } 421 } else { 422 Cpa32U count = 0; 423 424 for (count = 0; count < pPoolID->numElementsInPool; count++) { 425 pCurrentBlk = pPoolID->trackBlks[count]; 426 } 427 } 428 return CPA_STATUS_SUCCESS; 429 } 430