1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /** 4 ***************************************************************************** 5 * @file lac_buffer_desc.c Utility functions for setting buffer descriptors 6 * 7 * @ingroup LacBufferDesc 8 * 9 *****************************************************************************/ 10 11 /* 12 ******************************************************************************* 13 * Include header files 14 ******************************************************************************* 15 */ 16 #include "qat_utils.h" 17 #include "icp_accel_devices.h" 18 #include "icp_adf_debug.h" 19 #include "icp_adf_init.h" 20 #include "lac_list.h" 21 #include "lac_sal_types.h" 22 #include "lac_buffer_desc.h" 23 #include "lac_mem.h" 24 #include "cpa_cy_common.h" 25 26 /* 27 ******************************************************************************* 28 * Define public/global function definitions 29 ******************************************************************************* 30 */ 31 /* Invalid physical address value */ 32 #define INVALID_PHYSICAL_ADDRESS 0 33 34 /* Indicates what type of buffer writes need to be perfomed */ 35 typedef enum lac_buff_write_op_e { 36 WRITE_NORMAL = 0, 37 WRITE_AND_GET_SIZE, 38 WRITE_AND_ALLOW_ZERO_BUFFER, 39 } lac_buff_write_op_t; 40 41 /* This function implements the buffer description writes for the traditional 42 * APIs */ 43 static CpaStatus 44 LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList, 45 Cpa64U *pBufListAlignedPhyAddr, 46 CpaBoolean isPhysicalAddress, 47 Cpa64U *totalDataLenInBytes, 48 sal_service_t *pService, 49 lac_buff_write_op_t operationType) 50 { 51 Cpa32U numBuffers = 0; 52 icp_qat_addr_width_t bufListDescPhyAddr = 0; 53 icp_qat_addr_width_t bufListAlignedPhyAddr = 0; 54 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 55 icp_buffer_list_desc_t *pBufferListDesc = NULL; 56 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL; 57 58 if (WRITE_AND_GET_SIZE == operationType) { 59 *totalDataLenInBytes = 0; 60 } 61 62 numBuffers = pUserBufferList->numBuffers; 63 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 64 65 /* 66 * Get the physical address of this descriptor - need to offset by the 67 * alignment restrictions on the buffer descriptors 68 */ 69 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL( 70 (*pService), pUserBufferList->pPrivateMetaData); 71 72 if (bufListDescPhyAddr == 0) { 73 QAT_UTILS_LOG( 74 "Unable to get the physical address of the metadata.\n"); 75 return CPA_STATUS_FAIL; 76 } 77 78 bufListAlignedPhyAddr = 79 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr, 80 ICP_DESCRIPTOR_ALIGNMENT_BYTES); 81 82 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)( 83 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData + 84 ((LAC_ARCH_UINT)bufListAlignedPhyAddr - 85 (LAC_ARCH_UINT)bufListDescPhyAddr)); 86 87 /* Go past the Buffer List descriptor to the list of buffer descriptors 88 */ 89 pCurrFlatBufDesc = 90 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers)); 91 92 pBufferListDesc->numBuffers = numBuffers; 93 94 if (WRITE_AND_GET_SIZE != operationType) { 95 /* Defining zero buffers is useful for example if running zero 96 * length 97 * hash */ 98 if (0 == numBuffers) { 99 /* In the case where there are zero buffers within the 100 * BufList 101 * it is required by firmware that the number is set to 102 * 1 103 * but the phyBuffer and dataLenInBytes are set to 104 * NULL.*/ 105 pBufferListDesc->numBuffers = 1; 106 pCurrFlatBufDesc->dataLenInBytes = 0; 107 pCurrFlatBufDesc->phyBuffer = 0; 108 } 109 } 110 111 while (0 != numBuffers) { 112 pCurrFlatBufDesc->dataLenInBytes = 113 pCurrClientFlatBuffer->dataLenInBytes; 114 115 if (WRITE_AND_GET_SIZE == operationType) { 116 /* Calculate the total data length in bytes */ 117 *totalDataLenInBytes += 118 pCurrClientFlatBuffer->dataLenInBytes; 119 } 120 121 /* Check if providing a physical address in the function. If not 122 * we 123 * need to convert it to a physical one */ 124 if (CPA_TRUE == isPhysicalAddress) { 125 pCurrFlatBufDesc->phyBuffer = 126 LAC_MEM_CAST_PTR_TO_UINT64( 127 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData)); 128 } else { 129 pCurrFlatBufDesc->phyBuffer = 130 LAC_MEM_CAST_PTR_TO_UINT64( 131 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 132 (*pService), pCurrClientFlatBuffer->pData)); 133 134 if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) { 135 if (INVALID_PHYSICAL_ADDRESS == 136 pCurrFlatBufDesc->phyBuffer) { 137 QAT_UTILS_LOG( 138 "Unable to get the physical address of the client buffer.\n"); 139 return CPA_STATUS_FAIL; 140 } 141 } 142 } 143 144 pCurrFlatBufDesc++; 145 pCurrClientFlatBuffer++; 146 147 numBuffers--; 148 } 149 150 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr; 151 return CPA_STATUS_SUCCESS; 152 } 153 154 /* This function implements the buffer description writes for the traditional 155 * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and 156 * GCM aglorithms */ 157 CpaStatus 158 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer( 159 const CpaBufferList *pUserBufferList, 160 Cpa64U *pBufListAlignedPhyAddr, 161 CpaBoolean isPhysicalAddress, 162 sal_service_t *pService) 163 { 164 return LacBuffDesc_CommonBufferListDescWrite( 165 pUserBufferList, 166 pBufListAlignedPhyAddr, 167 isPhysicalAddress, 168 NULL, 169 pService, 170 WRITE_AND_ALLOW_ZERO_BUFFER); 171 } 172 173 /* This function implements the buffer description writes for the traditional 174 * APIs */ 175 CpaStatus 176 LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList, 177 Cpa64U *pBufListAlignedPhyAddr, 178 CpaBoolean isPhysicalAddress, 179 sal_service_t *pService) 180 { 181 return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList, 182 pBufListAlignedPhyAddr, 183 isPhysicalAddress, 184 NULL, 185 pService, 186 WRITE_NORMAL); 187 } 188 189 /* This function does the same processing as LacBuffDesc_BufferListDescWrite 190 * but calculate as well the total length in bytes of the buffer list. */ 191 CpaStatus 192 LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList, 193 Cpa64U *pBufListAlignedPhyAddr, 194 CpaBoolean isPhysicalAddress, 195 Cpa64U *totalDataLenInBytes, 196 sal_service_t *pService) 197 { 198 Cpa32U numBuffers = 0; 199 icp_qat_addr_width_t bufListDescPhyAddr = 0; 200 icp_qat_addr_width_t bufListAlignedPhyAddr = 0; 201 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 202 icp_buffer_list_desc_t *pBufferListDesc = NULL; 203 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL; 204 *totalDataLenInBytes = 0; 205 206 numBuffers = pUserBufferList->numBuffers; 207 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 208 209 /* 210 * Get the physical address of this descriptor - need to offset by the 211 * alignment restrictions on the buffer descriptors 212 */ 213 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL( 214 (*pService), pUserBufferList->pPrivateMetaData); 215 216 if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) { 217 QAT_UTILS_LOG( 218 "Unable to get the physical address of the metadata.\n"); 219 return CPA_STATUS_FAIL; 220 } 221 222 bufListAlignedPhyAddr = 223 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr, 224 ICP_DESCRIPTOR_ALIGNMENT_BYTES); 225 226 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)( 227 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData + 228 ((LAC_ARCH_UINT)bufListAlignedPhyAddr - 229 (LAC_ARCH_UINT)bufListDescPhyAddr)); 230 231 /* Go past the Buffer List descriptor to the list of buffer descriptors 232 */ 233 pCurrFlatBufDesc = 234 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers)); 235 236 pBufferListDesc->numBuffers = numBuffers; 237 238 while (0 != numBuffers) { 239 pCurrFlatBufDesc->dataLenInBytes = 240 pCurrClientFlatBuffer->dataLenInBytes; 241 242 /* Calculate the total data length in bytes */ 243 *totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes; 244 245 if (isPhysicalAddress == CPA_TRUE) { 246 pCurrFlatBufDesc->phyBuffer = 247 LAC_MEM_CAST_PTR_TO_UINT64( 248 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData)); 249 } else { 250 pCurrFlatBufDesc->phyBuffer = 251 LAC_MEM_CAST_PTR_TO_UINT64( 252 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 253 (*pService), pCurrClientFlatBuffer->pData)); 254 255 if (pCurrFlatBufDesc->phyBuffer == 0) { 256 QAT_UTILS_LOG( 257 "Unable to get the physical address of the client buffer.\n"); 258 return CPA_STATUS_FAIL; 259 } 260 } 261 262 pCurrFlatBufDesc++; 263 pCurrClientFlatBuffer++; 264 265 numBuffers--; 266 } 267 268 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr; 269 return CPA_STATUS_SUCCESS; 270 } 271 272 CpaStatus 273 LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer, 274 Cpa64U *pPktSize, 275 lac_aligment_shift_t alignmentShiftExpected) 276 { 277 LAC_CHECK_NULL_PARAM(pUserFlatBuffer); 278 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData); 279 280 if (0 == pUserFlatBuffer->dataLenInBytes) { 281 QAT_UTILS_LOG("FlatBuffer empty\n"); 282 return CPA_STATUS_INVALID_PARAM; 283 } 284 285 /* Expected alignment */ 286 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) { 287 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData, 288 alignmentShiftExpected)) { 289 QAT_UTILS_LOG( 290 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n", 291 1 << alignmentShiftExpected); 292 return CPA_STATUS_INVALID_PARAM; 293 } 294 } 295 296 /* Update the total size of the packet. This function being called in a 297 * loop 298 * for an entire buffer list we need to increment the value */ 299 *pPktSize += pUserFlatBuffer->dataLenInBytes; 300 301 return CPA_STATUS_SUCCESS; 302 } 303 304 CpaStatus 305 LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer, 306 Cpa64U *pPktSize, 307 lac_aligment_shift_t alignmentShiftExpected) 308 { 309 LAC_CHECK_NULL_PARAM(pUserFlatBuffer); 310 311 if (0 != pUserFlatBuffer->dataLenInBytes) { 312 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData); 313 } 314 315 /* Expected alignment */ 316 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) { 317 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData, 318 alignmentShiftExpected)) { 319 QAT_UTILS_LOG( 320 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n", 321 1 << alignmentShiftExpected); 322 return CPA_STATUS_INVALID_PARAM; 323 } 324 } 325 326 /* Update the total size of the packet. This function being called in a 327 * loop 328 * for an entire buffer list we need to increment the value */ 329 *pPktSize += pUserFlatBuffer->dataLenInBytes; 330 331 return CPA_STATUS_SUCCESS; 332 } 333 334 CpaStatus 335 LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList, 336 Cpa64U *pPktSize, 337 lac_aligment_shift_t alignmentShiftExpected) 338 { 339 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 340 Cpa32U numBuffers = 0; 341 CpaStatus status = CPA_STATUS_SUCCESS; 342 343 LAC_CHECK_NULL_PARAM(pUserBufferList); 344 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers); 345 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData); 346 347 numBuffers = pUserBufferList->numBuffers; 348 349 if (0 == pUserBufferList->numBuffers) { 350 QAT_UTILS_LOG("Number of buffers is 0.\n"); 351 return CPA_STATUS_INVALID_PARAM; 352 } 353 354 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 355 356 *pPktSize = 0; 357 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) { 358 status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer, 359 pPktSize, 360 alignmentShiftExpected); 361 362 pCurrClientFlatBuffer++; 363 numBuffers--; 364 } 365 return status; 366 } 367 368 CpaStatus 369 LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList, 370 Cpa64U *pPktSize, 371 lac_aligment_shift_t alignmentShiftExpected) 372 { 373 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 374 Cpa32U numBuffers = 0; 375 CpaStatus status = CPA_STATUS_SUCCESS; 376 377 LAC_CHECK_NULL_PARAM(pUserBufferList); 378 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers); 379 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData); 380 381 numBuffers = pUserBufferList->numBuffers; 382 383 if (0 == pUserBufferList->numBuffers) { 384 QAT_UTILS_LOG("Number of buffers is 0.\n"); 385 return CPA_STATUS_INVALID_PARAM; 386 } 387 388 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 389 390 *pPktSize = 0; 391 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) { 392 status = 393 LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer, 394 pPktSize, 395 alignmentShiftExpected); 396 397 pCurrClientFlatBuffer++; 398 numBuffers--; 399 } 400 return status; 401 } 402 403 /** 404 ****************************************************************************** 405 * @ingroup LacBufferDesc 406 *****************************************************************************/ 407 void 408 LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList, 409 Cpa64U *pPktSize) 410 { 411 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 412 Cpa32U numBuffers = 0; 413 414 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 415 numBuffers = pUserBufferList->numBuffers; 416 417 *pPktSize = 0; 418 while (0 != numBuffers) { 419 *pPktSize += pCurrClientFlatBuffer->dataLenInBytes; 420 pCurrClientFlatBuffer++; 421 numBuffers--; 422 } 423 } 424 425 void 426 LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList, 427 Cpa32U offset, 428 Cpa32U lenToZero) 429 { 430 Cpa32U zeroLen = 0, sizeLeftToZero = 0; 431 Cpa64U currentBufferSize = 0; 432 CpaFlatBuffer *pBuffer = NULL; 433 Cpa8U *pZero = NULL; 434 pBuffer = pBuffList->pBuffers; 435 436 /* Take a copy of total length to zero. */ 437 sizeLeftToZero = lenToZero; 438 439 while (sizeLeftToZero > 0) { 440 currentBufferSize = pBuffer->dataLenInBytes; 441 /* check where to start zeroing */ 442 if (offset >= currentBufferSize) { 443 /* Need to get to next buffer and reduce 444 * offset size by data len of buffer */ 445 offset = offset - pBuffer->dataLenInBytes; 446 pBuffer++; 447 } else { 448 /* Start to Zero from this position */ 449 pZero = (Cpa8U *)pBuffer->pData + offset; 450 451 /* Need to calculate the correct number of bytes to zero 452 * for this iteration and for this location. 453 */ 454 if (sizeLeftToZero >= pBuffer->dataLenInBytes) { 455 /* The size to zero is spanning buffers, zeroLen 456 * in 457 * this case is from pZero (position) to end of 458 * buffer. 459 */ 460 zeroLen = pBuffer->dataLenInBytes - offset; 461 } else { 462 /* zeroLen is set to sizeLeftToZero, then check 463 * if zeroLen and 464 * the offset is greater or equal to the size of 465 * the buffer, if 466 * yes, adjust the zeroLen to zero out the 467 * remainder of this 468 * buffer. 469 */ 470 zeroLen = sizeLeftToZero; 471 if ((zeroLen + offset) >= 472 pBuffer->dataLenInBytes) { 473 zeroLen = 474 pBuffer->dataLenInBytes - offset; 475 } 476 } /* end inner else */ 477 memset((void *)pZero, 0, zeroLen); 478 sizeLeftToZero = sizeLeftToZero - zeroLen; 479 /* offset is no longer required as any data left to zero 480 * is now 481 * at the start of the next buffer. set offset to zero 482 * and move on 483 * the buffer pointer to the next buffer. 484 */ 485 offset = 0; 486 pBuffer++; 487 488 } /* end outer else */ 489 490 } /* end while */ 491 } 492