1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 ***************************************************************************** 6 * @file lac_buffer_desc.c Utility functions for setting buffer descriptors 7 * 8 * @ingroup LacBufferDesc 9 * 10 *****************************************************************************/ 11 12 /* 13 ******************************************************************************* 14 * Include header files 15 ******************************************************************************* 16 */ 17 #include "qat_utils.h" 18 #include "icp_accel_devices.h" 19 #include "icp_adf_debug.h" 20 #include "icp_adf_init.h" 21 #include "lac_list.h" 22 #include "lac_sal_types.h" 23 #include "lac_buffer_desc.h" 24 #include "lac_mem.h" 25 #include "cpa_cy_common.h" 26 27 /* 28 ******************************************************************************* 29 * Define public/global function definitions 30 ******************************************************************************* 31 */ 32 /* Invalid physical address value */ 33 #define INVALID_PHYSICAL_ADDRESS 0 34 35 /* Indicates what type of buffer writes need to be perfomed */ 36 typedef enum lac_buff_write_op_e { 37 WRITE_NORMAL = 0, 38 WRITE_AND_GET_SIZE, 39 WRITE_AND_ALLOW_ZERO_BUFFER, 40 } lac_buff_write_op_t; 41 42 /* This function implements the buffer description writes for the traditional 43 * APIs */ 44 static CpaStatus 45 LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList, 46 Cpa64U *pBufListAlignedPhyAddr, 47 CpaBoolean isPhysicalAddress, 48 Cpa64U *totalDataLenInBytes, 49 sal_service_t *pService, 50 lac_buff_write_op_t operationType) 51 { 52 Cpa32U numBuffers = 0; 53 icp_qat_addr_width_t bufListDescPhyAddr = 0; 54 icp_qat_addr_width_t bufListAlignedPhyAddr = 0; 55 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 56 icp_buffer_list_desc_t *pBufferListDesc = NULL; 57 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL; 58 59 if (WRITE_AND_GET_SIZE == operationType) { 60 *totalDataLenInBytes = 0; 61 } 62 63 numBuffers = pUserBufferList->numBuffers; 64 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 65 66 /* 67 * Get the physical address of this descriptor - need to offset by the 68 * alignment restrictions on the buffer descriptors 69 */ 70 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL( 71 (*pService), pUserBufferList->pPrivateMetaData); 72 73 if (bufListDescPhyAddr == 0) { 74 QAT_UTILS_LOG( 75 "Unable to get the physical address of the metadata.\n"); 76 return CPA_STATUS_FAIL; 77 } 78 79 bufListAlignedPhyAddr = 80 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr, 81 ICP_DESCRIPTOR_ALIGNMENT_BYTES); 82 83 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)( 84 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData + 85 ((LAC_ARCH_UINT)bufListAlignedPhyAddr - 86 (LAC_ARCH_UINT)bufListDescPhyAddr)); 87 88 /* Go past the Buffer List descriptor to the list of buffer descriptors 89 */ 90 pCurrFlatBufDesc = 91 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers)); 92 93 pBufferListDesc->numBuffers = numBuffers; 94 95 if (WRITE_AND_GET_SIZE != operationType) { 96 /* Defining zero buffers is useful for example if running zero 97 * length 98 * hash */ 99 if (0 == numBuffers) { 100 /* In the case where there are zero buffers within the 101 * BufList 102 * it is required by firmware that the number is set to 103 * 1 104 * but the phyBuffer and dataLenInBytes are set to 105 * NULL.*/ 106 pBufferListDesc->numBuffers = 1; 107 pCurrFlatBufDesc->dataLenInBytes = 0; 108 pCurrFlatBufDesc->phyBuffer = 0; 109 } 110 } 111 112 while (0 != numBuffers) { 113 pCurrFlatBufDesc->dataLenInBytes = 114 pCurrClientFlatBuffer->dataLenInBytes; 115 116 if (WRITE_AND_GET_SIZE == operationType) { 117 /* Calculate the total data length in bytes */ 118 *totalDataLenInBytes += 119 pCurrClientFlatBuffer->dataLenInBytes; 120 } 121 122 /* Check if providing a physical address in the function. If not 123 * we 124 * need to convert it to a physical one */ 125 if (CPA_TRUE == isPhysicalAddress) { 126 pCurrFlatBufDesc->phyBuffer = 127 LAC_MEM_CAST_PTR_TO_UINT64( 128 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData)); 129 } else { 130 pCurrFlatBufDesc->phyBuffer = 131 LAC_MEM_CAST_PTR_TO_UINT64( 132 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 133 (*pService), pCurrClientFlatBuffer->pData)); 134 135 if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) { 136 if (INVALID_PHYSICAL_ADDRESS == 137 pCurrFlatBufDesc->phyBuffer) { 138 QAT_UTILS_LOG( 139 "Unable to get the physical address of the client buffer.\n"); 140 return CPA_STATUS_FAIL; 141 } 142 } 143 } 144 145 pCurrFlatBufDesc++; 146 pCurrClientFlatBuffer++; 147 148 numBuffers--; 149 } 150 151 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr; 152 return CPA_STATUS_SUCCESS; 153 } 154 155 /* This function implements the buffer description writes for the traditional 156 * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and 157 * GCM aglorithms */ 158 CpaStatus 159 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer( 160 const CpaBufferList *pUserBufferList, 161 Cpa64U *pBufListAlignedPhyAddr, 162 CpaBoolean isPhysicalAddress, 163 sal_service_t *pService) 164 { 165 return LacBuffDesc_CommonBufferListDescWrite( 166 pUserBufferList, 167 pBufListAlignedPhyAddr, 168 isPhysicalAddress, 169 NULL, 170 pService, 171 WRITE_AND_ALLOW_ZERO_BUFFER); 172 } 173 174 /* This function implements the buffer description writes for the traditional 175 * APIs */ 176 CpaStatus 177 LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList, 178 Cpa64U *pBufListAlignedPhyAddr, 179 CpaBoolean isPhysicalAddress, 180 sal_service_t *pService) 181 { 182 return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList, 183 pBufListAlignedPhyAddr, 184 isPhysicalAddress, 185 NULL, 186 pService, 187 WRITE_NORMAL); 188 } 189 190 /* This function does the same processing as LacBuffDesc_BufferListDescWrite 191 * but calculate as well the total length in bytes of the buffer list. */ 192 CpaStatus 193 LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList, 194 Cpa64U *pBufListAlignedPhyAddr, 195 CpaBoolean isPhysicalAddress, 196 Cpa64U *totalDataLenInBytes, 197 sal_service_t *pService) 198 { 199 Cpa32U numBuffers = 0; 200 icp_qat_addr_width_t bufListDescPhyAddr = 0; 201 icp_qat_addr_width_t bufListAlignedPhyAddr = 0; 202 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 203 icp_buffer_list_desc_t *pBufferListDesc = NULL; 204 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL; 205 *totalDataLenInBytes = 0; 206 207 numBuffers = pUserBufferList->numBuffers; 208 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 209 210 /* 211 * Get the physical address of this descriptor - need to offset by the 212 * alignment restrictions on the buffer descriptors 213 */ 214 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL( 215 (*pService), pUserBufferList->pPrivateMetaData); 216 217 if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) { 218 QAT_UTILS_LOG( 219 "Unable to get the physical address of the metadata.\n"); 220 return CPA_STATUS_FAIL; 221 } 222 223 bufListAlignedPhyAddr = 224 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr, 225 ICP_DESCRIPTOR_ALIGNMENT_BYTES); 226 227 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)( 228 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData + 229 ((LAC_ARCH_UINT)bufListAlignedPhyAddr - 230 (LAC_ARCH_UINT)bufListDescPhyAddr)); 231 232 /* Go past the Buffer List descriptor to the list of buffer descriptors 233 */ 234 pCurrFlatBufDesc = 235 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers)); 236 237 pBufferListDesc->numBuffers = numBuffers; 238 239 while (0 != numBuffers) { 240 pCurrFlatBufDesc->dataLenInBytes = 241 pCurrClientFlatBuffer->dataLenInBytes; 242 243 /* Calculate the total data length in bytes */ 244 *totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes; 245 246 if (isPhysicalAddress == CPA_TRUE) { 247 pCurrFlatBufDesc->phyBuffer = 248 LAC_MEM_CAST_PTR_TO_UINT64( 249 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData)); 250 } else { 251 pCurrFlatBufDesc->phyBuffer = 252 LAC_MEM_CAST_PTR_TO_UINT64( 253 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 254 (*pService), pCurrClientFlatBuffer->pData)); 255 256 if (pCurrFlatBufDesc->phyBuffer == 0) { 257 QAT_UTILS_LOG( 258 "Unable to get the physical address of the client buffer.\n"); 259 return CPA_STATUS_FAIL; 260 } 261 } 262 263 pCurrFlatBufDesc++; 264 pCurrClientFlatBuffer++; 265 266 numBuffers--; 267 } 268 269 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr; 270 return CPA_STATUS_SUCCESS; 271 } 272 273 CpaStatus 274 LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer, 275 Cpa64U *pPktSize, 276 lac_aligment_shift_t alignmentShiftExpected) 277 { 278 LAC_CHECK_NULL_PARAM(pUserFlatBuffer); 279 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData); 280 281 if (0 == pUserFlatBuffer->dataLenInBytes) { 282 QAT_UTILS_LOG("FlatBuffer empty\n"); 283 return CPA_STATUS_INVALID_PARAM; 284 } 285 286 /* Expected alignment */ 287 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) { 288 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData, 289 alignmentShiftExpected)) { 290 QAT_UTILS_LOG( 291 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n", 292 1 << alignmentShiftExpected); 293 return CPA_STATUS_INVALID_PARAM; 294 } 295 } 296 297 /* Update the total size of the packet. This function being called in a 298 * loop 299 * for an entire buffer list we need to increment the value */ 300 *pPktSize += pUserFlatBuffer->dataLenInBytes; 301 302 return CPA_STATUS_SUCCESS; 303 } 304 305 CpaStatus 306 LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer, 307 Cpa64U *pPktSize, 308 lac_aligment_shift_t alignmentShiftExpected) 309 { 310 LAC_CHECK_NULL_PARAM(pUserFlatBuffer); 311 312 if (0 != pUserFlatBuffer->dataLenInBytes) { 313 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData); 314 } 315 316 /* Expected alignment */ 317 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) { 318 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData, 319 alignmentShiftExpected)) { 320 QAT_UTILS_LOG( 321 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n", 322 1 << alignmentShiftExpected); 323 return CPA_STATUS_INVALID_PARAM; 324 } 325 } 326 327 /* Update the total size of the packet. This function being called in a 328 * loop 329 * for an entire buffer list we need to increment the value */ 330 *pPktSize += pUserFlatBuffer->dataLenInBytes; 331 332 return CPA_STATUS_SUCCESS; 333 } 334 335 CpaStatus 336 LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList, 337 Cpa64U *pPktSize, 338 lac_aligment_shift_t alignmentShiftExpected) 339 { 340 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 341 Cpa32U numBuffers = 0; 342 CpaStatus status = CPA_STATUS_SUCCESS; 343 344 LAC_CHECK_NULL_PARAM(pUserBufferList); 345 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers); 346 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData); 347 348 numBuffers = pUserBufferList->numBuffers; 349 350 if (0 == pUserBufferList->numBuffers) { 351 QAT_UTILS_LOG("Number of buffers is 0.\n"); 352 return CPA_STATUS_INVALID_PARAM; 353 } 354 355 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 356 357 *pPktSize = 0; 358 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) { 359 status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer, 360 pPktSize, 361 alignmentShiftExpected); 362 363 pCurrClientFlatBuffer++; 364 numBuffers--; 365 } 366 return status; 367 } 368 369 CpaStatus 370 LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList, 371 Cpa64U *pPktSize, 372 lac_aligment_shift_t alignmentShiftExpected) 373 { 374 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 375 Cpa32U numBuffers = 0; 376 CpaStatus status = CPA_STATUS_SUCCESS; 377 378 LAC_CHECK_NULL_PARAM(pUserBufferList); 379 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers); 380 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData); 381 382 numBuffers = pUserBufferList->numBuffers; 383 384 if (0 == pUserBufferList->numBuffers) { 385 QAT_UTILS_LOG("Number of buffers is 0.\n"); 386 return CPA_STATUS_INVALID_PARAM; 387 } 388 389 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 390 391 *pPktSize = 0; 392 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) { 393 status = 394 LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer, 395 pPktSize, 396 alignmentShiftExpected); 397 398 pCurrClientFlatBuffer++; 399 numBuffers--; 400 } 401 return status; 402 } 403 404 /** 405 ****************************************************************************** 406 * @ingroup LacBufferDesc 407 *****************************************************************************/ 408 void 409 LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList, 410 Cpa64U *pPktSize) 411 { 412 CpaFlatBuffer *pCurrClientFlatBuffer = NULL; 413 Cpa32U numBuffers = 0; 414 415 pCurrClientFlatBuffer = pUserBufferList->pBuffers; 416 numBuffers = pUserBufferList->numBuffers; 417 418 *pPktSize = 0; 419 while (0 != numBuffers) { 420 *pPktSize += pCurrClientFlatBuffer->dataLenInBytes; 421 pCurrClientFlatBuffer++; 422 numBuffers--; 423 } 424 } 425 426 void 427 LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList, 428 Cpa32U offset, 429 Cpa32U lenToZero) 430 { 431 Cpa32U zeroLen = 0, sizeLeftToZero = 0; 432 Cpa64U currentBufferSize = 0; 433 CpaFlatBuffer *pBuffer = NULL; 434 Cpa8U *pZero = NULL; 435 pBuffer = pBuffList->pBuffers; 436 437 /* Take a copy of total length to zero. */ 438 sizeLeftToZero = lenToZero; 439 440 while (sizeLeftToZero > 0) { 441 currentBufferSize = pBuffer->dataLenInBytes; 442 /* check where to start zeroing */ 443 if (offset >= currentBufferSize) { 444 /* Need to get to next buffer and reduce 445 * offset size by data len of buffer */ 446 offset = offset - pBuffer->dataLenInBytes; 447 pBuffer++; 448 } else { 449 /* Start to Zero from this position */ 450 pZero = (Cpa8U *)pBuffer->pData + offset; 451 452 /* Need to calculate the correct number of bytes to zero 453 * for this iteration and for this location. 454 */ 455 if (sizeLeftToZero >= pBuffer->dataLenInBytes) { 456 /* The size to zero is spanning buffers, zeroLen 457 * in 458 * this case is from pZero (position) to end of 459 * buffer. 460 */ 461 zeroLen = pBuffer->dataLenInBytes - offset; 462 } else { 463 /* zeroLen is set to sizeLeftToZero, then check 464 * if zeroLen and 465 * the offset is greater or equal to the size of 466 * the buffer, if 467 * yes, adjust the zeroLen to zero out the 468 * remainder of this 469 * buffer. 470 */ 471 zeroLen = sizeLeftToZero; 472 if ((zeroLen + offset) >= 473 pBuffer->dataLenInBytes) { 474 zeroLen = 475 pBuffer->dataLenInBytes - offset; 476 } 477 } /* end inner else */ 478 memset((void *)pZero, 0, zeroLen); 479 sizeLeftToZero = sizeLeftToZero - zeroLen; 480 /* offset is no longer required as any data left to zero 481 * is now 482 * at the start of the next buffer. set offset to zero 483 * and move on 484 * the buffer pointer to the next buffer. 485 */ 486 offset = 0; 487 pBuffer++; 488 489 } /* end outer else */ 490 491 } /* end while */ 492 } 493