1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 4 /** 5 *************************************************************************** 6 * @file lac_sym_dp.c 7 * Implementation of the symmetric data plane API 8 * 9 * @ingroup cpaCySymDp 10 ***************************************************************************/ 11 12 /* 13 ******************************************************************************* 14 * Include public/global header files 15 ******************************************************************************* 16 */ 17 18 #include "cpa.h" 19 #include "cpa_cy_sym.h" 20 #include "cpa_cy_sym_dp.h" 21 22 /* 23 ******************************************************************************* 24 * Include private header files 25 ******************************************************************************* 26 */ 27 28 #include "icp_accel_devices.h" 29 #include "icp_adf_init.h" 30 #include "icp_adf_transport.h" 31 #include "icp_adf_transport_dp.h" 32 #include "icp_adf_debug.h" 33 #include "icp_sal_poll.h" 34 35 #include "qat_utils.h" 36 37 #include "lac_list.h" 38 #include "lac_log.h" 39 #include "lac_mem.h" 40 #include "lac_sal_types_crypto.h" 41 #include "lac_sym.h" 42 #include "lac_sym_cipher.h" 43 #include "lac_sym_auth_enc.h" 44 #include "lac_sym_qat_cipher.h" 45 #include "sal_service_state.h" 46 #include "sal_hw_gen.h" 47 48 typedef void (*write_ringMsgFunc_t)(CpaCySymDpOpData *pRequest, 49 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg); 50 51 /** 52 ***************************************************************************** 53 * @ingroup cpaCySymDp 54 * Check that the operation data is valid 55 * 56 * @description 57 * Check that all the parameters defined in the operation data are valid 58 * 59 * @param[in] pRequest Pointer to an operation data for crypto 60 * data plane API 61 * 62 * @retval CPA_STATUS_SUCCESS Function executed successfully 63 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 64 * 65 *****************************************************************************/ 66 static CpaStatus 67 LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) 68 { 69 lac_session_desc_t *pSessionDesc = NULL; 70 CpaCySymCipherAlgorithm cipher = 0; 71 CpaCySymHashAlgorithm hash = 0; 72 Cpa32U capabilitiesMask = 0; 73 74 LAC_CHECK_NULL_PARAM(pRequest); 75 LAC_CHECK_NULL_PARAM(pRequest->instanceHandle); 76 LAC_CHECK_NULL_PARAM(pRequest->sessionCtx); 77 78 /* Ensure this is a crypto instance */ 79 SAL_CHECK_INSTANCE_TYPE(pRequest->instanceHandle, 80 (SAL_SERVICE_TYPE_CRYPTO | 81 SAL_SERVICE_TYPE_CRYPTO_SYM)); 82 83 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 84 if (NULL == pSessionDesc) { 85 do { 86 qatUtilsSleep(500); 87 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET( 88 pRequest->sessionCtx); 89 } while (NULL == pSessionDesc); 90 } 91 if (NULL == pSessionDesc) { 92 LAC_INVALID_PARAM_LOG("Session context not as expected"); 93 return CPA_STATUS_INVALID_PARAM; 94 } 95 96 if (CPA_FALSE == pSessionDesc->isDPSession) { 97 LAC_INVALID_PARAM_LOG( 98 "Session not initialised for data plane API"); 99 return CPA_STATUS_INVALID_PARAM; 100 } 101 102 /*check whether Payload size is zero for CHACHA-POLY */ 103 if ((CPA_CY_SYM_CIPHER_CHACHA == pSessionDesc->cipherAlgorithm) && 104 (CPA_CY_SYM_HASH_POLY == pSessionDesc->hashAlgorithm) && 105 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation)) { 106 if (!pRequest->messageLenToCipherInBytes) { 107 LAC_INVALID_PARAM_LOG( 108 "Invalid messageLenToCipherInBytes for CHACHA-POLY"); 109 return CPA_STATUS_INVALID_PARAM; 110 } 111 } 112 113 if (0 == pRequest->srcBuffer) { 114 LAC_INVALID_PARAM_LOG("Invalid srcBuffer"); 115 return CPA_STATUS_INVALID_PARAM; 116 } 117 if (0 == pRequest->dstBuffer) { 118 LAC_INVALID_PARAM_LOG("Invalid destBuffer"); 119 return CPA_STATUS_INVALID_PARAM; 120 } 121 if (0 == pRequest->thisPhys) { 122 LAC_INVALID_PARAM_LOG("Invalid thisPhys"); 123 return CPA_STATUS_INVALID_PARAM; 124 } 125 126 /* Check that src buffer Len = dst buffer Len 127 Note this also checks that they are of the same type */ 128 if (pRequest->srcBufferLen != pRequest->dstBufferLen) { 129 LAC_INVALID_PARAM_LOG( 130 "Source and Destination buffer lengths need to be equal"); 131 return CPA_STATUS_INVALID_PARAM; 132 } 133 134 /* digestVerify and digestIsAppended on Hash-Only operation not 135 * supported */ 136 if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify && 137 (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH)) { 138 LAC_INVALID_PARAM_LOG( 139 "digestVerify and digestIsAppended set " 140 "on Hash-Only operation is not supported"); 141 return CPA_STATUS_INVALID_PARAM; 142 } 143 144 /* Cipher specific tests */ 145 if (CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) { 146 /* Perform IV check */ 147 switch (pSessionDesc->cipherAlgorithm) { 148 case CPA_CY_SYM_CIPHER_AES_CTR: 149 case CPA_CY_SYM_CIPHER_3DES_CTR: 150 case CPA_CY_SYM_CIPHER_SM4_CTR: 151 case CPA_CY_SYM_CIPHER_AES_GCM: 152 case CPA_CY_SYM_CIPHER_CHACHA: 153 case CPA_CY_SYM_CIPHER_AES_CBC: 154 case CPA_CY_SYM_CIPHER_DES_CBC: 155 case CPA_CY_SYM_CIPHER_3DES_CBC: 156 case CPA_CY_SYM_CIPHER_SM4_CBC: 157 case CPA_CY_SYM_CIPHER_AES_F8: { 158 Cpa32U ivLenInBytes = LacSymQat_CipherIvSizeBytesGet( 159 pSessionDesc->cipherAlgorithm); 160 if (pRequest->ivLenInBytes != ivLenInBytes) { 161 if (!(/* GCM with 12 byte IV is OK */ 162 (LAC_CIPHER_IS_GCM( 163 pSessionDesc->cipherAlgorithm) && 164 pRequest->ivLenInBytes == 165 LAC_CIPHER_IV_SIZE_GCM_12))) { 166 LAC_INVALID_PARAM_LOG( 167 "invalid cipher IV size"); 168 return CPA_STATUS_INVALID_PARAM; 169 } 170 } 171 if (0 == pRequest->iv) { 172 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 173 return CPA_STATUS_INVALID_PARAM; 174 } 175 /* pRequest->pIv is only used for CCM so is not checked 176 * here */ 177 } break; 178 case CPA_CY_SYM_CIPHER_KASUMI_F8: { 179 if (LAC_CIPHER_KASUMI_F8_IV_LENGTH != 180 pRequest->ivLenInBytes) { 181 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 182 return CPA_STATUS_INVALID_PARAM; 183 } 184 if (0 == pRequest->iv) { 185 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 186 return CPA_STATUS_INVALID_PARAM; 187 } 188 } break; 189 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: { 190 if (ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ != 191 pRequest->ivLenInBytes) { 192 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 193 return CPA_STATUS_INVALID_PARAM; 194 } 195 if (0 == pRequest->iv) { 196 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 197 return CPA_STATUS_INVALID_PARAM; 198 } 199 } break; 200 case CPA_CY_SYM_CIPHER_ZUC_EEA3: { 201 if (ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ != 202 pRequest->ivLenInBytes) { 203 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 204 return CPA_STATUS_INVALID_PARAM; 205 } 206 if (0 == pRequest->iv) { 207 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 208 return CPA_STATUS_INVALID_PARAM; 209 } 210 } break; 211 case CPA_CY_SYM_CIPHER_AES_CCM: { 212 if (CPA_STATUS_SUCCESS != 213 LacSymAlgChain_CheckCCMData( 214 pRequest->pAdditionalAuthData, 215 pRequest->pIv, 216 pRequest->messageLenToCipherInBytes, 217 pRequest->ivLenInBytes)) { 218 return CPA_STATUS_INVALID_PARAM; 219 } 220 } break; 221 default: 222 break; 223 } 224 /* Perform algorithm-specific checks */ 225 switch (pSessionDesc->cipherAlgorithm) { 226 case CPA_CY_SYM_CIPHER_ARC4: 227 case CPA_CY_SYM_CIPHER_AES_CTR: 228 case CPA_CY_SYM_CIPHER_3DES_CTR: 229 case CPA_CY_SYM_CIPHER_SM4_CTR: 230 case CPA_CY_SYM_CIPHER_AES_CCM: 231 case CPA_CY_SYM_CIPHER_AES_GCM: 232 case CPA_CY_SYM_CIPHER_CHACHA: 233 case CPA_CY_SYM_CIPHER_KASUMI_F8: 234 case CPA_CY_SYM_CIPHER_AES_F8: 235 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: 236 case CPA_CY_SYM_CIPHER_ZUC_EEA3: 237 /* No action needed */ 238 break; 239 default: { 240 /* Mask & check below is based on assumption that block 241 * size is a power of 2. If data size is not a multiple 242 * of the block size, the "remainder" bits selected by 243 * the mask be non-zero 244 */ 245 if (pRequest->messageLenToCipherInBytes & 246 (LacSymQat_CipherBlockSizeBytesGet( 247 pSessionDesc->cipherAlgorithm) - 248 1)) { 249 LAC_INVALID_PARAM_LOG( 250 "Data size must be block size" 251 " multiple"); 252 return CPA_STATUS_INVALID_PARAM; 253 } 254 } 255 } 256 257 cipher = pSessionDesc->cipherAlgorithm; 258 hash = pSessionDesc->hashAlgorithm; 259 capabilitiesMask = 260 ((sal_crypto_service_t *)pRequest->instanceHandle) 261 ->generic_service_info.capabilitiesMask; 262 if (LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && 263 (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) { 264 /* For CHACHA and AES_GCM single pass there is an AAD 265 * buffer if aadLenInBytes is nonzero. AES_GMAC AAD is 266 * stored in source buffer, therefore there is no 267 * separate AAD buffer. */ 268 if ((0 != pSessionDesc->aadLenInBytes) && 269 (CPA_CY_SYM_HASH_AES_GMAC != 270 pSessionDesc->hashAlgorithm)) { 271 LAC_CHECK_NULL_PARAM( 272 pRequest->pAdditionalAuthData); 273 } 274 275 /* Ensure AAD length for AES_GMAC spc */ 276 if ((CPA_CY_SYM_HASH_AES_GMAC == hash) && 277 (ICP_QAT_FW_SPC_AAD_SZ_MAX < 278 pRequest->messageLenToHashInBytes)) { 279 LAC_INVALID_PARAM_LOG( 280 "aadLenInBytes for AES_GMAC"); 281 return CPA_STATUS_INVALID_PARAM; 282 } 283 } 284 } 285 286 /* Hash specific tests */ 287 if (CPA_CY_SYM_OP_CIPHER != pSessionDesc->symOperation) { 288 /* For CCM, snow3G and ZUC there is always an AAD buffer 289 For GCM there is an AAD buffer if aadLenInBytes is 290 nonzero */ 291 if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) || 292 (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm && 293 (0 != pSessionDesc->aadLenInBytes))) { 294 LAC_CHECK_NULL_PARAM(pRequest->pAdditionalAuthData); 295 if (0 == pRequest->additionalAuthData) { 296 LAC_INVALID_PARAM_LOG( 297 "Invalid additionalAuthData"); 298 return CPA_STATUS_INVALID_PARAM; 299 } 300 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 301 pSessionDesc->hashAlgorithm || 302 CPA_CY_SYM_HASH_ZUC_EIA3 == 303 pSessionDesc->hashAlgorithm) { 304 if (0 == pRequest->additionalAuthData) { 305 LAC_INVALID_PARAM_LOG( 306 "Invalid additionalAuthData"); 307 return CPA_STATUS_INVALID_PARAM; 308 } 309 } 310 311 if ((CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm) && 312 (!pSessionDesc->digestIsAppended) && 313 (0 == pRequest->digestResult)) { 314 LAC_INVALID_PARAM_LOG("Invalid digestResult"); 315 return CPA_STATUS_INVALID_PARAM; 316 } 317 318 if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) { 319 if ((pRequest->cryptoStartSrcOffsetInBytes + 320 pRequest->messageLenToCipherInBytes + 321 pSessionDesc->hashResultSize) > 322 pRequest->dstBufferLen) { 323 LAC_INVALID_PARAM_LOG( 324 "CCM - Not enough room for" 325 " digest in destination buffer"); 326 return CPA_STATUS_INVALID_PARAM; 327 } 328 } else if (CPA_TRUE == pSessionDesc->digestIsAppended) { 329 if (CPA_CY_SYM_HASH_AES_GMAC == 330 pSessionDesc->hashAlgorithm) { 331 if ((pRequest->hashStartSrcOffsetInBytes + 332 pRequest->messageLenToHashInBytes + 333 pSessionDesc->hashResultSize) > 334 pRequest->dstBufferLen) { 335 LAC_INVALID_PARAM_LOG( 336 "Append Digest - Not enough room for" 337 " digest in destination buffer for " 338 "AES GMAC algorithm"); 339 return CPA_STATUS_INVALID_PARAM; 340 } 341 } 342 if (CPA_CY_SYM_HASH_AES_GCM == 343 pSessionDesc->hashAlgorithm) { 344 if ((pRequest->cryptoStartSrcOffsetInBytes + 345 pRequest->messageLenToCipherInBytes + 346 pSessionDesc->hashResultSize) > 347 pRequest->dstBufferLen) { 348 LAC_INVALID_PARAM_LOG( 349 "Append Digest - Not enough room " 350 "for digest in destination buffer" 351 " for GCM algorithm"); 352 return CPA_STATUS_INVALID_PARAM; 353 } 354 } 355 356 if ((pRequest->hashStartSrcOffsetInBytes + 357 pRequest->messageLenToHashInBytes + 358 pSessionDesc->hashResultSize) > 359 pRequest->dstBufferLen) { 360 LAC_INVALID_PARAM_LOG( 361 "Append Digest - Not enough room for" 362 " digest in destination buffer"); 363 return CPA_STATUS_INVALID_PARAM; 364 } 365 } 366 if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 367 if (pRequest->messageLenToHashInBytes == 0 || 368 pRequest->pAdditionalAuthData != NULL) { 369 LAC_INVALID_PARAM_LOG( 370 "For AES_GMAC, AAD Length " 371 "(messageLenToHashInBytes) must be " 372 "non zero and pAdditionalAuthData " 373 "must be NULL"); 374 return CPA_STATUS_INVALID_PARAM; 375 } 376 } 377 } 378 379 if (CPA_DP_BUFLIST != pRequest->srcBufferLen) { 380 if ((CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) && 381 ((pRequest->messageLenToCipherInBytes + 382 pRequest->cryptoStartSrcOffsetInBytes) > 383 pRequest->srcBufferLen)) { 384 LAC_INVALID_PARAM_LOG( 385 "cipher len + offset greater than " 386 "srcBufferLen"); 387 return CPA_STATUS_INVALID_PARAM; 388 } else if ((CPA_CY_SYM_OP_CIPHER != 389 pSessionDesc->symOperation) && 390 (CPA_CY_SYM_HASH_AES_CCM != 391 pSessionDesc->hashAlgorithm) && 392 (CPA_CY_SYM_HASH_AES_GCM != 393 pSessionDesc->hashAlgorithm) && 394 (CPA_CY_SYM_HASH_AES_GMAC != 395 pSessionDesc->hashAlgorithm) && 396 ((pRequest->messageLenToHashInBytes + 397 pRequest->hashStartSrcOffsetInBytes) > 398 pRequest->srcBufferLen)) { 399 LAC_INVALID_PARAM_LOG( 400 "hash len + offset greater than srcBufferLen"); 401 return CPA_STATUS_INVALID_PARAM; 402 } 403 } else { 404 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->srcBuffer); 405 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->dstBuffer); 406 } 407 408 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->thisPhys); 409 410 return CPA_STATUS_SUCCESS; 411 } 412 413 /** 414 ***************************************************************************** 415 * @ingroup cpaCySymDp 416 * Write Message on the ring and write request params 417 * This is the optimized version, which should not be used for 418 * algorithm of CCM, GCM, CHACHA and RC4 419 * 420 * @description 421 * Write Message on the ring and write request params 422 * 423 * @param[in/out] pRequest Pointer to operation data for crypto 424 * data plane API 425 * @param[in/out] pCurrentQatMsg Pointer to ring memory where msg will 426 * be written 427 * 428 * @retval none 429 * 430 *****************************************************************************/ 431 432 void 433 LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, 434 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg) 435 { 436 lac_session_desc_t *pSessionDesc = 437 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 438 Cpa8U *pMsgDummy = NULL; 439 Cpa8U *pCacheDummyHdr = NULL; 440 Cpa8U *pCacheDummyFtr = NULL; 441 442 pMsgDummy = (Cpa8U *)pCurrentQatMsg; 443 /* Write Request */ 444 /* 445 * Fill in the header and footer bytes of the ET ring message - cached 446 * from the session descriptor. 447 */ 448 if (!pSessionDesc->useSymConstantsTable) { 449 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); 450 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); 451 } else { 452 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); 453 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); 454 } 455 memcpy(pMsgDummy, 456 pCacheDummyHdr, 457 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); 458 memset((pMsgDummy + 459 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)), 460 0, 461 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); 462 memcpy(pMsgDummy + 463 (LAC_LONG_WORD_IN_BYTES * LAC_START_OF_CACHE_FTR_IN_LW), 464 pCacheDummyFtr, 465 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); 466 467 SalQatMsg_CmnMidWrite(pCurrentQatMsg, 468 pRequest, 469 (CPA_DP_BUFLIST == pRequest->srcBufferLen ? 470 QAT_COMN_PTR_TYPE_SGL : 471 QAT_COMN_PTR_TYPE_FLAT), 472 pRequest->srcBuffer, 473 pRequest->dstBuffer, 474 pRequest->srcBufferLen, 475 pRequest->dstBufferLen); 476 477 /* Write Request Params */ 478 if (pSessionDesc->isCipher) { 479 480 LacSymQat_CipherRequestParamsPopulate( 481 pSessionDesc, 482 pCurrentQatMsg, 483 pRequest->cryptoStartSrcOffsetInBytes, 484 pRequest->messageLenToCipherInBytes, 485 pRequest->iv, 486 pRequest->pIv); 487 } 488 489 if (pSessionDesc->isAuth) { 490 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo = 491 &(pSessionDesc->hashStateBufferInfo); 492 icp_qat_fw_la_auth_req_params_t *pAuthReqPars = 493 (icp_qat_fw_la_auth_req_params_t 494 *)((Cpa8U *)&(pCurrentQatMsg->serv_specif_rqpars) + 495 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 496 497 if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 != 498 pSessionDesc->hashAlgorithm && 499 CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm && 500 CPA_CY_SYM_HASH_AES_GCM != pSessionDesc->hashAlgorithm && 501 CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm && 502 CPA_CY_SYM_HASH_ZUC_EIA3 != pSessionDesc->hashAlgorithm) && 503 (pHashStateBufferInfo->prefixAadSzQuadWords > 0)) { 504 /* prefixAadSzQuadWords > 0 when there is prefix data 505 - i.e. nested hash or HMAC no precompute cases 506 Note partials not supported on DP api so we do not need 507 dynamic hash state in this case */ 508 pRequest->additionalAuthData = 509 pHashStateBufferInfo->pDataPhys + 510 LAC_QUADWORDS_TO_BYTES( 511 pHashStateBufferInfo->stateStorageSzQuadWords); 512 } 513 514 /* The first 24 bytes in icp_qat_fw_la_auth_req_params_t can be 515 * copied directly from the op request data because they share a 516 * corresponding layout. The remaining 4 bytes are taken 517 * from the session message template and use values 518 * preconfigured at sessionInit (updated per request for some 519 * specific cases below) 520 */ 521 522 /* We force a specific compiler optimisation here. The length 523 * to be copied turns out to be always 16, and by coding a 524 * memcpy with a literal value the compiler will compile inline 525 * code (in fact, only two vector instructions) to effect the 526 * copy. This gives us a huge performance increase. 527 */ 528 unsigned long cplen = 529 (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) - 530 (unsigned long)pAuthReqPars; 531 if (cplen == 16) 532 memcpy(pAuthReqPars, 533 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 534 16); 535 else 536 memcpy(pAuthReqPars, 537 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 538 cplen); 539 540 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { 541 pAuthReqPars->hash_state_sz = 542 LAC_BYTES_TO_QUADWORDS(pAuthReqPars->u2.aad_sz); 543 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 544 pSessionDesc->hashAlgorithm || 545 CPA_CY_SYM_HASH_ZUC_EIA3 == 546 pSessionDesc->hashAlgorithm) { 547 pAuthReqPars->hash_state_sz = 548 LAC_BYTES_TO_QUADWORDS(pSessionDesc->aadLenInBytes); 549 } 550 } 551 552 } 553 554 /** 555 ***************************************************************************** 556 * @ingroup cpaCySymDp 557 * Write Message on the ring and write request params 558 * 559 * @description 560 * Write Message on the ring and write request params 561 * 562 * @param[in/out] pRequest Pointer to operation data for crypto 563 * data plane API 564 * @param[in/out] pCurrentQatMsg Pointer to ring memory where msg will 565 * be written 566 * 567 * @retval none 568 * 569 *****************************************************************************/ 570 571 void 572 LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, 573 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg) 574 { 575 lac_session_desc_t *pSessionDesc = 576 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 577 Cpa8U *pMsgDummy = NULL; 578 Cpa8U *pCacheDummyHdr = NULL; 579 Cpa8U *pCacheDummyFtr = NULL; 580 sal_qat_content_desc_info_t *pCdInfo = NULL; 581 Cpa8U *pHwBlockBaseInDRAM = NULL; 582 Cpa32U hwBlockOffsetInDRAM = 0; 583 Cpa32U sizeInBytes = 0; 584 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; 585 CpaCySymHashAlgorithm hash = pSessionDesc->hashAlgorithm; 586 sal_crypto_service_t *pService = 587 (sal_crypto_service_t *)pRequest->instanceHandle; 588 Cpa32U capabilitiesMask = 589 ((sal_crypto_service_t *)pRequest->instanceHandle) 590 ->generic_service_info.capabilitiesMask; 591 592 CpaBoolean isSpCcm = LAC_CIPHER_IS_CCM(cipher) && 593 LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask); 594 595 Cpa8U paddingLen = 0; 596 Cpa8U blockLen = 0; 597 Cpa32U aadDataLen = 0; 598 599 pMsgDummy = (Cpa8U *)pCurrentQatMsg; 600 /* Write Request */ 601 /* 602 * Fill in the header and footer bytes of the ET ring message - cached 603 * from the session descriptor. 604 */ 605 606 if ((NON_SPC != pSessionDesc->singlePassState) && 607 (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes))) { 608 pSessionDesc->singlePassState = SPC; 609 pSessionDesc->isCipher = CPA_TRUE; 610 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 611 pSessionDesc->isAuth = CPA_FALSE; 612 pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER; 613 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; 614 if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 615 pSessionDesc->aadLenInBytes = 616 pRequest->messageLenToHashInBytes; 617 } 618 /* New bit position (13) for SINGLE PASS. 619 * The FW provides a specific macro to use to set the proto flag 620 */ 621 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 622 pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO); 623 if (isCyGen2x(pService)) 624 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); 625 626 pCdInfo = &(pSessionDesc->contentDescInfo); 627 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 628 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == 629 pSessionDesc->cipherDirection) { 630 if (LAC_CIPHER_IS_GCM(cipher)) 631 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 632 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); 633 else if (LAC_CIPHER_IS_CHACHA(cipher)) 634 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 635 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); 636 } else if (isSpCcm) { 637 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 638 LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM); 639 } 640 641 /* Update slice type, as used algos changed */ 642 pSessionDesc->cipherSliceType = 643 LacCipher_GetCipherSliceType(pService, cipher, hash); 644 645 ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags, 646 pSessionDesc->cipherSliceType); 647 648 /* construct cipherConfig in CD in DRAM */ 649 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 650 pHwBlockBaseInDRAM + 651 hwBlockOffsetInDRAM, 652 &sizeInBytes); 653 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 654 pSessionDesc->reqSpcCacheHdr), 655 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 656 pSessionDesc->laCmdId, 657 pSessionDesc->cmnRequestFlags, 658 pSessionDesc->laCmdFlags); 659 } else if ((SPC == pSessionDesc->singlePassState) && 660 (LAC_CIPHER_SPC_IV_SIZE != pRequest->ivLenInBytes)) { 661 pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; 662 pSessionDesc->singlePassState = LIKELY_SPC; 663 pSessionDesc->isCipher = CPA_TRUE; 664 pSessionDesc->isAuthEncryptOp = CPA_TRUE; 665 pSessionDesc->isAuth = CPA_TRUE; 666 pCdInfo = &(pSessionDesc->contentDescInfo); 667 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 668 669 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 670 pSessionDesc->cipherDirection) { 671 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; 672 } else { 673 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; 674 } 675 676 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 677 pSessionDesc->laCmdFlags, 0); 678 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 679 ICP_QAT_FW_LA_GCM_PROTO); 680 681 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 682 pHwBlockBaseInDRAM + 683 hwBlockOffsetInDRAM, 684 &sizeInBytes); 685 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 686 pSessionDesc->reqCacheHdr), 687 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 688 pSessionDesc->laCmdId, 689 pSessionDesc->cmnRequestFlags, 690 pSessionDesc->laCmdFlags); 691 } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 692 pSessionDesc->aadLenInBytes = pRequest->messageLenToHashInBytes; 693 } 694 if (SPC == pSessionDesc->singlePassState) { 695 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr); 696 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr); 697 } else { 698 if (!pSessionDesc->useSymConstantsTable) { 699 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); 700 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); 701 } else { 702 pCacheDummyHdr = 703 (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); 704 pCacheDummyFtr = 705 (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); 706 } 707 } 708 memcpy(pMsgDummy, 709 pCacheDummyHdr, 710 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); 711 memset((pMsgDummy + 712 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)), 713 0, 714 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); 715 memcpy(pMsgDummy + 716 (LAC_LONG_WORD_IN_BYTES * LAC_START_OF_CACHE_FTR_IN_LW), 717 pCacheDummyFtr, 718 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); 719 720 SalQatMsg_CmnMidWrite(pCurrentQatMsg, 721 pRequest, 722 (CPA_DP_BUFLIST == pRequest->srcBufferLen ? 723 QAT_COMN_PTR_TYPE_SGL : 724 QAT_COMN_PTR_TYPE_FLAT), 725 pRequest->srcBuffer, 726 pRequest->dstBuffer, 727 pRequest->srcBufferLen, 728 pRequest->dstBufferLen); 729 730 if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm && 731 pSessionDesc->isAuth == CPA_TRUE) || 732 isSpCcm) { 733 /* prepare IV and AAD for CCM */ 734 LacSymAlgChain_PrepareCCMData( 735 pSessionDesc, 736 pRequest->pAdditionalAuthData, 737 pRequest->pIv, 738 pRequest->messageLenToCipherInBytes, 739 pRequest->ivLenInBytes); 740 741 /* According to the API, for CCM and GCM, 742 * messageLenToHashInBytes and hashStartSrcOffsetInBytes are not 743 * initialized by the user and must be set by the driver 744 */ 745 pRequest->hashStartSrcOffsetInBytes = 746 pRequest->cryptoStartSrcOffsetInBytes; 747 pRequest->messageLenToHashInBytes = 748 pRequest->messageLenToCipherInBytes; 749 } else if ((SPC != pSessionDesc->singlePassState) && 750 (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm || 751 CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm)) { 752 /* GCM case */ 753 if (CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm) { 754 /* According to the API, for CCM and GCM, 755 * messageLenToHashInBytes and hashStartSrcOffsetInBytes 756 * are not initialized by the user and must be set 757 * by the driver 758 */ 759 pRequest->hashStartSrcOffsetInBytes = 760 pRequest->cryptoStartSrcOffsetInBytes; 761 pRequest->messageLenToHashInBytes = 762 pRequest->messageLenToCipherInBytes; 763 764 LacSymAlgChain_PrepareGCMData( 765 pSessionDesc, pRequest->pAdditionalAuthData); 766 } 767 768 if (LAC_CIPHER_IV_SIZE_GCM_12 == pRequest->ivLenInBytes) { 769 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( 770 pCurrentQatMsg->comn_hdr.serv_specif_flags, 771 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); 772 } 773 } 774 775 /* Write Request Params */ 776 if (pSessionDesc->isCipher) { 777 if (CPA_CY_SYM_CIPHER_ARC4 == pSessionDesc->cipherAlgorithm) { 778 /* ARC4 does not have an IV but the field is used to 779 * store the initial state */ 780 pRequest->iv = 781 pSessionDesc->cipherARC4InitialStatePhysAddr; 782 } 783 784 ICP_QAT_FW_LA_SLICE_TYPE_SET( 785 pCurrentQatMsg->comn_hdr.serv_specif_flags, 786 pSessionDesc->cipherSliceType); 787 788 LacSymQat_CipherRequestParamsPopulate( 789 pSessionDesc, 790 pCurrentQatMsg, 791 pRequest->cryptoStartSrcOffsetInBytes, 792 pRequest->messageLenToCipherInBytes, 793 pRequest->iv, 794 pRequest->pIv); 795 if (SPC == pSessionDesc->singlePassState) { 796 icp_qat_fw_la_cipher_req_params_t *pCipherReqParams = 797 (icp_qat_fw_la_cipher_req_params_t 798 *)((Cpa8U *)&( 799 pCurrentQatMsg->serv_specif_rqpars) + 800 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 801 802 icp_qat_fw_la_cipher_20_req_params_t *pCipher20ReqParams = 803 (void 804 *)((Cpa8U *)&( 805 pCurrentQatMsg->serv_specif_rqpars) + 806 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 807 808 if (isCyGen4x(pService)) { 809 pCipher20ReqParams->spc_aad_addr = 810 (uint64_t)pRequest->additionalAuthData; 811 pCipher20ReqParams->spc_aad_sz = 812 pSessionDesc->aadLenInBytes; 813 pCipher20ReqParams->spc_aad_offset = 0; 814 815 if (isSpCcm) 816 pCipher20ReqParams->spc_aad_sz += 817 LAC_CIPHER_CCM_AAD_OFFSET; 818 819 pCipher20ReqParams->spc_auth_res_addr = 820 (uint64_t)pRequest->digestResult; 821 pCipher20ReqParams->spc_auth_res_sz = 822 (Cpa8U)pSessionDesc->hashResultSize; 823 } else { 824 pCipherReqParams->spc_aad_addr = 825 (uint64_t)pRequest->additionalAuthData; 826 pCipherReqParams->spc_aad_sz = 827 (Cpa16U)pSessionDesc->aadLenInBytes; 828 829 pCipherReqParams->spc_auth_res_addr = 830 (uint64_t)pRequest->digestResult; 831 pCipherReqParams->spc_auth_res_sz = 832 (Cpa8U)pSessionDesc->hashResultSize; 833 } 834 835 /* For CHACHA and AES_GCM single pass AAD buffer needs 836 * alignment if aadLenInBytes is nonzero. In case of 837 * AES-GMAC, AAD buffer passed in the src buffer. 838 */ 839 if (0 != pSessionDesc->aadLenInBytes && 840 CPA_CY_SYM_HASH_AES_GMAC != 841 pSessionDesc->hashAlgorithm) { 842 blockLen = LacSymQat_CipherBlockSizeBytesGet( 843 pSessionDesc->cipherAlgorithm); 844 aadDataLen = pSessionDesc->aadLenInBytes; 845 846 /* In case of AES_CCM, B0 block size and 2 bytes 847 * of AAD len 848 * encoding need to be added to total AAD data 849 * len */ 850 if (isSpCcm) 851 aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET; 852 853 if ((aadDataLen % blockLen) != 0) { 854 paddingLen = 855 blockLen - (aadDataLen % blockLen); 856 memset(&pRequest->pAdditionalAuthData 857 [aadDataLen], 858 0, 859 paddingLen); 860 } 861 } 862 } 863 } 864 865 if (pSessionDesc->isAuth) { 866 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo = 867 &(pSessionDesc->hashStateBufferInfo); 868 icp_qat_fw_la_auth_req_params_t *pAuthReqPars = 869 (icp_qat_fw_la_auth_req_params_t 870 *)((Cpa8U *)&(pCurrentQatMsg->serv_specif_rqpars) + 871 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 872 873 if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 != 874 pSessionDesc->hashAlgorithm && 875 CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm && 876 CPA_CY_SYM_HASH_AES_GCM != pSessionDesc->hashAlgorithm && 877 CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm && 878 CPA_CY_SYM_HASH_ZUC_EIA3 != pSessionDesc->hashAlgorithm) && 879 (pHashStateBufferInfo->prefixAadSzQuadWords > 0)) { 880 /* prefixAadSzQuadWords > 0 when there is prefix data 881 - i.e. nested hash or HMAC no precompute cases 882 Note partials not supported on DP api so we do not need 883 dynamic hash state in this case */ 884 pRequest->additionalAuthData = 885 pHashStateBufferInfo->pDataPhys + 886 LAC_QUADWORDS_TO_BYTES( 887 pHashStateBufferInfo->stateStorageSzQuadWords); 888 } 889 890 /* The first 24 bytes in icp_qat_fw_la_auth_req_params_t can be 891 * copied directly from the op request data because they share a 892 * corresponding layout. The remaining 4 bytes are taken 893 * from the session message template and use values 894 * preconfigured at sessionInit (updated per request for some 895 * specific cases below) 896 */ 897 memcpy(pAuthReqPars, 898 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 899 ((uintptr_t) & 900 (pAuthReqPars->u2.inner_prefix_sz) - 901 (uintptr_t)pAuthReqPars)); 902 903 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { 904 pAuthReqPars->hash_state_sz = 905 LAC_BYTES_TO_QUADWORDS(pAuthReqPars->u2.aad_sz); 906 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 907 pSessionDesc->hashAlgorithm || 908 CPA_CY_SYM_HASH_ZUC_EIA3 == 909 pSessionDesc->hashAlgorithm) { 910 pAuthReqPars->hash_state_sz = 911 LAC_BYTES_TO_QUADWORDS(pSessionDesc->aadLenInBytes); 912 } 913 } 914 915 } 916 917 CpaStatus 918 cpaCySymDpSessionCtxGetSize(const CpaInstanceHandle instanceHandle, 919 const CpaCySymSessionSetupData *pSessionSetupData, 920 Cpa32U *pSessionCtxSizeInBytes) 921 { 922 CpaStatus status = CPA_STATUS_SUCCESS; 923 924 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 925 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 926 /* All other param checks are common with trad api */ 927 /* Check for valid pointers */ 928 LAC_CHECK_NULL_PARAM(pSessionCtxSizeInBytes); 929 status = cpaCySymSessionCtxGetSize(instanceHandle, 930 pSessionSetupData, 931 pSessionCtxSizeInBytes); 932 933 return status; 934 } 935 936 CpaStatus 937 cpaCySymDpSessionCtxGetDynamicSize( 938 const CpaInstanceHandle instanceHandle, 939 const CpaCySymSessionSetupData *pSessionSetupData, 940 Cpa32U *pSessionCtxSizeInBytes) 941 { 942 CpaStatus status = CPA_STATUS_SUCCESS; 943 944 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 945 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 946 /* All other param checks are common with trad api */ 947 /* Check for valid pointers */ 948 LAC_CHECK_NULL_PARAM(pSessionCtxSizeInBytes); 949 status = cpaCySymSessionCtxGetDynamicSize(instanceHandle, 950 pSessionSetupData, 951 pSessionCtxSizeInBytes); 952 953 return status; 954 } 955 956 /** @ingroup cpaCySymDp */ 957 CpaStatus 958 cpaCySymDpInitSession(CpaInstanceHandle instanceHandle, 959 const CpaCySymSessionSetupData *pSessionSetupData, 960 CpaCySymDpSessionCtx sessionCtx) 961 { 962 CpaStatus status = CPA_STATUS_FAIL; 963 sal_service_t *pService = NULL; 964 965 966 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 967 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 968 (SAL_SERVICE_TYPE_CRYPTO | 969 SAL_SERVICE_TYPE_CRYPTO_SYM)); 970 LAC_CHECK_NULL_PARAM(pSessionSetupData); 971 pService = (sal_service_t *)instanceHandle; 972 973 /* Check crypto service is running otherwise return an error */ 974 SAL_RUNNING_CHECK(pService); 975 976 status = LacSym_InitSession(instanceHandle, 977 NULL, /* Callback */ 978 pSessionSetupData, 979 CPA_TRUE, /* isDPSession */ 980 sessionCtx); 981 return status; 982 } 983 984 CpaStatus 985 cpaCySymDpRemoveSession(const CpaInstanceHandle instanceHandle, 986 CpaCySymDpSessionCtx sessionCtx) 987 { 988 989 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 990 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 991 /* All other param checks are common with trad api */ 992 993 return cpaCySymRemoveSession(instanceHandle, sessionCtx); 994 } 995 996 CpaStatus 997 cpaCySymDpRegCbFunc(const CpaInstanceHandle instanceHandle, 998 const CpaCySymDpCbFunc pSymDpCb) 999 { 1000 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; 1001 1002 1003 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1004 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1005 (SAL_SERVICE_TYPE_CRYPTO | 1006 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1007 LAC_CHECK_NULL_PARAM(pSymDpCb); 1008 SAL_RUNNING_CHECK(instanceHandle); 1009 pService->pSymDpCb = pSymDpCb; 1010 1011 return CPA_STATUS_SUCCESS; 1012 } 1013 1014 CpaStatus 1015 cpaCySymDpEnqueueOp(CpaCySymDpOpData *pRequest, const CpaBoolean performOpNow) 1016 { 1017 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg = NULL; 1018 icp_comms_trans_handle trans_handle = NULL; 1019 lac_session_desc_t *pSessionDesc = NULL; 1020 write_ringMsgFunc_t callFunc; 1021 1022 CpaStatus status = CPA_STATUS_SUCCESS; 1023 1024 1025 LAC_CHECK_NULL_PARAM(pRequest); 1026 status = LacDp_EnqueueParamCheck(pRequest); 1027 if (CPA_STATUS_SUCCESS != status) { 1028 return status; 1029 } 1030 1031 /* Check if SAL is running in crypto data plane otherwise return an 1032 * error */ 1033 SAL_RUNNING_CHECK(pRequest->instanceHandle); 1034 1035 trans_handle = ((sal_crypto_service_t *)pRequest->instanceHandle) 1036 ->trans_handle_sym_tx; 1037 1038 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 1039 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 1040 if (NULL == pCurrentQatMsg) { 1041 /* 1042 * No space is available on the queue. 1043 */ 1044 return CPA_STATUS_RETRY; 1045 } 1046 1047 callFunc = (write_ringMsgFunc_t)pSessionDesc->writeRingMsgFunc; 1048 1049 LAC_CHECK_NULL_PARAM(callFunc); 1050 1051 callFunc(pRequest, pCurrentQatMsg); 1052 1053 qatUtilsAtomicInc(&(pSessionDesc->u.pendingDpCbCount)); 1054 1055 if (CPA_TRUE == performOpNow) { 1056 SalQatMsg_updateQueueTail(trans_handle); 1057 } 1058 1059 return CPA_STATUS_SUCCESS; 1060 } 1061 1062 CpaStatus 1063 cpaCySymDpPerformOpNow(const CpaInstanceHandle instanceHandle) 1064 { 1065 icp_comms_trans_handle trans_handle = NULL; 1066 1067 1068 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1069 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1070 (SAL_SERVICE_TYPE_CRYPTO | 1071 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1072 1073 /* Check if SAL is initialised otherwise return an error */ 1074 SAL_RUNNING_CHECK(instanceHandle); 1075 1076 trans_handle = 1077 ((sal_crypto_service_t *)instanceHandle)->trans_handle_sym_tx; 1078 1079 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 1080 SalQatMsg_updateQueueTail(trans_handle); 1081 } 1082 1083 return CPA_STATUS_SUCCESS; 1084 } 1085 1086 CpaStatus 1087 cpaCySymDpEnqueueOpBatch(const Cpa32U numberRequests, 1088 CpaCySymDpOpData *pRequests[], 1089 const CpaBoolean performOpNow) 1090 { 1091 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg = NULL; 1092 icp_comms_trans_handle trans_handle = NULL; 1093 lac_session_desc_t *pSessionDesc = NULL; 1094 write_ringMsgFunc_t callFunc; 1095 Cpa32U i = 0; 1096 1097 CpaStatus status = CPA_STATUS_SUCCESS; 1098 sal_crypto_service_t *pService = NULL; 1099 1100 1101 LAC_CHECK_NULL_PARAM(pRequests); 1102 LAC_CHECK_NULL_PARAM(pRequests[0]); 1103 LAC_CHECK_NULL_PARAM(pRequests[0]->instanceHandle); 1104 1105 pService = (sal_crypto_service_t *)(pRequests[0]->instanceHandle); 1106 1107 if ((0 == numberRequests) || 1108 (numberRequests > pService->maxNumSymReqBatch)) { 1109 LAC_INVALID_PARAM_LOG1( 1110 "The number of requests needs to be between 1 " 1111 "and %d", 1112 pService->maxNumSymReqBatch); 1113 return CPA_STATUS_INVALID_PARAM; 1114 } 1115 1116 for (i = 0; i < numberRequests; i++) { 1117 status = LacDp_EnqueueParamCheck(pRequests[i]); 1118 if (CPA_STATUS_SUCCESS != status) { 1119 return status; 1120 } 1121 1122 /* Check that all instance handles are the same */ 1123 if (pRequests[i]->instanceHandle != 1124 pRequests[0]->instanceHandle) { 1125 LAC_INVALID_PARAM_LOG( 1126 "All instance handles should be the same " 1127 "in the requests"); 1128 return CPA_STATUS_INVALID_PARAM; 1129 } 1130 } 1131 1132 /* Check if SAL is running in crypto data plane otherwise return an 1133 * error */ 1134 SAL_RUNNING_CHECK(pRequests[0]->instanceHandle); 1135 1136 trans_handle = ((sal_crypto_service_t *)pRequests[0]->instanceHandle) 1137 ->trans_handle_sym_tx; 1138 pSessionDesc = 1139 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequests[0]->sessionCtx); 1140 icp_adf_getQueueMemory(trans_handle, 1141 numberRequests, 1142 (void **)&pCurrentQatMsg); 1143 if (NULL == pCurrentQatMsg) { 1144 /* 1145 * No space is available on the queue. 1146 */ 1147 return CPA_STATUS_RETRY; 1148 } 1149 1150 for (i = 0; i < numberRequests; i++) { 1151 pSessionDesc = 1152 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequests[i]->sessionCtx); 1153 callFunc = (write_ringMsgFunc_t)pSessionDesc->writeRingMsgFunc; 1154 callFunc(pRequests[i], pCurrentQatMsg); 1155 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 1156 qatUtilsAtomicAdd(1, &(pSessionDesc->u.pendingDpCbCount)); 1157 } 1158 1159 if (CPA_TRUE == performOpNow) { 1160 SalQatMsg_updateQueueTail(trans_handle); 1161 } 1162 1163 return CPA_STATUS_SUCCESS; 1164 } 1165 1166 CpaStatus 1167 icp_sal_CyPollDpInstance(const CpaInstanceHandle instanceHandle, 1168 const Cpa32U responseQuota) 1169 { 1170 icp_comms_trans_handle trans_handle = NULL; 1171 1172 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1173 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1174 (SAL_SERVICE_TYPE_CRYPTO | 1175 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1176 1177 /* Check if SAL is initialised otherwise return an error */ 1178 SAL_RUNNING_CHECK(instanceHandle); 1179 1180 trans_handle = 1181 ((sal_crypto_service_t *)instanceHandle)->trans_handle_sym_rx; 1182 1183 return icp_adf_pollQueue(trans_handle, responseQuota); 1184 } 1185