1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 5 /** 6 *************************************************************************** 7 * @file lac_sym_dp.c 8 * Implementation of the symmetric data plane API 9 * 10 * @ingroup cpaCySymDp 11 ***************************************************************************/ 12 13 /* 14 ******************************************************************************* 15 * Include public/global header files 16 ******************************************************************************* 17 */ 18 19 #include "cpa.h" 20 #include "cpa_cy_sym.h" 21 #include "cpa_cy_sym_dp.h" 22 23 /* 24 ******************************************************************************* 25 * Include private header files 26 ******************************************************************************* 27 */ 28 29 #include "icp_accel_devices.h" 30 #include "icp_adf_init.h" 31 #include "icp_adf_transport.h" 32 #include "icp_adf_transport_dp.h" 33 #include "icp_adf_debug.h" 34 #include "icp_sal_poll.h" 35 36 #include "qat_utils.h" 37 38 #include "lac_list.h" 39 #include "lac_log.h" 40 #include "lac_mem.h" 41 #include "lac_sal_types_crypto.h" 42 #include "lac_sym.h" 43 #include "lac_sym_cipher.h" 44 #include "lac_sym_auth_enc.h" 45 #include "lac_sym_qat_cipher.h" 46 #include "sal_service_state.h" 47 #include "sal_hw_gen.h" 48 49 typedef void (*write_ringMsgFunc_t)(CpaCySymDpOpData *pRequest, 50 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg); 51 52 /** 53 ***************************************************************************** 54 * @ingroup cpaCySymDp 55 * Check that the operation data is valid 56 * 57 * @description 58 * Check that all the parameters defined in the operation data are valid 59 * 60 * @param[in] pRequest Pointer to an operation data for crypto 61 * data plane API 62 * 63 * @retval CPA_STATUS_SUCCESS Function executed successfully 64 * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in 65 * 66 *****************************************************************************/ 67 static CpaStatus 68 LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) 69 { 70 lac_session_desc_t *pSessionDesc = NULL; 71 CpaCySymCipherAlgorithm cipher = 0; 72 CpaCySymHashAlgorithm hash = 0; 73 Cpa32U capabilitiesMask = 0; 74 75 LAC_CHECK_NULL_PARAM(pRequest); 76 LAC_CHECK_NULL_PARAM(pRequest->instanceHandle); 77 LAC_CHECK_NULL_PARAM(pRequest->sessionCtx); 78 79 /* Ensure this is a crypto instance */ 80 SAL_CHECK_INSTANCE_TYPE(pRequest->instanceHandle, 81 (SAL_SERVICE_TYPE_CRYPTO | 82 SAL_SERVICE_TYPE_CRYPTO_SYM)); 83 84 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 85 if (NULL == pSessionDesc) { 86 do { 87 qatUtilsSleep(500); 88 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET( 89 pRequest->sessionCtx); 90 } while (NULL == pSessionDesc); 91 } 92 if (NULL == pSessionDesc) { 93 LAC_INVALID_PARAM_LOG("Session context not as expected"); 94 return CPA_STATUS_INVALID_PARAM; 95 } 96 97 if (CPA_FALSE == pSessionDesc->isDPSession) { 98 LAC_INVALID_PARAM_LOG( 99 "Session not initialised for data plane API"); 100 return CPA_STATUS_INVALID_PARAM; 101 } 102 103 /*check whether Payload size is zero for CHACHA-POLY */ 104 if ((CPA_CY_SYM_CIPHER_CHACHA == pSessionDesc->cipherAlgorithm) && 105 (CPA_CY_SYM_HASH_POLY == pSessionDesc->hashAlgorithm) && 106 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation)) { 107 if (!pRequest->messageLenToCipherInBytes) { 108 LAC_INVALID_PARAM_LOG( 109 "Invalid messageLenToCipherInBytes for CHACHA-POLY"); 110 return CPA_STATUS_INVALID_PARAM; 111 } 112 } 113 114 if (0 == pRequest->srcBuffer) { 115 LAC_INVALID_PARAM_LOG("Invalid srcBuffer"); 116 return CPA_STATUS_INVALID_PARAM; 117 } 118 if (0 == pRequest->dstBuffer) { 119 LAC_INVALID_PARAM_LOG("Invalid destBuffer"); 120 return CPA_STATUS_INVALID_PARAM; 121 } 122 if (0 == pRequest->thisPhys) { 123 LAC_INVALID_PARAM_LOG("Invalid thisPhys"); 124 return CPA_STATUS_INVALID_PARAM; 125 } 126 127 /* Check that src buffer Len = dst buffer Len 128 Note this also checks that they are of the same type */ 129 if (pRequest->srcBufferLen != pRequest->dstBufferLen) { 130 LAC_INVALID_PARAM_LOG( 131 "Source and Destination buffer lengths need to be equal"); 132 return CPA_STATUS_INVALID_PARAM; 133 } 134 135 /* digestVerify and digestIsAppended on Hash-Only operation not 136 * supported */ 137 if (pSessionDesc->digestIsAppended && pSessionDesc->digestVerify && 138 (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH)) { 139 LAC_INVALID_PARAM_LOG( 140 "digestVerify and digestIsAppended set " 141 "on Hash-Only operation is not supported"); 142 return CPA_STATUS_INVALID_PARAM; 143 } 144 145 /* Cipher specific tests */ 146 if (CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) { 147 /* Perform IV check */ 148 switch (pSessionDesc->cipherAlgorithm) { 149 case CPA_CY_SYM_CIPHER_AES_CTR: 150 case CPA_CY_SYM_CIPHER_3DES_CTR: 151 case CPA_CY_SYM_CIPHER_SM4_CTR: 152 case CPA_CY_SYM_CIPHER_AES_GCM: 153 case CPA_CY_SYM_CIPHER_CHACHA: 154 case CPA_CY_SYM_CIPHER_AES_CBC: 155 case CPA_CY_SYM_CIPHER_DES_CBC: 156 case CPA_CY_SYM_CIPHER_3DES_CBC: 157 case CPA_CY_SYM_CIPHER_SM4_CBC: 158 case CPA_CY_SYM_CIPHER_AES_F8: { 159 Cpa32U ivLenInBytes = LacSymQat_CipherIvSizeBytesGet( 160 pSessionDesc->cipherAlgorithm); 161 if (pRequest->ivLenInBytes != ivLenInBytes) { 162 if (!(/* GCM with 12 byte IV is OK */ 163 (LAC_CIPHER_IS_GCM( 164 pSessionDesc->cipherAlgorithm) && 165 pRequest->ivLenInBytes == 166 LAC_CIPHER_IV_SIZE_GCM_12))) { 167 LAC_INVALID_PARAM_LOG( 168 "invalid cipher IV size"); 169 return CPA_STATUS_INVALID_PARAM; 170 } 171 } 172 if (0 == pRequest->iv) { 173 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 174 return CPA_STATUS_INVALID_PARAM; 175 } 176 /* pRequest->pIv is only used for CCM so is not checked 177 * here */ 178 } break; 179 case CPA_CY_SYM_CIPHER_KASUMI_F8: { 180 if (LAC_CIPHER_KASUMI_F8_IV_LENGTH != 181 pRequest->ivLenInBytes) { 182 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 183 return CPA_STATUS_INVALID_PARAM; 184 } 185 if (0 == pRequest->iv) { 186 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 187 return CPA_STATUS_INVALID_PARAM; 188 } 189 } break; 190 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: { 191 if (ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ != 192 pRequest->ivLenInBytes) { 193 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 194 return CPA_STATUS_INVALID_PARAM; 195 } 196 if (0 == pRequest->iv) { 197 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 198 return CPA_STATUS_INVALID_PARAM; 199 } 200 } break; 201 case CPA_CY_SYM_CIPHER_ZUC_EEA3: { 202 if (ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ != 203 pRequest->ivLenInBytes) { 204 LAC_INVALID_PARAM_LOG("invalid cipher IV size"); 205 return CPA_STATUS_INVALID_PARAM; 206 } 207 if (0 == pRequest->iv) { 208 LAC_INVALID_PARAM_LOG("invalid iv of 0"); 209 return CPA_STATUS_INVALID_PARAM; 210 } 211 } break; 212 case CPA_CY_SYM_CIPHER_AES_CCM: { 213 if (CPA_STATUS_SUCCESS != 214 LacSymAlgChain_CheckCCMData( 215 pRequest->pAdditionalAuthData, 216 pRequest->pIv, 217 pRequest->messageLenToCipherInBytes, 218 pRequest->ivLenInBytes)) { 219 return CPA_STATUS_INVALID_PARAM; 220 } 221 } break; 222 default: 223 break; 224 } 225 /* Perform algorithm-specific checks */ 226 switch (pSessionDesc->cipherAlgorithm) { 227 case CPA_CY_SYM_CIPHER_ARC4: 228 case CPA_CY_SYM_CIPHER_AES_CTR: 229 case CPA_CY_SYM_CIPHER_3DES_CTR: 230 case CPA_CY_SYM_CIPHER_SM4_CTR: 231 case CPA_CY_SYM_CIPHER_AES_CCM: 232 case CPA_CY_SYM_CIPHER_AES_GCM: 233 case CPA_CY_SYM_CIPHER_CHACHA: 234 case CPA_CY_SYM_CIPHER_KASUMI_F8: 235 case CPA_CY_SYM_CIPHER_AES_F8: 236 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: 237 case CPA_CY_SYM_CIPHER_ZUC_EEA3: 238 /* No action needed */ 239 break; 240 default: { 241 /* Mask & check below is based on assumption that block 242 * size is a power of 2. If data size is not a multiple 243 * of the block size, the "remainder" bits selected by 244 * the mask be non-zero 245 */ 246 if (pRequest->messageLenToCipherInBytes & 247 (LacSymQat_CipherBlockSizeBytesGet( 248 pSessionDesc->cipherAlgorithm) - 249 1)) { 250 LAC_INVALID_PARAM_LOG( 251 "Data size must be block size" 252 " multiple"); 253 return CPA_STATUS_INVALID_PARAM; 254 } 255 } 256 } 257 258 cipher = pSessionDesc->cipherAlgorithm; 259 hash = pSessionDesc->hashAlgorithm; 260 capabilitiesMask = 261 ((sal_crypto_service_t *)pRequest->instanceHandle) 262 ->generic_service_info.capabilitiesMask; 263 if (LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && 264 (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) { 265 /* For CHACHA and AES_GCM single pass there is an AAD 266 * buffer if aadLenInBytes is nonzero. AES_GMAC AAD is 267 * stored in source buffer, therefore there is no 268 * separate AAD buffer. */ 269 if ((0 != pSessionDesc->aadLenInBytes) && 270 (CPA_CY_SYM_HASH_AES_GMAC != 271 pSessionDesc->hashAlgorithm)) { 272 LAC_CHECK_NULL_PARAM( 273 pRequest->pAdditionalAuthData); 274 } 275 276 /* Ensure AAD length for AES_GMAC spc */ 277 if ((CPA_CY_SYM_HASH_AES_GMAC == hash) && 278 (ICP_QAT_FW_SPC_AAD_SZ_MAX < 279 pRequest->messageLenToHashInBytes)) { 280 LAC_INVALID_PARAM_LOG( 281 "aadLenInBytes for AES_GMAC"); 282 return CPA_STATUS_INVALID_PARAM; 283 } 284 } 285 } 286 287 /* Hash specific tests */ 288 if (CPA_CY_SYM_OP_CIPHER != pSessionDesc->symOperation) { 289 /* For CCM, snow3G and ZUC there is always an AAD buffer 290 For GCM there is an AAD buffer if aadLenInBytes is 291 nonzero */ 292 if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) || 293 (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm && 294 (0 != pSessionDesc->aadLenInBytes))) { 295 LAC_CHECK_NULL_PARAM(pRequest->pAdditionalAuthData); 296 if (0 == pRequest->additionalAuthData) { 297 LAC_INVALID_PARAM_LOG( 298 "Invalid additionalAuthData"); 299 return CPA_STATUS_INVALID_PARAM; 300 } 301 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 302 pSessionDesc->hashAlgorithm || 303 CPA_CY_SYM_HASH_ZUC_EIA3 == 304 pSessionDesc->hashAlgorithm) { 305 if (0 == pRequest->additionalAuthData) { 306 LAC_INVALID_PARAM_LOG( 307 "Invalid additionalAuthData"); 308 return CPA_STATUS_INVALID_PARAM; 309 } 310 } 311 312 if ((CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm) && 313 (!pSessionDesc->digestIsAppended) && 314 (0 == pRequest->digestResult)) { 315 LAC_INVALID_PARAM_LOG("Invalid digestResult"); 316 return CPA_STATUS_INVALID_PARAM; 317 } 318 319 if (CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm) { 320 if ((pRequest->cryptoStartSrcOffsetInBytes + 321 pRequest->messageLenToCipherInBytes + 322 pSessionDesc->hashResultSize) > 323 pRequest->dstBufferLen) { 324 LAC_INVALID_PARAM_LOG( 325 "CCM - Not enough room for" 326 " digest in destination buffer"); 327 return CPA_STATUS_INVALID_PARAM; 328 } 329 } else if (CPA_TRUE == pSessionDesc->digestIsAppended) { 330 if (CPA_CY_SYM_HASH_AES_GMAC == 331 pSessionDesc->hashAlgorithm) { 332 if ((pRequest->hashStartSrcOffsetInBytes + 333 pRequest->messageLenToHashInBytes + 334 pSessionDesc->hashResultSize) > 335 pRequest->dstBufferLen) { 336 LAC_INVALID_PARAM_LOG( 337 "Append Digest - Not enough room for" 338 " digest in destination buffer for " 339 "AES GMAC algorithm"); 340 return CPA_STATUS_INVALID_PARAM; 341 } 342 } 343 if (CPA_CY_SYM_HASH_AES_GCM == 344 pSessionDesc->hashAlgorithm) { 345 if ((pRequest->cryptoStartSrcOffsetInBytes + 346 pRequest->messageLenToCipherInBytes + 347 pSessionDesc->hashResultSize) > 348 pRequest->dstBufferLen) { 349 LAC_INVALID_PARAM_LOG( 350 "Append Digest - Not enough room " 351 "for digest in destination buffer" 352 " for GCM algorithm"); 353 return CPA_STATUS_INVALID_PARAM; 354 } 355 } 356 357 if ((pRequest->hashStartSrcOffsetInBytes + 358 pRequest->messageLenToHashInBytes + 359 pSessionDesc->hashResultSize) > 360 pRequest->dstBufferLen) { 361 LAC_INVALID_PARAM_LOG( 362 "Append Digest - Not enough room for" 363 " digest in destination buffer"); 364 return CPA_STATUS_INVALID_PARAM; 365 } 366 } 367 if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 368 if (pRequest->messageLenToHashInBytes == 0 || 369 pRequest->pAdditionalAuthData != NULL) { 370 LAC_INVALID_PARAM_LOG( 371 "For AES_GMAC, AAD Length " 372 "(messageLenToHashInBytes) must be " 373 "non zero and pAdditionalAuthData " 374 "must be NULL"); 375 return CPA_STATUS_INVALID_PARAM; 376 } 377 } 378 } 379 380 if (CPA_DP_BUFLIST != pRequest->srcBufferLen) { 381 if ((CPA_CY_SYM_OP_HASH != pSessionDesc->symOperation) && 382 ((pRequest->messageLenToCipherInBytes + 383 pRequest->cryptoStartSrcOffsetInBytes) > 384 pRequest->srcBufferLen)) { 385 LAC_INVALID_PARAM_LOG( 386 "cipher len + offset greater than " 387 "srcBufferLen"); 388 return CPA_STATUS_INVALID_PARAM; 389 } else if ((CPA_CY_SYM_OP_CIPHER != 390 pSessionDesc->symOperation) && 391 (CPA_CY_SYM_HASH_AES_CCM != 392 pSessionDesc->hashAlgorithm) && 393 (CPA_CY_SYM_HASH_AES_GCM != 394 pSessionDesc->hashAlgorithm) && 395 (CPA_CY_SYM_HASH_AES_GMAC != 396 pSessionDesc->hashAlgorithm) && 397 ((pRequest->messageLenToHashInBytes + 398 pRequest->hashStartSrcOffsetInBytes) > 399 pRequest->srcBufferLen)) { 400 LAC_INVALID_PARAM_LOG( 401 "hash len + offset greater than srcBufferLen"); 402 return CPA_STATUS_INVALID_PARAM; 403 } 404 } else { 405 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->srcBuffer); 406 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->dstBuffer); 407 } 408 409 LAC_CHECK_8_BYTE_ALIGNMENT(pRequest->thisPhys); 410 411 return CPA_STATUS_SUCCESS; 412 } 413 414 /** 415 ***************************************************************************** 416 * @ingroup cpaCySymDp 417 * Write Message on the ring and write request params 418 * This is the optimized version, which should not be used for 419 * algorithm of CCM, GCM, CHACHA and RC4 420 * 421 * @description 422 * Write Message on the ring and write request params 423 * 424 * @param[in/out] pRequest Pointer to operation data for crypto 425 * data plane API 426 * @param[in/out] pCurrentQatMsg Pointer to ring memory where msg will 427 * be written 428 * 429 * @retval none 430 * 431 *****************************************************************************/ 432 433 void 434 LacDp_WriteRingMsgOpt(CpaCySymDpOpData *pRequest, 435 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg) 436 { 437 lac_session_desc_t *pSessionDesc = 438 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 439 Cpa8U *pMsgDummy = NULL; 440 Cpa8U *pCacheDummyHdr = NULL; 441 Cpa8U *pCacheDummyFtr = NULL; 442 443 pMsgDummy = (Cpa8U *)pCurrentQatMsg; 444 /* Write Request */ 445 /* 446 * Fill in the header and footer bytes of the ET ring message - cached 447 * from the session descriptor. 448 */ 449 if (!pSessionDesc->useSymConstantsTable) { 450 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); 451 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); 452 } else { 453 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); 454 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); 455 } 456 memcpy(pMsgDummy, 457 pCacheDummyHdr, 458 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); 459 memset((pMsgDummy + 460 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)), 461 0, 462 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); 463 memcpy(pMsgDummy + 464 (LAC_LONG_WORD_IN_BYTES * LAC_START_OF_CACHE_FTR_IN_LW), 465 pCacheDummyFtr, 466 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); 467 468 SalQatMsg_CmnMidWrite(pCurrentQatMsg, 469 pRequest, 470 (CPA_DP_BUFLIST == pRequest->srcBufferLen ? 471 QAT_COMN_PTR_TYPE_SGL : 472 QAT_COMN_PTR_TYPE_FLAT), 473 pRequest->srcBuffer, 474 pRequest->dstBuffer, 475 pRequest->srcBufferLen, 476 pRequest->dstBufferLen); 477 478 /* Write Request Params */ 479 if (pSessionDesc->isCipher) { 480 481 LacSymQat_CipherRequestParamsPopulate( 482 pSessionDesc, 483 pCurrentQatMsg, 484 pRequest->cryptoStartSrcOffsetInBytes, 485 pRequest->messageLenToCipherInBytes, 486 pRequest->iv, 487 pRequest->pIv); 488 } 489 490 if (pSessionDesc->isAuth) { 491 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo = 492 &(pSessionDesc->hashStateBufferInfo); 493 icp_qat_fw_la_auth_req_params_t *pAuthReqPars = 494 (icp_qat_fw_la_auth_req_params_t 495 *)((Cpa8U *)&(pCurrentQatMsg->serv_specif_rqpars) + 496 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 497 498 if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 != 499 pSessionDesc->hashAlgorithm && 500 CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm && 501 CPA_CY_SYM_HASH_AES_GCM != pSessionDesc->hashAlgorithm && 502 CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm && 503 CPA_CY_SYM_HASH_ZUC_EIA3 != pSessionDesc->hashAlgorithm) && 504 (pHashStateBufferInfo->prefixAadSzQuadWords > 0)) { 505 /* prefixAadSzQuadWords > 0 when there is prefix data 506 - i.e. nested hash or HMAC no precompute cases 507 Note partials not supported on DP api so we do not need 508 dynamic hash state in this case */ 509 pRequest->additionalAuthData = 510 pHashStateBufferInfo->pDataPhys + 511 LAC_QUADWORDS_TO_BYTES( 512 pHashStateBufferInfo->stateStorageSzQuadWords); 513 } 514 515 /* The first 24 bytes in icp_qat_fw_la_auth_req_params_t can be 516 * copied directly from the op request data because they share a 517 * corresponding layout. The remaining 4 bytes are taken 518 * from the session message template and use values 519 * preconfigured at sessionInit (updated per request for some 520 * specific cases below) 521 */ 522 523 /* We force a specific compiler optimisation here. The length 524 * to be copied turns out to be always 16, and by coding a 525 * memcpy with a literal value the compiler will compile inline 526 * code (in fact, only two vector instructions) to effect the 527 * copy. This gives us a huge performance increase. 528 */ 529 unsigned long cplen = 530 (unsigned long)&(pAuthReqPars->u2.inner_prefix_sz) - 531 (unsigned long)pAuthReqPars; 532 if (cplen == 16) 533 memcpy(pAuthReqPars, 534 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 535 16); 536 else 537 memcpy(pAuthReqPars, 538 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 539 cplen); 540 541 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { 542 pAuthReqPars->hash_state_sz = 543 LAC_BYTES_TO_QUADWORDS(pAuthReqPars->u2.aad_sz); 544 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 545 pSessionDesc->hashAlgorithm || 546 CPA_CY_SYM_HASH_ZUC_EIA3 == 547 pSessionDesc->hashAlgorithm) { 548 pAuthReqPars->hash_state_sz = 549 LAC_BYTES_TO_QUADWORDS(pSessionDesc->aadLenInBytes); 550 } 551 } 552 553 } 554 555 /** 556 ***************************************************************************** 557 * @ingroup cpaCySymDp 558 * Write Message on the ring and write request params 559 * 560 * @description 561 * Write Message on the ring and write request params 562 * 563 * @param[in/out] pRequest Pointer to operation data for crypto 564 * data plane API 565 * @param[in/out] pCurrentQatMsg Pointer to ring memory where msg will 566 * be written 567 * 568 * @retval none 569 * 570 *****************************************************************************/ 571 572 void 573 LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, 574 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg) 575 { 576 lac_session_desc_t *pSessionDesc = 577 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 578 Cpa8U *pMsgDummy = NULL; 579 Cpa8U *pCacheDummyHdr = NULL; 580 Cpa8U *pCacheDummyFtr = NULL; 581 sal_qat_content_desc_info_t *pCdInfo = NULL; 582 Cpa8U *pHwBlockBaseInDRAM = NULL; 583 Cpa32U hwBlockOffsetInDRAM = 0; 584 Cpa32U sizeInBytes = 0; 585 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; 586 CpaCySymHashAlgorithm hash = pSessionDesc->hashAlgorithm; 587 sal_crypto_service_t *pService = 588 (sal_crypto_service_t *)pRequest->instanceHandle; 589 Cpa32U capabilitiesMask = 590 ((sal_crypto_service_t *)pRequest->instanceHandle) 591 ->generic_service_info.capabilitiesMask; 592 593 CpaBoolean isSpCcm = LAC_CIPHER_IS_CCM(cipher) && 594 LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask); 595 596 Cpa8U paddingLen = 0; 597 Cpa8U blockLen = 0; 598 Cpa32U aadDataLen = 0; 599 600 pMsgDummy = (Cpa8U *)pCurrentQatMsg; 601 /* Write Request */ 602 /* 603 * Fill in the header and footer bytes of the ET ring message - cached 604 * from the session descriptor. 605 */ 606 607 if ((NON_SPC != pSessionDesc->singlePassState) && 608 (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes))) { 609 pSessionDesc->singlePassState = SPC; 610 pSessionDesc->isCipher = CPA_TRUE; 611 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 612 pSessionDesc->isAuth = CPA_FALSE; 613 pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER; 614 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; 615 if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 616 pSessionDesc->aadLenInBytes = 617 pRequest->messageLenToHashInBytes; 618 } 619 /* New bit position (13) for SINGLE PASS. 620 * The FW provides a specific macro to use to set the proto flag 621 */ 622 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 623 pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO); 624 if (isCyGen2x(pService)) 625 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); 626 627 pCdInfo = &(pSessionDesc->contentDescInfo); 628 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 629 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == 630 pSessionDesc->cipherDirection) { 631 if (LAC_CIPHER_IS_GCM(cipher)) 632 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 633 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); 634 else if (LAC_CIPHER_IS_CHACHA(cipher)) 635 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 636 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); 637 } else if (isSpCcm) { 638 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 639 LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM); 640 } 641 642 /* Update slice type, as used algos changed */ 643 pSessionDesc->cipherSliceType = 644 LacCipher_GetCipherSliceType(pService, cipher, hash); 645 646 ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags, 647 pSessionDesc->cipherSliceType); 648 649 /* construct cipherConfig in CD in DRAM */ 650 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 651 pHwBlockBaseInDRAM + 652 hwBlockOffsetInDRAM, 653 &sizeInBytes); 654 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 655 pSessionDesc->reqSpcCacheHdr), 656 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 657 pSessionDesc->laCmdId, 658 pSessionDesc->cmnRequestFlags, 659 pSessionDesc->laCmdFlags); 660 } else if ((SPC == pSessionDesc->singlePassState) && 661 (LAC_CIPHER_SPC_IV_SIZE != pRequest->ivLenInBytes)) { 662 pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; 663 pSessionDesc->singlePassState = LIKELY_SPC; 664 pSessionDesc->isCipher = CPA_TRUE; 665 pSessionDesc->isAuthEncryptOp = CPA_TRUE; 666 pSessionDesc->isAuth = CPA_TRUE; 667 pCdInfo = &(pSessionDesc->contentDescInfo); 668 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 669 670 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 671 pSessionDesc->cipherDirection) { 672 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; 673 } else { 674 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; 675 } 676 677 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 678 pSessionDesc->laCmdFlags, 0); 679 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 680 ICP_QAT_FW_LA_GCM_PROTO); 681 682 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 683 pHwBlockBaseInDRAM + 684 hwBlockOffsetInDRAM, 685 &sizeInBytes); 686 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 687 pSessionDesc->reqCacheHdr), 688 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 689 pSessionDesc->laCmdId, 690 pSessionDesc->cmnRequestFlags, 691 pSessionDesc->laCmdFlags); 692 } else if (CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm) { 693 pSessionDesc->aadLenInBytes = pRequest->messageLenToHashInBytes; 694 } 695 if (SPC == pSessionDesc->singlePassState) { 696 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr); 697 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr); 698 } else { 699 if (!pSessionDesc->useSymConstantsTable) { 700 pCacheDummyHdr = (Cpa8U *)&(pSessionDesc->reqCacheHdr); 701 pCacheDummyFtr = (Cpa8U *)&(pSessionDesc->reqCacheFtr); 702 } else { 703 pCacheDummyHdr = 704 (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); 705 pCacheDummyFtr = 706 (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); 707 } 708 } 709 memcpy(pMsgDummy, 710 pCacheDummyHdr, 711 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); 712 memset((pMsgDummy + 713 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)), 714 0, 715 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); 716 memcpy(pMsgDummy + 717 (LAC_LONG_WORD_IN_BYTES * LAC_START_OF_CACHE_FTR_IN_LW), 718 pCacheDummyFtr, 719 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); 720 721 SalQatMsg_CmnMidWrite(pCurrentQatMsg, 722 pRequest, 723 (CPA_DP_BUFLIST == pRequest->srcBufferLen ? 724 QAT_COMN_PTR_TYPE_SGL : 725 QAT_COMN_PTR_TYPE_FLAT), 726 pRequest->srcBuffer, 727 pRequest->dstBuffer, 728 pRequest->srcBufferLen, 729 pRequest->dstBufferLen); 730 731 if ((CPA_CY_SYM_HASH_AES_CCM == pSessionDesc->hashAlgorithm && 732 pSessionDesc->isAuth == CPA_TRUE) || 733 isSpCcm) { 734 /* prepare IV and AAD for CCM */ 735 LacSymAlgChain_PrepareCCMData( 736 pSessionDesc, 737 pRequest->pAdditionalAuthData, 738 pRequest->pIv, 739 pRequest->messageLenToCipherInBytes, 740 pRequest->ivLenInBytes); 741 742 /* According to the API, for CCM and GCM, 743 * messageLenToHashInBytes and hashStartSrcOffsetInBytes are not 744 * initialized by the user and must be set by the driver 745 */ 746 pRequest->hashStartSrcOffsetInBytes = 747 pRequest->cryptoStartSrcOffsetInBytes; 748 pRequest->messageLenToHashInBytes = 749 pRequest->messageLenToCipherInBytes; 750 } else if ((SPC != pSessionDesc->singlePassState) && 751 (CPA_CY_SYM_HASH_AES_GCM == pSessionDesc->hashAlgorithm || 752 CPA_CY_SYM_HASH_AES_GMAC == pSessionDesc->hashAlgorithm)) { 753 /* GCM case */ 754 if (CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm) { 755 /* According to the API, for CCM and GCM, 756 * messageLenToHashInBytes and hashStartSrcOffsetInBytes 757 * are not initialized by the user and must be set 758 * by the driver 759 */ 760 pRequest->hashStartSrcOffsetInBytes = 761 pRequest->cryptoStartSrcOffsetInBytes; 762 pRequest->messageLenToHashInBytes = 763 pRequest->messageLenToCipherInBytes; 764 765 LacSymAlgChain_PrepareGCMData( 766 pSessionDesc, pRequest->pAdditionalAuthData); 767 } 768 769 if (LAC_CIPHER_IV_SIZE_GCM_12 == pRequest->ivLenInBytes) { 770 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( 771 pCurrentQatMsg->comn_hdr.serv_specif_flags, 772 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); 773 } 774 } 775 776 /* Write Request Params */ 777 if (pSessionDesc->isCipher) { 778 if (CPA_CY_SYM_CIPHER_ARC4 == pSessionDesc->cipherAlgorithm) { 779 /* ARC4 does not have an IV but the field is used to 780 * store the initial state */ 781 pRequest->iv = 782 pSessionDesc->cipherARC4InitialStatePhysAddr; 783 } 784 785 ICP_QAT_FW_LA_SLICE_TYPE_SET( 786 pCurrentQatMsg->comn_hdr.serv_specif_flags, 787 pSessionDesc->cipherSliceType); 788 789 LacSymQat_CipherRequestParamsPopulate( 790 pSessionDesc, 791 pCurrentQatMsg, 792 pRequest->cryptoStartSrcOffsetInBytes, 793 pRequest->messageLenToCipherInBytes, 794 pRequest->iv, 795 pRequest->pIv); 796 if (SPC == pSessionDesc->singlePassState) { 797 icp_qat_fw_la_cipher_req_params_t *pCipherReqParams = 798 (icp_qat_fw_la_cipher_req_params_t 799 *)((Cpa8U *)&( 800 pCurrentQatMsg->serv_specif_rqpars) + 801 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 802 803 icp_qat_fw_la_cipher_20_req_params_t *pCipher20ReqParams = 804 (void 805 *)((Cpa8U *)&( 806 pCurrentQatMsg->serv_specif_rqpars) + 807 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 808 809 if (isCyGen4x(pService)) { 810 pCipher20ReqParams->spc_aad_addr = 811 (uint64_t)pRequest->additionalAuthData; 812 pCipher20ReqParams->spc_aad_sz = 813 pSessionDesc->aadLenInBytes; 814 pCipher20ReqParams->spc_aad_offset = 0; 815 816 if (isSpCcm) 817 pCipher20ReqParams->spc_aad_sz += 818 LAC_CIPHER_CCM_AAD_OFFSET; 819 820 pCipher20ReqParams->spc_auth_res_addr = 821 (uint64_t)pRequest->digestResult; 822 pCipher20ReqParams->spc_auth_res_sz = 823 (Cpa8U)pSessionDesc->hashResultSize; 824 } else { 825 pCipherReqParams->spc_aad_addr = 826 (uint64_t)pRequest->additionalAuthData; 827 pCipherReqParams->spc_aad_sz = 828 (Cpa16U)pSessionDesc->aadLenInBytes; 829 830 pCipherReqParams->spc_auth_res_addr = 831 (uint64_t)pRequest->digestResult; 832 pCipherReqParams->spc_auth_res_sz = 833 (Cpa8U)pSessionDesc->hashResultSize; 834 } 835 836 /* For CHACHA and AES_GCM single pass AAD buffer needs 837 * alignment if aadLenInBytes is nonzero. In case of 838 * AES-GMAC, AAD buffer passed in the src buffer. 839 */ 840 if (0 != pSessionDesc->aadLenInBytes && 841 CPA_CY_SYM_HASH_AES_GMAC != 842 pSessionDesc->hashAlgorithm) { 843 blockLen = LacSymQat_CipherBlockSizeBytesGet( 844 pSessionDesc->cipherAlgorithm); 845 aadDataLen = pSessionDesc->aadLenInBytes; 846 847 /* In case of AES_CCM, B0 block size and 2 bytes 848 * of AAD len 849 * encoding need to be added to total AAD data 850 * len */ 851 if (isSpCcm) 852 aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET; 853 854 if ((aadDataLen % blockLen) != 0) { 855 paddingLen = 856 blockLen - (aadDataLen % blockLen); 857 memset(&pRequest->pAdditionalAuthData 858 [aadDataLen], 859 0, 860 paddingLen); 861 } 862 } 863 } 864 } 865 866 if (pSessionDesc->isAuth) { 867 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo = 868 &(pSessionDesc->hashStateBufferInfo); 869 icp_qat_fw_la_auth_req_params_t *pAuthReqPars = 870 (icp_qat_fw_la_auth_req_params_t 871 *)((Cpa8U *)&(pCurrentQatMsg->serv_specif_rqpars) + 872 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 873 874 if ((CPA_CY_SYM_HASH_SNOW3G_UIA2 != 875 pSessionDesc->hashAlgorithm && 876 CPA_CY_SYM_HASH_AES_CCM != pSessionDesc->hashAlgorithm && 877 CPA_CY_SYM_HASH_AES_GCM != pSessionDesc->hashAlgorithm && 878 CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm && 879 CPA_CY_SYM_HASH_ZUC_EIA3 != pSessionDesc->hashAlgorithm) && 880 (pHashStateBufferInfo->prefixAadSzQuadWords > 0)) { 881 /* prefixAadSzQuadWords > 0 when there is prefix data 882 - i.e. nested hash or HMAC no precompute cases 883 Note partials not supported on DP api so we do not need 884 dynamic hash state in this case */ 885 pRequest->additionalAuthData = 886 pHashStateBufferInfo->pDataPhys + 887 LAC_QUADWORDS_TO_BYTES( 888 pHashStateBufferInfo->stateStorageSzQuadWords); 889 } 890 891 /* The first 24 bytes in icp_qat_fw_la_auth_req_params_t can be 892 * copied directly from the op request data because they share a 893 * corresponding layout. The remaining 4 bytes are taken 894 * from the session message template and use values 895 * preconfigured at sessionInit (updated per request for some 896 * specific cases below) 897 */ 898 memcpy(pAuthReqPars, 899 (Cpa32U *)&(pRequest->hashStartSrcOffsetInBytes), 900 ((uintptr_t) & 901 (pAuthReqPars->u2.inner_prefix_sz) - 902 (uintptr_t)pAuthReqPars)); 903 904 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { 905 pAuthReqPars->hash_state_sz = 906 LAC_BYTES_TO_QUADWORDS(pAuthReqPars->u2.aad_sz); 907 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 908 pSessionDesc->hashAlgorithm || 909 CPA_CY_SYM_HASH_ZUC_EIA3 == 910 pSessionDesc->hashAlgorithm) { 911 pAuthReqPars->hash_state_sz = 912 LAC_BYTES_TO_QUADWORDS(pSessionDesc->aadLenInBytes); 913 } 914 } 915 916 } 917 918 CpaStatus 919 cpaCySymDpSessionCtxGetSize(const CpaInstanceHandle instanceHandle, 920 const CpaCySymSessionSetupData *pSessionSetupData, 921 Cpa32U *pSessionCtxSizeInBytes) 922 { 923 CpaStatus status = CPA_STATUS_SUCCESS; 924 925 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 926 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 927 /* All other param checks are common with trad api */ 928 /* Check for valid pointers */ 929 LAC_CHECK_NULL_PARAM(pSessionCtxSizeInBytes); 930 status = cpaCySymSessionCtxGetSize(instanceHandle, 931 pSessionSetupData, 932 pSessionCtxSizeInBytes); 933 934 return status; 935 } 936 937 CpaStatus 938 cpaCySymDpSessionCtxGetDynamicSize( 939 const CpaInstanceHandle instanceHandle, 940 const CpaCySymSessionSetupData *pSessionSetupData, 941 Cpa32U *pSessionCtxSizeInBytes) 942 { 943 CpaStatus status = CPA_STATUS_SUCCESS; 944 945 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 946 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 947 /* All other param checks are common with trad api */ 948 /* Check for valid pointers */ 949 LAC_CHECK_NULL_PARAM(pSessionCtxSizeInBytes); 950 status = cpaCySymSessionCtxGetDynamicSize(instanceHandle, 951 pSessionSetupData, 952 pSessionCtxSizeInBytes); 953 954 return status; 955 } 956 957 /** @ingroup cpaCySymDp */ 958 CpaStatus 959 cpaCySymDpInitSession(CpaInstanceHandle instanceHandle, 960 const CpaCySymSessionSetupData *pSessionSetupData, 961 CpaCySymDpSessionCtx sessionCtx) 962 { 963 CpaStatus status = CPA_STATUS_FAIL; 964 sal_service_t *pService = NULL; 965 966 967 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 968 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 969 (SAL_SERVICE_TYPE_CRYPTO | 970 SAL_SERVICE_TYPE_CRYPTO_SYM)); 971 LAC_CHECK_NULL_PARAM(pSessionSetupData); 972 pService = (sal_service_t *)instanceHandle; 973 974 /* Check crypto service is running otherwise return an error */ 975 SAL_RUNNING_CHECK(pService); 976 977 status = LacSym_InitSession(instanceHandle, 978 NULL, /* Callback */ 979 pSessionSetupData, 980 CPA_TRUE, /* isDPSession */ 981 sessionCtx); 982 return status; 983 } 984 985 CpaStatus 986 cpaCySymDpRemoveSession(const CpaInstanceHandle instanceHandle, 987 CpaCySymDpSessionCtx sessionCtx) 988 { 989 990 /* CPA_INSTANCE_HANDLE_SINGLE is not supported on DP apis */ 991 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 992 /* All other param checks are common with trad api */ 993 994 return cpaCySymRemoveSession(instanceHandle, sessionCtx); 995 } 996 997 CpaStatus 998 cpaCySymDpRegCbFunc(const CpaInstanceHandle instanceHandle, 999 const CpaCySymDpCbFunc pSymDpCb) 1000 { 1001 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; 1002 1003 1004 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1005 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1006 (SAL_SERVICE_TYPE_CRYPTO | 1007 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1008 LAC_CHECK_NULL_PARAM(pSymDpCb); 1009 SAL_RUNNING_CHECK(instanceHandle); 1010 pService->pSymDpCb = pSymDpCb; 1011 1012 return CPA_STATUS_SUCCESS; 1013 } 1014 1015 CpaStatus 1016 cpaCySymDpEnqueueOp(CpaCySymDpOpData *pRequest, const CpaBoolean performOpNow) 1017 { 1018 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg = NULL; 1019 icp_comms_trans_handle trans_handle = NULL; 1020 lac_session_desc_t *pSessionDesc = NULL; 1021 write_ringMsgFunc_t callFunc; 1022 1023 CpaStatus status = CPA_STATUS_SUCCESS; 1024 1025 1026 LAC_CHECK_NULL_PARAM(pRequest); 1027 status = LacDp_EnqueueParamCheck(pRequest); 1028 if (CPA_STATUS_SUCCESS != status) { 1029 return status; 1030 } 1031 1032 /* Check if SAL is running in crypto data plane otherwise return an 1033 * error */ 1034 SAL_RUNNING_CHECK(pRequest->instanceHandle); 1035 1036 trans_handle = ((sal_crypto_service_t *)pRequest->instanceHandle) 1037 ->trans_handle_sym_tx; 1038 1039 pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequest->sessionCtx); 1040 icp_adf_getSingleQueueAddr(trans_handle, (void **)&pCurrentQatMsg); 1041 if (NULL == pCurrentQatMsg) { 1042 /* 1043 * No space is available on the queue. 1044 */ 1045 return CPA_STATUS_RETRY; 1046 } 1047 1048 callFunc = (write_ringMsgFunc_t)pSessionDesc->writeRingMsgFunc; 1049 1050 LAC_CHECK_NULL_PARAM(callFunc); 1051 1052 callFunc(pRequest, pCurrentQatMsg); 1053 1054 qatUtilsAtomicInc(&(pSessionDesc->u.pendingDpCbCount)); 1055 1056 if (CPA_TRUE == performOpNow) { 1057 SalQatMsg_updateQueueTail(trans_handle); 1058 } 1059 1060 return CPA_STATUS_SUCCESS; 1061 } 1062 1063 CpaStatus 1064 cpaCySymDpPerformOpNow(const CpaInstanceHandle instanceHandle) 1065 { 1066 icp_comms_trans_handle trans_handle = NULL; 1067 1068 1069 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1070 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1071 (SAL_SERVICE_TYPE_CRYPTO | 1072 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1073 1074 /* Check if SAL is initialised otherwise return an error */ 1075 SAL_RUNNING_CHECK(instanceHandle); 1076 1077 trans_handle = 1078 ((sal_crypto_service_t *)instanceHandle)->trans_handle_sym_tx; 1079 1080 if (CPA_TRUE == icp_adf_queueDataToSend(trans_handle)) { 1081 SalQatMsg_updateQueueTail(trans_handle); 1082 } 1083 1084 return CPA_STATUS_SUCCESS; 1085 } 1086 1087 CpaStatus 1088 cpaCySymDpEnqueueOpBatch(const Cpa32U numberRequests, 1089 CpaCySymDpOpData *pRequests[], 1090 const CpaBoolean performOpNow) 1091 { 1092 icp_qat_fw_la_bulk_req_t *pCurrentQatMsg = NULL; 1093 icp_comms_trans_handle trans_handle = NULL; 1094 lac_session_desc_t *pSessionDesc = NULL; 1095 write_ringMsgFunc_t callFunc; 1096 Cpa32U i = 0; 1097 1098 CpaStatus status = CPA_STATUS_SUCCESS; 1099 sal_crypto_service_t *pService = NULL; 1100 1101 1102 LAC_CHECK_NULL_PARAM(pRequests); 1103 LAC_CHECK_NULL_PARAM(pRequests[0]); 1104 LAC_CHECK_NULL_PARAM(pRequests[0]->instanceHandle); 1105 1106 pService = (sal_crypto_service_t *)(pRequests[0]->instanceHandle); 1107 1108 if ((0 == numberRequests) || 1109 (numberRequests > pService->maxNumSymReqBatch)) { 1110 LAC_INVALID_PARAM_LOG1( 1111 "The number of requests needs to be between 1 " 1112 "and %d", 1113 pService->maxNumSymReqBatch); 1114 return CPA_STATUS_INVALID_PARAM; 1115 } 1116 1117 for (i = 0; i < numberRequests; i++) { 1118 status = LacDp_EnqueueParamCheck(pRequests[i]); 1119 if (CPA_STATUS_SUCCESS != status) { 1120 return status; 1121 } 1122 1123 /* Check that all instance handles are the same */ 1124 if (pRequests[i]->instanceHandle != 1125 pRequests[0]->instanceHandle) { 1126 LAC_INVALID_PARAM_LOG( 1127 "All instance handles should be the same " 1128 "in the requests"); 1129 return CPA_STATUS_INVALID_PARAM; 1130 } 1131 } 1132 1133 /* Check if SAL is running in crypto data plane otherwise return an 1134 * error */ 1135 SAL_RUNNING_CHECK(pRequests[0]->instanceHandle); 1136 1137 trans_handle = ((sal_crypto_service_t *)pRequests[0]->instanceHandle) 1138 ->trans_handle_sym_tx; 1139 pSessionDesc = 1140 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequests[0]->sessionCtx); 1141 icp_adf_getQueueMemory(trans_handle, 1142 numberRequests, 1143 (void **)&pCurrentQatMsg); 1144 if (NULL == pCurrentQatMsg) { 1145 /* 1146 * No space is available on the queue. 1147 */ 1148 return CPA_STATUS_RETRY; 1149 } 1150 1151 for (i = 0; i < numberRequests; i++) { 1152 pSessionDesc = 1153 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pRequests[i]->sessionCtx); 1154 callFunc = (write_ringMsgFunc_t)pSessionDesc->writeRingMsgFunc; 1155 callFunc(pRequests[i], pCurrentQatMsg); 1156 icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); 1157 qatUtilsAtomicAdd(1, &(pSessionDesc->u.pendingDpCbCount)); 1158 } 1159 1160 if (CPA_TRUE == performOpNow) { 1161 SalQatMsg_updateQueueTail(trans_handle); 1162 } 1163 1164 return CPA_STATUS_SUCCESS; 1165 } 1166 1167 CpaStatus 1168 icp_sal_CyPollDpInstance(const CpaInstanceHandle instanceHandle, 1169 const Cpa32U responseQuota) 1170 { 1171 icp_comms_trans_handle trans_handle = NULL; 1172 1173 LAC_CHECK_INSTANCE_HANDLE(instanceHandle); 1174 SAL_CHECK_INSTANCE_TYPE(instanceHandle, 1175 (SAL_SERVICE_TYPE_CRYPTO | 1176 SAL_SERVICE_TYPE_CRYPTO_SYM)); 1177 1178 /* Check if SAL is initialised otherwise return an error */ 1179 SAL_RUNNING_CHECK(instanceHandle); 1180 1181 trans_handle = 1182 ((sal_crypto_service_t *)instanceHandle)->trans_handle_sym_rx; 1183 1184 return icp_adf_pollQueue(trans_handle, responseQuota); 1185 } 1186