1 /*************************************************************************** 2 * 3 * <COPYRIGHT_TAG> 4 * 5 ***************************************************************************/ 6 7 /** 8 *************************************************************************** 9 * @file lac_sym_alg_chain.c Algorithm Chaining Perform 10 * 11 * @ingroup LacAlgChain 12 ***************************************************************************/ 13 14 /* 15 ******************************************************************************* 16 * Include public/global header files 17 ******************************************************************************* 18 */ 19 20 #include "cpa.h" 21 #include "cpa_cy_sym.h" 22 23 #include "icp_accel_devices.h" 24 #include "icp_adf_init.h" 25 #include "icp_adf_transport.h" 26 #include "icp_adf_debug.h" 27 28 /* 29 ******************************************************************************* 30 * Include private header files 31 ******************************************************************************* 32 */ 33 34 #include "lac_mem.h" 35 #include "lac_log.h" 36 #include "lac_sym.h" 37 #include "lac_list.h" 38 #include "icp_qat_fw_la.h" 39 #include "lac_sal_types_crypto.h" 40 #include "lac_sal.h" 41 #include "lac_sal_ctrl.h" 42 #include "lac_sym_alg_chain.h" 43 #include "lac_sym_cipher.h" 44 #include "lac_sym_cipher_defs.h" 45 #include "lac_sym_hash.h" 46 #include "lac_sym_hash_defs.h" 47 #include "lac_sym_qat_cipher.h" 48 #include "lac_sym_qat_hash.h" 49 #include "lac_sym_stats.h" 50 #include "lac_sym_queue.h" 51 #include "lac_sym_cb.h" 52 #include "sal_string_parse.h" 53 #include "lac_sym_auth_enc.h" 54 #include "lac_sym_qat.h" 55 #include "sal_hw_gen.h" 56 57 /** 58 * @ingroup LacAlgChain 59 * This callback function will be invoked whenever a hash precompute 60 * operation completes. It will dequeue and send any QAT requests 61 * which were queued up while the precompute was in progress. 62 * 63 * @param[in] callbackTag Opaque value provided by user. This will 64 * be a pointer to the session descriptor. 65 * 66 * @retval 67 * None 68 * 69 */ 70 static void 71 LacSymAlgChain_HashPrecomputeDoneCb(void *callbackTag) 72 { 73 LacSymCb_PendingReqsDequeue((lac_session_desc_t *)callbackTag); 74 } 75 76 /** 77 * @ingroup LacAlgChain 78 * Walk the buffer list and find the address for the given offset within 79 * a buffer. 80 * 81 * @param[in] pBufferList Buffer List 82 * @param[in] packetOffset Offset in the buffer list for which address 83 * is to be found. 84 * @param[out] ppDataPtr This is where the sought pointer will be put 85 * @param[out] pSpaceLeft Pointer to a variable in which information about 86 * available space from the given offset to the end 87 * of the flat buffer it is located in will be returned 88 * 89 * @retval CPA_STATUS_SUCCESS Address with a given offset is found in the list 90 * @retval CPA_STATUS_FAIL Address with a given offset not found in the list. 91 * 92 */ 93 static CpaStatus 94 LacSymAlgChain_PtrFromOffsetGet(const CpaBufferList *pBufferList, 95 const Cpa32U packetOffset, 96 Cpa8U **ppDataPtr) 97 { 98 Cpa32U currentOffset = 0; 99 Cpa32U i = 0; 100 101 for (i = 0; i < pBufferList->numBuffers; i++) { 102 Cpa8U *pCurrData = pBufferList->pBuffers[i].pData; 103 Cpa32U currDataSize = pBufferList->pBuffers[i].dataLenInBytes; 104 105 /* If the offset is within the address space of the current 106 * buffer */ 107 if ((packetOffset >= currentOffset) && 108 (packetOffset < (currentOffset + currDataSize))) { 109 /* increment by offset of the address in the current 110 * buffer */ 111 *ppDataPtr = pCurrData + (packetOffset - currentOffset); 112 return CPA_STATUS_SUCCESS; 113 } 114 115 /* Increment by the size of the buffer */ 116 currentOffset += currDataSize; 117 } 118 119 return CPA_STATUS_FAIL; 120 } 121 122 /** 123 * @ingroup LacAlgChain 124 * Function which checks for support of partial packets for symmetric 125 * crypto operations 126 * 127 * @param[in] pService Pointer to service descriptor 128 * @param[in/out] pSessionDesc Pointer to session descriptor 129 * 130 */ 131 static void 132 LacSymCheck_IsPartialSupported(Cpa32U capabilitiesMask, 133 lac_session_desc_t *pSessionDesc) 134 { 135 CpaBoolean isHashPartialSupported = CPA_FALSE; 136 CpaBoolean isCipherPartialSupported = CPA_FALSE; 137 CpaBoolean isPartialSupported = CPA_FALSE; 138 139 switch (pSessionDesc->cipherAlgorithm) { 140 /* Following ciphers don't support partial */ 141 case CPA_CY_SYM_CIPHER_KASUMI_F8: 142 case CPA_CY_SYM_CIPHER_AES_F8: 143 case CPA_CY_SYM_CIPHER_SNOW3G_UEA2: 144 case CPA_CY_SYM_CIPHER_CHACHA: 145 case CPA_CY_SYM_CIPHER_ZUC_EEA3: 146 break; 147 /* All others support partial */ 148 default: 149 isCipherPartialSupported = CPA_TRUE; 150 break; 151 } 152 switch (pSessionDesc->hashAlgorithm) { 153 /* Following hash don't support partial */ 154 case CPA_CY_SYM_HASH_KASUMI_F9: 155 case CPA_CY_SYM_HASH_SNOW3G_UIA2: 156 case CPA_CY_SYM_HASH_POLY: 157 case CPA_CY_SYM_HASH_ZUC_EIA3: 158 break; 159 /* Following hash may support partial based on device capabilities */ 160 case CPA_CY_SYM_HASH_SHA3_256: 161 if (ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask) { 162 isHashPartialSupported = CPA_TRUE; 163 } 164 break; 165 /* All others support partial */ 166 default: 167 isHashPartialSupported = CPA_TRUE; 168 break; 169 } 170 switch (pSessionDesc->symOperation) { 171 case CPA_CY_SYM_OP_CIPHER: 172 isPartialSupported = isCipherPartialSupported; 173 break; 174 case CPA_CY_SYM_OP_HASH: 175 isPartialSupported = isHashPartialSupported; 176 break; 177 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: 178 if (isCipherPartialSupported && isHashPartialSupported) { 179 isPartialSupported = CPA_TRUE; 180 } 181 break; 182 case CPA_CY_SYM_OP_NONE: 183 break; 184 default: 185 break; 186 } 187 188 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) { 189 /* UCS slice has no support for state flush and 190 * because of that is not able to do partial processing */ 191 isPartialSupported = CPA_FALSE; 192 } 193 194 pSessionDesc->isPartialSupported = isPartialSupported; 195 } 196 197 static void 198 LacAlgChain_CipherCDBuild_ForOptimisedCD( 199 const CpaCySymCipherSetupData *pCipherData, 200 lac_session_desc_t *pSessionDesc, 201 icp_qat_fw_slice_t nextSlice, 202 Cpa8U cipherOffsetInConstantsTable, 203 Cpa8U *pOptimisedHwBlockBaseInDRAM, 204 Cpa32U *pOptimisedHwBlockOffsetInDRAM) 205 { 206 Cpa8U *pCipherKeyField = NULL; 207 Cpa32U sizeInBytes = 0; 208 pCipherKeyField = pOptimisedHwBlockBaseInDRAM; 209 210 /* Need to build up the alternative CD for SHRAM Constants Table use 211 * with an optimised content desc of 64 bytes for this case. Cipher key 212 * will be in the Content desc in DRAM, The cipher config data 213 * is now in the SHRAM constants table. */ 214 215 LacSymQat_CipherHwBlockPopulateKeySetup( 216 pSessionDesc, 217 pCipherData, 218 pCipherData->cipherKeyLenInBytes, 219 pSessionDesc->cipherSliceType, 220 pCipherKeyField, 221 &sizeInBytes); 222 223 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr), 224 pSessionDesc->cipherAlgorithm, 225 pSessionDesc->cipherKeyLenInBytes, 226 pSessionDesc->cipherSliceType, 227 nextSlice, 228 cipherOffsetInConstantsTable); 229 230 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes; 231 } 232 233 static void 234 LacAlgChain_CipherCDBuild_ForSHRAM(const CpaCySymCipherSetupData *pCipherData, 235 lac_session_desc_t *pSessionDesc, 236 icp_qat_fw_slice_t nextSlice, 237 Cpa8U cipherOffsetInConstantsTable) 238 { 239 Cpa32U sizeInBytes = 0; 240 Cpa8U *pCipherKeyField = NULL; 241 /* Need to build up the alternative CD for SHRAM Constants Table use 242 * Cipher key will be in the Request, The cipher config data is now in 243 * the SHRAM constants table. And nothing is now stored in the content 244 * desc */ 245 pCipherKeyField = (Cpa8U *)&( 246 pSessionDesc->shramReqCacheHdr.cd_pars.s1.serv_specif_fields); 247 248 LacSymQat_CipherHwBlockPopulateKeySetup( 249 pSessionDesc, 250 pCipherData, 251 pCipherData->cipherKeyLenInBytes, 252 pSessionDesc->cipherSliceType, 253 pCipherKeyField, 254 &sizeInBytes); 255 256 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->shramReqCacheFtr), 257 pSessionDesc->cipherAlgorithm, 258 pSessionDesc->cipherKeyLenInBytes, 259 pSessionDesc->cipherSliceType, 260 nextSlice, 261 cipherOffsetInConstantsTable); 262 } 263 264 static void 265 LacAlgChain_CipherCDBuild(const CpaCySymCipherSetupData *pCipherData, 266 lac_session_desc_t *pSessionDesc, 267 icp_qat_fw_slice_t nextSlice, 268 Cpa8U cipherOffsetInConstantsTable, 269 icp_qat_fw_comn_flags *pCmnRequestFlags, 270 icp_qat_fw_serv_specif_flags *pLaCmdFlags, 271 Cpa8U *pHwBlockBaseInDRAM, 272 Cpa32U *pHwBlockOffsetInDRAM, 273 Cpa32U capabilitiesMask) 274 { 275 Cpa8U *pCipherKeyField = NULL; 276 Cpa8U cipherOffsetInReqQW = 0; 277 Cpa32U sizeInBytes = 0; 278 void *pCfgData = NULL; 279 Cpa32U cfgOffset = 0; 280 281 /* Construct the ContentDescriptor in DRAM */ 282 cipherOffsetInReqQW = (*pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES); 283 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET( 284 *pLaCmdFlags, ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP); 285 286 /* construct cipherConfig in CD in DRAM */ 287 cfgOffset = *pHwBlockOffsetInDRAM; 288 pCfgData = pHwBlockBaseInDRAM + cfgOffset; 289 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 290 pCfgData, 291 &sizeInBytes); 292 293 ICP_QAT_FW_LA_SLICE_TYPE_SET(*pLaCmdFlags, 294 pSessionDesc->cipherSliceType); 295 296 *pHwBlockOffsetInDRAM += sizeInBytes; 297 298 /* Cipher key will be in CD in DRAM. 299 * The Request contains a ptr to the CD. 300 * This ptr will be copied into the request later once the CD is 301 * fully constructed, but the flag is set here. */ 302 pCipherKeyField = pHwBlockBaseInDRAM + *pHwBlockOffsetInDRAM; 303 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(*pCmnRequestFlags, 304 QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 305 306 LacSymQat_CipherHwBlockPopulateKeySetup( 307 pSessionDesc, 308 pCipherData, 309 pCipherData->cipherKeyLenInBytes, 310 pSessionDesc->cipherSliceType, 311 pCipherKeyField, 312 &sizeInBytes); 313 /* update offset */ 314 *pHwBlockOffsetInDRAM += sizeInBytes; 315 316 LacSymQat_CipherCtrlBlockWrite(&(pSessionDesc->reqCacheFtr), 317 pSessionDesc->cipherAlgorithm, 318 pSessionDesc->cipherKeyLenInBytes, 319 pSessionDesc->cipherSliceType, 320 nextSlice, 321 cipherOffsetInReqQW); 322 if (NON_SPC != pSessionDesc->singlePassState) { 323 LacSymQat_CipherCtrlBlockWrite( 324 &(pSessionDesc->reqSpcCacheFtr), 325 pSessionDesc->cipherAlgorithm, 326 pSessionDesc->cipherKeyLenInBytes, 327 pSessionDesc->cipherSliceType, 328 ICP_QAT_FW_SLICE_DRAM_WR, 329 cipherOffsetInReqQW); 330 } 331 } 332 333 static void 334 LacAlgChain_HashCDBuild( 335 const CpaCySymHashSetupData *pHashData, 336 CpaInstanceHandle instanceHandle, 337 lac_session_desc_t *pSessionDesc, 338 icp_qat_fw_slice_t nextSlice, 339 Cpa8U hashOffsetInConstantsTable, 340 icp_qat_fw_comn_flags *pCmnRequestFlags, 341 icp_qat_fw_serv_specif_flags *pLaCmdFlags, 342 lac_sym_qat_hash_precompute_info_t *pPrecomputeData, 343 lac_sym_qat_hash_precompute_info_t *pPrecomputeDataOptimisedCd, 344 Cpa8U *pHwBlockBaseInDRAM, 345 Cpa32U *pHwBlockOffsetInDRAM, 346 Cpa8U *pOptimisedHwBlockBaseInDRAM, 347 Cpa32U *pOptimisedHwBlockOffsetInDRAM) 348 { 349 Cpa32U sizeInBytes = 0; 350 Cpa32U hwBlockOffsetInQuadWords = 351 *pHwBlockOffsetInDRAM / LAC_QUAD_WORD_IN_BYTES; 352 353 /* build: 354 * - the hash part of the ContentDescriptor in DRAM */ 355 /* - the hash part of the CD control block in the Request template */ 356 LacSymQat_HashContentDescInit(&(pSessionDesc->reqCacheFtr), 357 instanceHandle, 358 pHashData, 359 pHwBlockBaseInDRAM, 360 hwBlockOffsetInQuadWords, 361 nextSlice, 362 pSessionDesc->qatHashMode, 363 CPA_FALSE, 364 CPA_FALSE, 365 pSessionDesc->useStatefulSha3ContentDesc, 366 pPrecomputeData, 367 &sizeInBytes); 368 369 /* Using DRAM CD so update offset */ 370 *pHwBlockOffsetInDRAM += sizeInBytes; 371 372 sizeInBytes = 0; 373 374 if (pSessionDesc->useOptimisedContentDesc) { 375 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr), 376 instanceHandle, 377 pHashData, 378 pOptimisedHwBlockBaseInDRAM, 379 hashOffsetInConstantsTable, 380 nextSlice, 381 pSessionDesc->qatHashMode, 382 CPA_TRUE, 383 CPA_TRUE, 384 CPA_FALSE, 385 pPrecomputeDataOptimisedCd, 386 &sizeInBytes); 387 388 *pOptimisedHwBlockOffsetInDRAM += sizeInBytes; 389 } else if (pSessionDesc->useSymConstantsTable) { 390 /* Need to build up the alternative CD for SHRAM Constants Table 391 * use */ 392 LacSymQat_HashContentDescInit(&(pSessionDesc->shramReqCacheFtr), 393 instanceHandle, 394 pHashData, 395 pHwBlockBaseInDRAM, 396 hashOffsetInConstantsTable, 397 nextSlice, 398 pSessionDesc->qatHashMode, 399 CPA_TRUE, 400 CPA_FALSE, 401 CPA_FALSE, 402 pPrecomputeData, 403 &sizeInBytes); 404 } 405 } 406 407 static Cpa16U 408 LacAlgChain_GetCipherConfigSize(lac_session_desc_t *pSessionDesc) 409 { 410 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == pSessionDesc->cipherSliceType) { 411 return sizeof(icp_qat_hw_ucs_cipher_config_t); 412 } else { 413 return sizeof(icp_qat_hw_cipher_config_t); 414 } 415 } 416 417 static Cpa16U 418 LacAlgChain_GetCipherConfigOffset(lac_session_desc_t *pSessionDesc) 419 { 420 Cpa16U offset = 0; 421 422 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == pSessionDesc->symOperation || 423 SPC == pSessionDesc->singlePassState) { 424 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = 425 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc 426 ->reqCacheFtr.cd_ctrl; 427 offset = cd_ctrl->cipher_cfg_offset; 428 } else if (CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) { 429 icp_qat_fw_cipher_cd_ctrl_hdr_t *cd_ctrl = 430 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&pSessionDesc 431 ->reqCacheFtr.cd_ctrl; 432 offset = cd_ctrl->cipher_cfg_offset; 433 } 434 435 return offset * LAC_QUAD_WORD_IN_BYTES; 436 } 437 438 CpaStatus 439 LacAlgChain_SessionAADUpdate(lac_session_desc_t *pSessionDesc, 440 Cpa32U newAADLength) 441 { 442 icp_qat_la_bulk_req_ftr_t *req_ftr = &pSessionDesc->reqCacheFtr; 443 icp_qat_la_auth_req_params_t *req_params = &req_ftr->serv_specif_rqpars; 444 445 if (!pSessionDesc) 446 return CPA_STATUS_FAIL; 447 448 pSessionDesc->aadLenInBytes = newAADLength; 449 req_params->u2.aad_sz = 450 LAC_ALIGN_POW2_ROUNDUP(newAADLength, LAC_HASH_AES_GCM_BLOCK_SIZE); 451 452 if (SPC == pSessionDesc->singlePassState) { 453 Cpa8U *pHwBlockBaseInDRAM = NULL; 454 Cpa32U hwBlockOffsetInDRAM = 0; 455 Cpa32U pSizeInBytes = 0; 456 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; 457 458 pHwBlockBaseInDRAM = 459 (Cpa8U *)pSessionDesc->contentDescInfo.pData; 460 if (pSessionDesc->cipherDirection == 461 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) { 462 if (LAC_CIPHER_IS_GCM(cipher)) { 463 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 464 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); 465 } else { 466 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 467 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); 468 } 469 } 470 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 471 pHwBlockBaseInDRAM + 472 hwBlockOffsetInDRAM, 473 &pSizeInBytes); 474 } 475 476 return CPA_STATUS_SUCCESS; 477 } 478 479 CpaStatus 480 LacAlgChain_SessionCipherKeyUpdate(lac_session_desc_t *pSessionDesc, 481 Cpa8U *pCipherKey) 482 { 483 CpaStatus status = CPA_STATUS_SUCCESS; 484 485 if (pSessionDesc == NULL || pCipherKey == NULL) 486 return CPA_STATUS_FAIL; 487 488 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { 489 LacSymQat_CipherArc4StateInit( 490 pCipherKey, 491 pSessionDesc->cipherKeyLenInBytes, 492 pSessionDesc->cipherARC4InitialState); 493 } else { 494 CpaCySymCipherSetupData cipherSetupData = { 0 }; 495 Cpa32U sizeInBytes; 496 Cpa8U *pCipherKeyField; 497 Cpa16U cipherConfigSize; 498 Cpa16U cipherConfigOffset; 499 sal_qat_content_desc_info_t *pCdInfo = 500 &(pSessionDesc->contentDescInfo); 501 502 cipherSetupData.cipherAlgorithm = pSessionDesc->cipherAlgorithm; 503 cipherSetupData.cipherKeyLenInBytes = 504 pSessionDesc->cipherKeyLenInBytes; 505 cipherSetupData.pCipherKey = pCipherKey; 506 cipherSetupData.cipherDirection = pSessionDesc->cipherDirection; 507 508 cipherConfigSize = 509 LacAlgChain_GetCipherConfigSize(pSessionDesc); 510 cipherConfigOffset = 511 LacAlgChain_GetCipherConfigOffset(pSessionDesc); 512 513 pCipherKeyField = (Cpa8U *)pCdInfo->pData + cipherConfigOffset + 514 cipherConfigSize; 515 516 switch (pSessionDesc->symOperation) { 517 case CPA_CY_SYM_OP_CIPHER: { 518 LacSymQat_CipherHwBlockPopulateKeySetup( 519 pSessionDesc, 520 &(cipherSetupData), 521 cipherSetupData.cipherKeyLenInBytes, 522 pSessionDesc->cipherSliceType, 523 pCipherKeyField, 524 &sizeInBytes); 525 526 if (pSessionDesc->useSymConstantsTable) { 527 pCipherKeyField = (Cpa8U *)&( 528 pSessionDesc->shramReqCacheHdr.cd_pars.s1 529 .serv_specif_fields); 530 531 LacSymQat_CipherHwBlockPopulateKeySetup( 532 pSessionDesc, 533 &(cipherSetupData), 534 cipherSetupData.cipherKeyLenInBytes, 535 pSessionDesc->cipherSliceType, 536 pCipherKeyField, 537 &sizeInBytes); 538 } 539 } break; 540 541 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: { 542 LacSymQat_CipherHwBlockPopulateKeySetup( 543 pSessionDesc, 544 &(cipherSetupData), 545 cipherSetupData.cipherKeyLenInBytes, 546 pSessionDesc->cipherSliceType, 547 pCipherKeyField, 548 &sizeInBytes); 549 } break; 550 551 default: 552 LAC_LOG_ERROR("Invalid sym operation\n"); 553 status = CPA_STATUS_INVALID_PARAM; 554 break; 555 } 556 } 557 return status; 558 } 559 560 CpaStatus 561 LacAlgChain_SessionAuthKeyUpdate(lac_session_desc_t *pSessionDesc, 562 Cpa8U *pAuthKey) 563 { 564 CpaStatus status = CPA_STATUS_SUCCESS; 565 Cpa8U *pHwBlockBaseInDRAM = NULL; 566 Cpa8U *pOutHashSetup = NULL; 567 Cpa8U *pInnerState1 = NULL; 568 Cpa8U *pInnerState2 = NULL; 569 CpaCySymSessionSetupData sessionSetup = { 0 }; 570 Cpa16U cipherConfigSize; 571 572 if (pSessionDesc == NULL || pAuthKey == NULL) 573 return CPA_STATUS_FAIL; 574 575 cipherConfigSize = LacAlgChain_GetCipherConfigSize(pSessionDesc); 576 577 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = 578 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr 579 .cd_ctrl; 580 581 pHwBlockBaseInDRAM = (Cpa8U *)pSessionDesc->contentDescInfo.pData; 582 583 sessionSetup.hashSetupData.hashAlgorithm = pSessionDesc->hashAlgorithm; 584 sessionSetup.hashSetupData.hashMode = pSessionDesc->hashMode; 585 sessionSetup.hashSetupData.authModeSetupData.authKey = pAuthKey; 586 sessionSetup.hashSetupData.authModeSetupData.authKeyLenInBytes = 587 pSessionDesc->authKeyLenInBytes; 588 sessionSetup.hashSetupData.authModeSetupData.aadLenInBytes = 589 pSessionDesc->aadLenInBytes; 590 sessionSetup.hashSetupData.digestResultLenInBytes = 591 pSessionDesc->hashResultSize; 592 593 sessionSetup.cipherSetupData.cipherAlgorithm = 594 pSessionDesc->cipherAlgorithm; 595 sessionSetup.cipherSetupData.cipherKeyLenInBytes = 596 pSessionDesc->cipherKeyLenInBytes; 597 598 /* Calculate hash states offsets */ 599 pInnerState1 = pHwBlockBaseInDRAM + 600 cd_ctrl->hash_cfg_offset * LAC_QUAD_WORD_IN_BYTES + 601 sizeof(icp_qat_hw_auth_setup_t); 602 603 pInnerState2 = pInnerState1 + cd_ctrl->inner_state1_sz; 604 605 pOutHashSetup = pInnerState2 + cd_ctrl->inner_state2_sz; 606 607 /* Calculate offset of cipher key */ 608 if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { 609 sessionSetup.cipherSetupData.pCipherKey = 610 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize; 611 } else if (pSessionDesc->laCmdId == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { 612 sessionSetup.cipherSetupData.pCipherKey = 613 pOutHashSetup + cipherConfigSize; 614 } else if (SPC == pSessionDesc->singlePassState) { 615 CpaCySymCipherAlgorithm cipher = pSessionDesc->cipherAlgorithm; 616 Cpa32U hwBlockOffsetInDRAM = 0; 617 618 if (pSessionDesc->cipherDirection == 619 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) { 620 sessionSetup.cipherSetupData.pCipherKey = 621 (Cpa8U *)pHwBlockBaseInDRAM + cipherConfigSize; 622 } else { 623 if (LAC_CIPHER_IS_GCM(cipher)) 624 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 625 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); 626 else 627 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 628 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); 629 sessionSetup.cipherSetupData.pCipherKey = 630 (Cpa8U *)pHwBlockBaseInDRAM + hwBlockOffsetInDRAM + 631 cipherConfigSize; 632 } 633 } 634 635 if (!sessionSetup.cipherSetupData.pCipherKey) 636 return CPA_STATUS_FAIL; 637 638 if (CPA_CY_SYM_HASH_SHA3_256 == pSessionDesc->hashAlgorithm) { 639 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) { 640 lac_sym_qat_hash_state_buffer_info_t 641 *pHashStateBufferInfo = 642 &(pSessionDesc->hashStateBufferInfo); 643 644 sal_crypto_service_t *pService = 645 (sal_crypto_service_t *)pSessionDesc->pInstance; 646 647 status = LacHash_StatePrefixAadBufferInit( 648 &(pService->generic_service_info), 649 &(sessionSetup.hashSetupData), 650 &(pSessionDesc->reqCacheFtr), 651 pSessionDesc->qatHashMode, 652 pSessionDesc->hashStatePrefixBuffer, 653 pHashStateBufferInfo); 654 /* SHRAM Constants Table not used for Auth-Enc */ 655 } 656 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pSessionDesc->hashAlgorithm) { 657 Cpa8U *authKey = (Cpa8U *)pOutHashSetup + cipherConfigSize; 658 memcpy(authKey, pAuthKey, pSessionDesc->authKeyLenInBytes); 659 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm || 660 CPA_CY_SYM_HASH_AES_CBC_MAC == pSessionDesc->hashAlgorithm) { 661 memcpy(pInnerState2, pAuthKey, pSessionDesc->authKeyLenInBytes); 662 } else if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm || 663 CPA_CY_SYM_HASH_KASUMI_F9 == pSessionDesc->hashAlgorithm || 664 IS_HASH_MODE_1(pSessionDesc->qatHashMode)) { 665 if (CPA_CY_SYM_HASH_AES_CMAC == pSessionDesc->hashAlgorithm) { 666 memset(pInnerState2, 0, cd_ctrl->inner_state2_sz); 667 } 668 669 /* Block messages until precompute is completed */ 670 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE; 671 672 status = LacHash_PrecomputeDataCreate( 673 pSessionDesc->pInstance, 674 (CpaCySymSessionSetupData *)&(sessionSetup), 675 LacSymAlgChain_HashPrecomputeDoneCb, 676 pSessionDesc, 677 pSessionDesc->hashStatePrefixBuffer, 678 pInnerState1, 679 pInnerState2); 680 } 681 682 return status; 683 } 684 685 static void 686 buildCmdData(sal_crypto_service_t *pService, 687 lac_session_desc_t *pSessionDesc, 688 CpaCySymAlgChainOrder *chainOrder, 689 Cpa16U *proto, 690 icp_qat_fw_serv_specif_flags *laCmdFlags, 691 icp_qat_fw_comn_flags *cmnRequestFlags) 692 { 693 /* LW 28 is used to set hash flags for AlgChaining. */ 694 icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *cd_ctrl = 695 (icp_qat_fw_cipher_auth_cd_ctrl_hdr_t *)&pSessionDesc->reqCacheFtr 696 .cd_ctrl; 697 698 /* proto refers to Protocol Flags, which is legacy FW <=> IA interface 699 * for ZUC and Snow3G. Use extended protocol flags for AlgChaining. 700 */ 701 *proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ 702 703 switch (pSessionDesc->symOperation) { 704 case CPA_CY_SYM_OP_CIPHER: 705 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; 706 707 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == 708 pSessionDesc->cipherAlgorithm) { 709 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; 710 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == 711 pSessionDesc->cipherAlgorithm) { 712 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; 713 } 714 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) { 715 *proto = ICP_QAT_FW_LA_CCM_PROTO; 716 } 717 break; 718 719 case CPA_CY_SYM_OP_HASH: 720 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_AUTH; 721 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 722 pSessionDesc->hashAlgorithm) { 723 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; 724 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == 725 pSessionDesc->hashAlgorithm) { 726 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; 727 } 728 break; 729 730 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: 731 if (LAC_CIPHER_IS_CCM(pSessionDesc->cipherAlgorithm)) { 732 *proto = ICP_QAT_FW_LA_CCM_PROTO; 733 734 /* Derive chainOrder from direction for isAuthEncryptOp 735 * cases */ 736 /* For CCM & GCM modes: force digest verify flag _TRUE 737 for decrypt and _FALSE for encrypt. For all other 738 cases use user defined value */ 739 740 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 741 pSessionDesc->cipherDirection) { 742 *chainOrder = 743 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 744 pSessionDesc->digestVerify = CPA_FALSE; 745 } else { 746 *chainOrder = 747 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 748 if (CPA_TRUE == pService->forceAEADMacVerify) { 749 pSessionDesc->digestVerify = CPA_TRUE; 750 } 751 } 752 } else if (LAC_CIPHER_IS_GCM(pSessionDesc->cipherAlgorithm)) { 753 *proto = ICP_QAT_FW_LA_GCM_PROTO; 754 755 /* Derive chainOrder from direction for isAuthEncryptOp 756 * cases */ 757 /* For CCM & GCM modes: force digest verify flag _TRUE 758 for decrypt and _FALSE for encrypt. For all other 759 cases use user defined value */ 760 761 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 762 pSessionDesc->cipherDirection) { 763 *chainOrder = 764 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 765 pSessionDesc->digestVerify = CPA_FALSE; 766 } else { 767 *chainOrder = 768 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 769 if (CPA_TRUE == pService->forceAEADMacVerify) { 770 pSessionDesc->digestVerify = CPA_TRUE; 771 } 772 } 773 } else if (LAC_CIPHER_IS_CHACHA( 774 pSessionDesc->cipherAlgorithm)) { 775 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 776 pSessionDesc->cipherDirection) { 777 *chainOrder = 778 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 779 } else { 780 *chainOrder = 781 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 782 } 783 } else { 784 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 785 786 if (CPA_CY_SYM_CIPHER_SNOW3G_UEA2 == 787 pSessionDesc->cipherAlgorithm) { 788 *proto = ICP_QAT_FW_LA_SNOW_3G_PROTO; 789 } else if (CPA_CY_SYM_CIPHER_ZUC_EEA3 == 790 pSessionDesc->cipherAlgorithm) { 791 *proto = ICP_QAT_FW_LA_ZUC_3G_PROTO; 792 } 793 794 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 795 pSessionDesc->hashAlgorithm) { 796 /* Need to set LW 28 hash flags as well. */ 797 ICP_QAT_FW_HASH_FLAG_SNOW3G_UIA2_SET( 798 cd_ctrl->hash_flags, QAT_FW_LA_SNOW3G_UIA2); 799 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == 800 pSessionDesc->hashAlgorithm) { 801 /* Need to set LW 28 hash flags as well. */ 802 ICP_QAT_FW_HASH_FLAG_ZUC_EIA3_SET( 803 cd_ctrl->hash_flags, QAT_FW_LA_ZUC_EIA3); 804 } 805 } 806 807 if (CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH == 808 *chainOrder) { 809 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; 810 } else if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER == 811 *chainOrder) { 812 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; 813 } 814 break; 815 816 default: 817 break; 818 } 819 820 /* 821 * Build the header flags with the default settings for this session. 822 */ 823 if (pSessionDesc->isDPSession == CPA_TRUE) { 824 *cmnRequestFlags = 825 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 826 LAC_SYM_DP_QAT_PTR_TYPE); 827 } else { 828 *cmnRequestFlags = 829 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 830 LAC_SYM_DEFAULT_QAT_PTR_TYPE); 831 } 832 833 LacSymQat_LaSetDefaultFlags(laCmdFlags, pSessionDesc->symOperation); 834 835 return; 836 } 837 838 static void 839 updateLaCmdFlags(lac_session_desc_t *pSessionDesc, 840 Cpa16U proto, 841 icp_qat_fw_serv_specif_flags *laCmdFlags) 842 { 843 if (pSessionDesc->isAuth) { 844 if (pSessionDesc->digestVerify) { 845 ICP_QAT_FW_LA_CMP_AUTH_SET(*laCmdFlags, 846 ICP_QAT_FW_LA_CMP_AUTH_RES); 847 ICP_QAT_FW_LA_RET_AUTH_SET( 848 *laCmdFlags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); 849 } else { 850 ICP_QAT_FW_LA_RET_AUTH_SET(*laCmdFlags, 851 ICP_QAT_FW_LA_RET_AUTH_RES); 852 ICP_QAT_FW_LA_CMP_AUTH_SET( 853 *laCmdFlags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES); 854 } 855 } 856 857 if ((CPA_CY_SYM_CIPHER_ZUC_EEA3 == pSessionDesc->cipherAlgorithm) || 858 (CPA_CY_SYM_HASH_ZUC_EIA3 == pSessionDesc->hashAlgorithm)) { 859 /* New bit position (12) for ZUC. The FW provides a specific 860 * macro to use to set the ZUC proto flag. With the new FW I/F 861 * this needs to be set for both Cipher and Auth */ 862 ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(*laCmdFlags, proto); 863 } else { 864 /* Configure the common header */ 865 ICP_QAT_FW_LA_PROTO_SET(*laCmdFlags, proto); 866 } 867 868 /* set Append flag, if digest is appended */ 869 if (pSessionDesc->digestIsAppended) { 870 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( 871 *laCmdFlags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); 872 } else { 873 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( 874 *laCmdFlags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); 875 } 876 } 877 878 static lac_single_pass_state_t 879 LacSymAlgChain_GetSpcState(CpaCySymCipherAlgorithm cipher, 880 CpaCySymHashAlgorithm hash, 881 Cpa32U capabilitiesMask) 882 { 883 lac_single_pass_state_t state = NON_SPC; 884 if (capabilitiesMask & ICP_ACCEL_CAPABILITIES_CHACHA_POLY) { 885 switch (cipher) { 886 case CPA_CY_SYM_CIPHER_CHACHA: { 887 if (CPA_CY_SYM_HASH_POLY == hash) 888 state = SPC; 889 break; 890 } 891 case CPA_CY_SYM_CIPHER_AES_GCM: { 892 if ((CPA_CY_SYM_HASH_AES_GCM == hash) || 893 (CPA_CY_SYM_HASH_AES_GMAC == hash)) 894 state = LIKELY_SPC; 895 break; 896 } 897 case CPA_CY_SYM_CIPHER_AES_CCM: { 898 if (LAC_CIPHER_AES_V2(capabilitiesMask)) 899 state = SPC; 900 } 901 default: 902 /* Do Nothing as it is NON_SPC */ 903 break; 904 } 905 } 906 return state; 907 } 908 909 static CpaBoolean 910 LacAlgChain_UseStatefulSha3ContentDesc(CpaBoolean partialsNotRequired, 911 Cpa32U capabilitiesMask, 912 lac_session_desc_t *pSessionDesc) 913 { 914 CpaBoolean hasSha3Ext = 915 ICP_ACCEL_CAPABILITIES_SHA3_EXT & capabilitiesMask; 916 CpaBoolean useStatefulSha3DescFlag = CPA_FALSE; 917 918 if (hasSha3Ext && !partialsNotRequired && 919 (pSessionDesc->symOperation == CPA_CY_SYM_OP_HASH) && 920 LAC_HASH_IS_SHA3(pSessionDesc->hashAlgorithm)) { 921 useStatefulSha3DescFlag = CPA_TRUE; 922 } 923 return useStatefulSha3DescFlag; 924 } 925 926 /** @ingroup LacAlgChain */ 927 CpaStatus 928 LacAlgChain_SessionInit(const CpaInstanceHandle instanceHandle, 929 const CpaCySymSessionSetupData *pSessionSetupData, 930 lac_session_desc_t *pSessionDesc) 931 { 932 CpaStatus stat, status = CPA_STATUS_SUCCESS; 933 sal_qat_content_desc_info_t *pCdInfo = NULL; 934 sal_qat_content_desc_info_t *pCdInfoOptimised = NULL; 935 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; 936 Cpa32U capabilitiesMask = 937 pService->generic_service_info.capabilitiesMask; 938 Cpa8U *pHwBlockBaseInDRAM = NULL; 939 Cpa8U *pOptimisedHwBlockBaseInDRAM = NULL; 940 Cpa32U hwBlockOffsetInDRAM = 0; 941 Cpa32U optimisedHwBlockOffsetInDRAM = 0; 942 Cpa8U cipherOffsetInConstantsTable = 0; 943 Cpa8U hashOffsetInConstantsTable = 0; 944 icp_qat_fw_comn_flags cmnRequestFlags = 0; 945 icp_qat_fw_comn_req_t *pMsg = NULL; 946 icp_qat_fw_comn_req_t *pMsgS = NULL; 947 const CpaCySymCipherSetupData *pCipherData; 948 const CpaCySymHashSetupData *pHashData; 949 Cpa16U proto = ICP_QAT_FW_LA_NO_PROTO; /* no CCM/GCM/Snow3G */ 950 CpaCySymAlgChainOrder chainOrder = 0; 951 lac_sym_qat_hash_precompute_info_t precomputeData = { 0 }; 952 lac_sym_qat_hash_precompute_info_t precomputeDataOptimisedCd = { 0 }; 953 954 pCipherData = &(pSessionSetupData->cipherSetupData); 955 pHashData = &(pSessionSetupData->hashSetupData); 956 957 /*------------------------------------------------------------------------- 958 * Populate session data 959 *-----------------------------------------------------------------------*/ 960 961 /* Initialise Request Queue */ 962 stat = LAC_SPINLOCK_INIT(&pSessionDesc->requestQueueLock); 963 if (CPA_STATUS_SUCCESS != stat) { 964 LAC_LOG_ERROR("Spinlock init failed for sessionLock"); 965 return CPA_STATUS_RESOURCE; 966 } 967 968 pSessionDesc->pRequestQueueHead = NULL; 969 pSessionDesc->pRequestQueueTail = NULL; 970 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE; 971 pSessionDesc->pInstance = instanceHandle; 972 pSessionDesc->digestIsAppended = pSessionSetupData->digestIsAppended; 973 pSessionDesc->digestVerify = pSessionSetupData->verifyDigest; 974 975 /* Reset the pending callback counter */ 976 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingCbCount); 977 qatUtilsAtomicSet(0, &pSessionDesc->u.pendingDpCbCount); 978 979 /* Partial state must be set to full, to indicate that next packet 980 * expected on the session is a full packet or the start of a 981 * partial packet. */ 982 pSessionDesc->partialState = CPA_CY_SYM_PACKET_TYPE_FULL; 983 984 pSessionDesc->symOperation = pSessionSetupData->symOperation; 985 switch (pSessionDesc->symOperation) { 986 case CPA_CY_SYM_OP_CIPHER: 987 pSessionDesc->isCipher = CPA_TRUE; 988 pSessionDesc->isAuth = CPA_FALSE; 989 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 990 pSessionDesc->singlePassState = NON_SPC; 991 break; 992 case CPA_CY_SYM_OP_HASH: 993 pSessionDesc->isCipher = CPA_FALSE; 994 pSessionDesc->isAuth = CPA_TRUE; 995 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 996 pSessionDesc->singlePassState = NON_SPC; 997 break; 998 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: { 999 pSessionDesc->isCipher = CPA_TRUE; 1000 pSessionDesc->isAuth = CPA_TRUE; 1001 pSessionDesc->singlePassState = 1002 LacSymAlgChain_GetSpcState(pCipherData->cipherAlgorithm, 1003 pHashData->hashAlgorithm, 1004 capabilitiesMask); 1005 1006 switch (pSessionSetupData->cipherSetupData.cipherAlgorithm) { 1007 case CPA_CY_SYM_CIPHER_AES_CCM: { 1008 pSessionDesc->isAuthEncryptOp = CPA_TRUE; 1009 pSessionDesc->digestIsAppended = CPA_TRUE; 1010 } break; 1011 case CPA_CY_SYM_CIPHER_AES_GCM: 1012 case CPA_CY_SYM_CIPHER_CHACHA: 1013 pSessionDesc->isAuthEncryptOp = CPA_TRUE; 1014 break; 1015 default: { 1016 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 1017 /* Use the chainOrder passed in */ 1018 chainOrder = pSessionSetupData->algChainOrder; 1019 if ((chainOrder != 1020 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER) && 1021 (chainOrder != 1022 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH)) { 1023 LAC_INVALID_PARAM_LOG("algChainOrder"); 1024 return CPA_STATUS_INVALID_PARAM; 1025 } 1026 } break; 1027 } 1028 } break; 1029 default: 1030 pSessionDesc->singlePassState = NON_SPC; 1031 break; 1032 } 1033 1034 if (pSessionDesc->isCipher) { 1035 /* Populate cipher specific session data */ 1036 1037 status = LacCipher_SessionSetupDataCheck(pCipherData, 1038 capabilitiesMask); 1039 1040 if (CPA_STATUS_SUCCESS == status) { 1041 pSessionDesc->cipherAlgorithm = 1042 pCipherData->cipherAlgorithm; 1043 pSessionDesc->cipherKeyLenInBytes = 1044 pCipherData->cipherKeyLenInBytes; 1045 pSessionDesc->cipherDirection = 1046 pCipherData->cipherDirection; 1047 1048 /* ARC4 base key isn't added to the content descriptor, 1049 * because we don't need to pass it directly to the QAT 1050 * engine. Instead an initial cipher state & key matrix 1051 * is derived from the base key and provided to the QAT 1052 * through the state pointer in the request params. 1053 * We'll store this initial state in the session 1054 * descriptor. */ 1055 1056 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { 1057 LacSymQat_CipherArc4StateInit( 1058 pCipherData->pCipherKey, 1059 pSessionDesc->cipherKeyLenInBytes, 1060 pSessionDesc->cipherARC4InitialState); 1061 1062 pSessionDesc->cipherARC4InitialStatePhysAddr = 1063 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 1064 pService->generic_service_info, 1065 pSessionDesc->cipherARC4InitialState); 1066 1067 if (0 == 1068 pSessionDesc 1069 ->cipherARC4InitialStatePhysAddr) { 1070 LAC_LOG_ERROR( 1071 "Unable to get the physical address of " 1072 "the initial state for ARC4\n"); 1073 status = CPA_STATUS_FAIL; 1074 } 1075 } 1076 } 1077 } 1078 1079 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) { 1080 /* Populate auth-specific session data */ 1081 const CpaCySymHashSetupData *pHashData = 1082 &pSessionSetupData->hashSetupData; 1083 1084 status = LacHash_HashContextCheck(instanceHandle, pHashData); 1085 if (CPA_STATUS_SUCCESS == status) { 1086 pSessionDesc->hashResultSize = 1087 pHashData->digestResultLenInBytes; 1088 pSessionDesc->hashMode = pHashData->hashMode; 1089 pSessionDesc->hashAlgorithm = pHashData->hashAlgorithm; 1090 1091 /* Save the authentication key length for further update 1092 */ 1093 if (CPA_CY_SYM_HASH_MODE_AUTH == pHashData->hashMode) { 1094 pSessionDesc->authKeyLenInBytes = 1095 pHashData->authModeSetupData 1096 .authKeyLenInBytes; 1097 } 1098 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp || 1099 (pHashData->hashAlgorithm == 1100 CPA_CY_SYM_HASH_SNOW3G_UIA2 || 1101 pHashData->hashAlgorithm == 1102 CPA_CY_SYM_HASH_ZUC_EIA3)) { 1103 pSessionDesc->aadLenInBytes = 1104 pHashData->authModeSetupData.aadLenInBytes; 1105 } 1106 1107 /* Set the QAT hash mode */ 1108 if ((pHashData->hashMode == 1109 CPA_CY_SYM_HASH_MODE_NESTED) || 1110 (pHashData->hashMode == 1111 CPA_CY_SYM_HASH_MODE_PLAIN) || 1112 (pHashData->hashMode == CPA_CY_SYM_HASH_MODE_AUTH && 1113 pHashData->hashAlgorithm == 1114 CPA_CY_SYM_HASH_AES_CBC_MAC)) { 1115 pSessionDesc->qatHashMode = 1116 ICP_QAT_HW_AUTH_MODE0; 1117 } else /* CPA_CY_SYM_HASH_MODE_AUTH 1118 && anything except CPA_CY_SYM_HASH_AES_CBC_MAC 1119 */ 1120 { 1121 if (IS_HMAC_ALG(pHashData->hashAlgorithm)) { 1122 /* SHA3 HMAC and SM3 do not support 1123 * precompute, force MODE2 for AUTH */ 1124 if (LAC_HASH_IS_SHA3( 1125 pHashData->hashAlgorithm) || 1126 (CPA_CY_SYM_HASH_SM3 == 1127 pHashData->hashAlgorithm)) { 1128 pSessionDesc->qatHashMode = 1129 ICP_QAT_HW_AUTH_MODE2; 1130 } else { 1131 pSessionDesc->qatHashMode = 1132 pService->qatHmacMode; 1133 } 1134 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == 1135 pHashData->hashAlgorithm) { 1136 pSessionDesc->qatHashMode = 1137 ICP_QAT_HW_AUTH_MODE0; 1138 } else { 1139 pSessionDesc->qatHashMode = 1140 ICP_QAT_HW_AUTH_MODE1; 1141 } 1142 } 1143 } 1144 } 1145 1146 /*------------------------------------------------------------------------- 1147 * build the message templates 1148 * create two content descriptors in the case we can support using SHRAM 1149 * constants and an optimised content descriptor. we have to do this in 1150 * case of partials. 64 byte content descriptor is used in the SHRAM 1151 * case for AES-128-HMAC-SHA1 1152 *-----------------------------------------------------------------------*/ 1153 if (CPA_STATUS_SUCCESS == status) { 1154 pSessionDesc->cipherSliceType = 1155 LacCipher_GetCipherSliceType(pService, 1156 pSessionDesc->cipherAlgorithm, 1157 pSessionDesc->hashAlgorithm); 1158 1159 LacSymCheck_IsPartialSupported(capabilitiesMask, pSessionDesc); 1160 pSessionDesc->useOptimisedContentDesc = CPA_FALSE; 1161 pSessionDesc->useStatefulSha3ContentDesc = CPA_FALSE; 1162 1163 /* Build configuration data */ 1164 buildCmdData(pService, 1165 pSessionDesc, 1166 &chainOrder, 1167 &proto, 1168 &pSessionDesc->laCmdFlags, 1169 &cmnRequestFlags); 1170 1171 if (ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE == 1172 pSessionDesc->cipherSliceType) 1173 pSessionDesc->useSymConstantsTable = CPA_FALSE; 1174 else 1175 pSessionDesc->useSymConstantsTable = 1176 LacSymQat_UseSymConstantsTable( 1177 pSessionDesc, 1178 &cipherOffsetInConstantsTable, 1179 &hashOffsetInConstantsTable); 1180 1181 /* for a certain combination of Algorthm Chaining we want to 1182 use an optimised cd block */ 1183 1184 if (pSessionDesc->symOperation == 1185 CPA_CY_SYM_OP_ALGORITHM_CHAINING && 1186 pSessionDesc->useSymConstantsTable == CPA_TRUE) { 1187 pSessionDesc->useOptimisedContentDesc = 1188 LacSymQat_UseOptimisedContentDesc(pSessionDesc); 1189 } 1190 1191 /* check whether we need to construct content desc for stateful 1192 * SHA3 */ 1193 pSessionDesc->useStatefulSha3ContentDesc = 1194 LacAlgChain_UseStatefulSha3ContentDesc( 1195 pSessionSetupData->partialsNotRequired, 1196 capabilitiesMask, 1197 pSessionDesc); 1198 1199 /* setup some convenience pointers */ 1200 pCdInfo = &(pSessionDesc->contentDescInfo); 1201 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 1202 hwBlockOffsetInDRAM = 0; 1203 1204 /* set up the pointer for the optimised content desc if this is 1205 * possible we still have to support both cd types in case of 1206 * partials so we construct both */ 1207 if (pSessionDesc->useOptimisedContentDesc == CPA_TRUE) { 1208 pCdInfoOptimised = 1209 &(pSessionDesc->contentDescOptimisedInfo); 1210 pOptimisedHwBlockBaseInDRAM = 1211 (Cpa8U *)pCdInfoOptimised->pData; 1212 optimisedHwBlockOffsetInDRAM = 0; 1213 } 1214 1215 switch (pSessionDesc->symOperation) { 1216 case CPA_CY_SYM_OP_CIPHER: { 1217 LacAlgChain_CipherCDBuild( 1218 pCipherData, 1219 pSessionDesc, 1220 ICP_QAT_FW_SLICE_DRAM_WR, 1221 cipherOffsetInConstantsTable, 1222 &pSessionDesc->cmnRequestFlags, 1223 &pSessionDesc->laCmdFlags, 1224 pHwBlockBaseInDRAM, 1225 &hwBlockOffsetInDRAM, 1226 capabilitiesMask); 1227 1228 if (pSessionDesc->useSymConstantsTable) { 1229 LacAlgChain_CipherCDBuild_ForSHRAM( 1230 pCipherData, 1231 pSessionDesc, 1232 ICP_QAT_FW_SLICE_DRAM_WR, 1233 cipherOffsetInConstantsTable); 1234 } 1235 } break; 1236 case CPA_CY_SYM_OP_HASH: 1237 LacAlgChain_HashCDBuild(pHashData, 1238 instanceHandle, 1239 pSessionDesc, 1240 ICP_QAT_FW_SLICE_NULL, 1241 hashOffsetInConstantsTable, 1242 &pSessionDesc->cmnRequestFlags, 1243 &pSessionDesc->laCmdFlags, 1244 &precomputeData, 1245 &precomputeDataOptimisedCd, 1246 pHwBlockBaseInDRAM, 1247 &hwBlockOffsetInDRAM, 1248 NULL, 1249 NULL); 1250 break; 1251 case CPA_CY_SYM_OP_ALGORITHM_CHAINING: 1252 /* For CCM/GCM, CPM firmware currently expects the 1253 * cipher and hash h/w setup blocks to be arranged 1254 * according to the chain order (Except for GCM/CCM, 1255 * order doesn't actually matter as long as the config 1256 * offsets are set correctly in CD control blocks 1257 */ 1258 if (CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER == 1259 chainOrder) { 1260 LacAlgChain_HashCDBuild( 1261 pHashData, 1262 instanceHandle, 1263 pSessionDesc, 1264 ICP_QAT_FW_SLICE_CIPHER, 1265 hashOffsetInConstantsTable, 1266 &pSessionDesc->cmnRequestFlags, 1267 &pSessionDesc->laCmdFlags, 1268 &precomputeData, 1269 &precomputeDataOptimisedCd, 1270 pHwBlockBaseInDRAM, 1271 &hwBlockOffsetInDRAM, 1272 pOptimisedHwBlockBaseInDRAM, 1273 &optimisedHwBlockOffsetInDRAM); 1274 1275 LacAlgChain_CipherCDBuild( 1276 pCipherData, 1277 pSessionDesc, 1278 ICP_QAT_FW_SLICE_DRAM_WR, 1279 cipherOffsetInConstantsTable, 1280 &pSessionDesc->cmnRequestFlags, 1281 &pSessionDesc->laCmdFlags, 1282 pHwBlockBaseInDRAM, 1283 &hwBlockOffsetInDRAM, 1284 capabilitiesMask); 1285 1286 if (pSessionDesc->useOptimisedContentDesc) { 1287 LacAlgChain_CipherCDBuild_ForOptimisedCD( 1288 pCipherData, 1289 pSessionDesc, 1290 ICP_QAT_FW_SLICE_DRAM_WR, 1291 cipherOffsetInConstantsTable, 1292 pOptimisedHwBlockBaseInDRAM, 1293 &optimisedHwBlockOffsetInDRAM); 1294 } 1295 1296 if (NON_SPC != pSessionDesc->singlePassState) { 1297 pCdInfo->hwBlkSzQuadWords = 1298 (LAC_BYTES_TO_QUADWORDS( 1299 hwBlockOffsetInDRAM)); 1300 pMsg = (icp_qat_fw_comn_req_t *)&( 1301 pSessionDesc->reqSpcCacheHdr); 1302 SalQatMsg_ContentDescHdrWrite( 1303 (icp_qat_fw_comn_req_t *)pMsg, 1304 pCdInfo); 1305 } 1306 } else /* CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH */ 1307 { 1308 LacAlgChain_CipherCDBuild( 1309 pCipherData, 1310 pSessionDesc, 1311 ICP_QAT_FW_SLICE_AUTH, 1312 cipherOffsetInConstantsTable, 1313 &pSessionDesc->cmnRequestFlags, 1314 &pSessionDesc->laCmdFlags, 1315 pHwBlockBaseInDRAM, 1316 &hwBlockOffsetInDRAM, 1317 capabilitiesMask); 1318 1319 if (pSessionDesc->useOptimisedContentDesc) { 1320 LacAlgChain_CipherCDBuild_ForOptimisedCD( 1321 pCipherData, 1322 pSessionDesc, 1323 ICP_QAT_FW_SLICE_AUTH, 1324 cipherOffsetInConstantsTable, 1325 pOptimisedHwBlockBaseInDRAM, 1326 &optimisedHwBlockOffsetInDRAM); 1327 } 1328 1329 if (NON_SPC != pSessionDesc->singlePassState) { 1330 pCdInfo->hwBlkSzQuadWords = 1331 LAC_BYTES_TO_QUADWORDS( 1332 hwBlockOffsetInDRAM); 1333 pMsg = (icp_qat_fw_comn_req_t *)&( 1334 pSessionDesc->reqSpcCacheHdr); 1335 SalQatMsg_ContentDescHdrWrite( 1336 (icp_qat_fw_comn_req_t *)pMsg, 1337 pCdInfo); 1338 } 1339 LacAlgChain_HashCDBuild( 1340 pHashData, 1341 instanceHandle, 1342 pSessionDesc, 1343 ICP_QAT_FW_SLICE_DRAM_WR, 1344 hashOffsetInConstantsTable, 1345 &pSessionDesc->cmnRequestFlags, 1346 &pSessionDesc->laCmdFlags, 1347 &precomputeData, 1348 &precomputeDataOptimisedCd, 1349 pHwBlockBaseInDRAM, 1350 &hwBlockOffsetInDRAM, 1351 pOptimisedHwBlockBaseInDRAM, 1352 &optimisedHwBlockOffsetInDRAM); 1353 } 1354 break; 1355 default: 1356 LAC_LOG_ERROR("Invalid sym operation\n"); 1357 status = CPA_STATUS_INVALID_PARAM; 1358 } 1359 } 1360 1361 if ((CPA_STATUS_SUCCESS == status) && pSessionDesc->isAuth) { 1362 lac_sym_qat_hash_state_buffer_info_t *pHashStateBufferInfo = 1363 &(pSessionDesc->hashStateBufferInfo); 1364 CpaBoolean hashStateBuffer = CPA_TRUE; 1365 1366 /* set up fields in both the cd_ctrl and reqParams which 1367 * describe the ReqParams block */ 1368 LacSymQat_HashSetupReqParamsMetaData( 1369 &(pSessionDesc->reqCacheFtr), 1370 instanceHandle, 1371 pHashData, 1372 hashStateBuffer, 1373 pSessionDesc->qatHashMode, 1374 pSessionDesc->digestVerify); 1375 1376 if (pSessionDesc->useSymConstantsTable) { 1377 /* Need to set up for SHRAM Constants Table use also */ 1378 LacSymQat_HashSetupReqParamsMetaData( 1379 &(pSessionDesc->shramReqCacheFtr), 1380 instanceHandle, 1381 pHashData, 1382 hashStateBuffer, 1383 pSessionDesc->qatHashMode, 1384 pSessionDesc->digestVerify); 1385 } 1386 1387 /* populate the hash state prefix buffer info structure 1388 * (part of user allocated session memory & the 1389 * buffer itself. For CCM/GCM the buffer is stored in the 1390 * cookie and is not initialised here) */ 1391 if (CPA_FALSE == pSessionDesc->isAuthEncryptOp) { 1392 LAC_CHECK_64_BYTE_ALIGNMENT( 1393 &(pSessionDesc->hashStatePrefixBuffer[0])); 1394 status = LacHash_StatePrefixAadBufferInit( 1395 &(pService->generic_service_info), 1396 pHashData, 1397 &(pSessionDesc->reqCacheFtr), 1398 pSessionDesc->qatHashMode, 1399 pSessionDesc->hashStatePrefixBuffer, 1400 pHashStateBufferInfo); 1401 /* SHRAM Constants Table not used for Auth-Enc */ 1402 } 1403 1404 if (CPA_STATUS_SUCCESS == status) { 1405 if (IS_HASH_MODE_1(pSessionDesc->qatHashMode) || 1406 CPA_CY_SYM_HASH_ZUC_EIA3 == 1407 pHashData->hashAlgorithm) { 1408 LAC_CHECK_64_BYTE_ALIGNMENT( 1409 &(pSessionDesc->hashStatePrefixBuffer[0])); 1410 1411 /* Block messages until precompute is completed 1412 */ 1413 pSessionDesc->nonBlockingOpsInProgress = 1414 CPA_FALSE; 1415 status = LacHash_PrecomputeDataCreate( 1416 instanceHandle, 1417 (CpaCySymSessionSetupData *) 1418 pSessionSetupData, 1419 LacSymAlgChain_HashPrecomputeDoneCb, 1420 pSessionDesc, 1421 pSessionDesc->hashStatePrefixBuffer, 1422 precomputeData.pState1, 1423 precomputeData.pState2); 1424 if (pSessionDesc->useOptimisedContentDesc) { 1425 status = LacHash_PrecomputeDataCreate( 1426 instanceHandle, 1427 (CpaCySymSessionSetupData *) 1428 pSessionSetupData, 1429 LacSymAlgChain_HashPrecomputeDoneCb, 1430 pSessionDesc, 1431 pSessionDesc->hashStatePrefixBuffer, 1432 precomputeDataOptimisedCd.pState1, 1433 precomputeDataOptimisedCd.pState2); 1434 } 1435 } else if (pHashData->hashAlgorithm == 1436 CPA_CY_SYM_HASH_AES_CBC_MAC) { 1437 if (NULL != precomputeData.pState2) { 1438 LAC_OS_BZERO(precomputeData.pState2, 1439 precomputeData.state2Size); 1440 memcpy(precomputeData.pState2, 1441 pHashData->authModeSetupData 1442 .authKey, 1443 pHashData->authModeSetupData 1444 .authKeyLenInBytes); 1445 } 1446 } 1447 } 1448 } 1449 1450 if (CPA_STATUS_SUCCESS == status) { 1451 1452 /* Configure the ContentDescriptor field 1453 in the request if not done already */ 1454 pCdInfo->hwBlkSzQuadWords = 1455 LAC_BYTES_TO_QUADWORDS(hwBlockOffsetInDRAM); 1456 pMsg = (icp_qat_fw_comn_req_t *)&(pSessionDesc->reqCacheHdr); 1457 SalQatMsg_ContentDescHdrWrite((icp_qat_fw_comn_req_t *)pMsg, 1458 pCdInfo); 1459 1460 pMsgS = 1461 (icp_qat_fw_comn_req_t *)&(pSessionDesc->shramReqCacheHdr); 1462 /*If we are using the optimised CD then 1463 we have to set this up correctly in the SHARM reqCache*/ 1464 if (pSessionDesc->useOptimisedContentDesc) { 1465 pCdInfoOptimised->hwBlkSzQuadWords = 1466 LAC_BYTES_TO_QUADWORDS( 1467 optimisedHwBlockOffsetInDRAM); 1468 SalQatMsg_ContentDescHdrWrite( 1469 (icp_qat_fw_comn_req_t *)pMsgS, pCdInfoOptimised); 1470 } 1471 1472 /* Updates command flags basing on configured alg */ 1473 updateLaCmdFlags(pSessionDesc, 1474 proto, 1475 &pSessionDesc->laCmdFlags); 1476 1477 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsg, 1478 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 1479 pSessionDesc->laCmdId, 1480 pSessionDesc->cmnRequestFlags, 1481 pSessionDesc->laCmdFlags); 1482 1483 /* Need to duplicate if SHRAM Constants Table used */ 1484 if (pSessionDesc->useSymConstantsTable) { 1485 ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET( 1486 pSessionDesc->laCmdFlags, 1487 ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP); 1488 1489 if (pSessionDesc->isCipher && 1490 !pSessionDesc->useOptimisedContentDesc) { 1491 ICP_QAT_FW_COMN_CD_FLD_TYPE_SET( 1492 cmnRequestFlags, 1493 QAT_COMN_CD_FLD_TYPE_16BYTE_DATA); 1494 } 1495 1496 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)pMsgS, 1497 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 1498 pSessionDesc->laCmdId, 1499 cmnRequestFlags, 1500 pSessionDesc->laCmdFlags); 1501 } 1502 } 1503 1504 return status; 1505 } 1506 1507 static void 1508 LacAlgChain_StatefulSha3_SkipStateLoadFlags(icp_qat_fw_la_bulk_req_t *pMsg, 1509 Cpa32U packetType, 1510 icp_qat_hw_auth_mode_t qatHashMode) 1511 { 1512 icp_qat_fw_auth_cd_ctrl_hdr_t *pAuthCdCtrlHdr = NULL; 1513 1514 pAuthCdCtrlHdr = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 1515 1516 if (IS_HASH_MODE_2(qatHashMode)) { 1517 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || 1518 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { 1519 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( 1520 pAuthCdCtrlHdr->hash_flags, 1521 QAT_FW_LA_SKIP_INNER_STATE1_LOAD); 1522 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( 1523 pAuthCdCtrlHdr->hash_flags, 1524 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); 1525 } else if (ICP_QAT_FW_LA_PARTIAL_END == packetType) { 1526 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( 1527 pAuthCdCtrlHdr->hash_flags, 1528 QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); 1529 } 1530 } else { 1531 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || 1532 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { 1533 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( 1534 pAuthCdCtrlHdr->hash_flags, 1535 QAT_FW_LA_SKIP_INNER_STATE1_LOAD); 1536 } 1537 } 1538 } 1539 1540 /** @ingroup LacAlgChain */ 1541 CpaStatus 1542 LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, 1543 lac_session_desc_t *pSessionDesc, 1544 void *pCallbackTag, 1545 const CpaCySymOpData *pOpData, 1546 const CpaBufferList *pSrcBuffer, 1547 CpaBufferList *pDstBuffer, 1548 CpaBoolean *pVerifyResult) 1549 { 1550 CpaStatus status = CPA_STATUS_SUCCESS; 1551 sal_crypto_service_t *pService = (sal_crypto_service_t *)instanceHandle; 1552 Cpa32U capabilitiesMask = 1553 pService->generic_service_info.capabilitiesMask; 1554 lac_sym_bulk_cookie_t *pCookie = NULL; 1555 lac_sym_cookie_t *pSymCookie = NULL; 1556 icp_qat_fw_la_bulk_req_t *pMsg = NULL; 1557 Cpa8U *pMsgDummy = NULL; 1558 Cpa8U *pCacheDummyHdr = NULL; 1559 Cpa8U *pCacheDummyFtr = NULL; 1560 Cpa32U qatPacketType = 0; 1561 CpaBufferList *pBufferList = NULL; 1562 Cpa8U *pDigestResult = NULL; 1563 Cpa64U srcAddrPhys = 0; 1564 Cpa64U dstAddrPhys = 0; 1565 icp_qat_fw_la_cmd_id_t laCmdId; 1566 sal_qat_content_desc_info_t *pCdInfo = NULL; 1567 Cpa8U *pHwBlockBaseInDRAM = NULL; 1568 Cpa32U hwBlockOffsetInDRAM = 0; 1569 Cpa32U sizeInBytes = 0; 1570 icp_qat_fw_cipher_cd_ctrl_hdr_t *pSpcCdCtrlHdr = NULL; 1571 CpaCySymCipherAlgorithm cipher; 1572 CpaCySymHashAlgorithm hash; 1573 Cpa8U paddingLen = 0; 1574 Cpa8U blockLen = 0; 1575 CpaBoolean digestIsAppended = CPA_FALSE; 1576 Cpa32U aadLenInBytes = 0; 1577 Cpa64U srcPktSize = 0; 1578 Cpa64U dstPktSize = 0; 1579 1580 /* Set the command id */ 1581 laCmdId = pSessionDesc->laCmdId; 1582 1583 cipher = pSessionDesc->cipherAlgorithm; 1584 hash = pSessionDesc->hashAlgorithm; 1585 1586 CpaBoolean isSpCcm = 1587 (LAC_CIPHER_IS_CCM(cipher) && LAC_CIPHER_AES_V2(capabilitiesMask)); 1588 1589 if (CPA_CY_SYM_HASH_AES_GMAC == hash) { 1590 pSessionDesc->aadLenInBytes = pOpData->messageLenToHashInBytes; 1591 if (pOpData->messageLenToHashInBytes == 0 || 1592 pOpData->pAdditionalAuthData != NULL) { 1593 LAC_INVALID_PARAM_LOG( 1594 "For AES_GMAC, AAD Length " 1595 "(messageLenToHashInBytes) must " 1596 "be non zero and pAdditionalAuthData " 1597 "must be NULL"); 1598 return CPA_STATUS_INVALID_PARAM; 1599 } 1600 } 1601 1602 aadLenInBytes = pSessionDesc->aadLenInBytes; 1603 1604 /* Convert Alg Chain Request to Cipher Request for CCP and 1605 * AES_GCM single pass */ 1606 if ((NON_SPC != pSessionDesc->singlePassState) && 1607 (isSpCcm || (LAC_CIPHER_SPC_IV_SIZE == pOpData->ivLenInBytes))) { 1608 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER; 1609 laCmdId = pSessionDesc->laCmdId; 1610 pSessionDesc->symOperation = CPA_CY_SYM_OP_CIPHER; 1611 pSessionDesc->singlePassState = SPC; 1612 pSessionDesc->isCipher = CPA_TRUE; 1613 pSessionDesc->isAuthEncryptOp = CPA_FALSE; 1614 pSessionDesc->isAuth = CPA_FALSE; 1615 1616 if (CPA_CY_SYM_HASH_AES_GMAC == hash) { 1617 if (ICP_QAT_FW_SPC_AAD_SZ_MAX < aadLenInBytes) { 1618 LAC_INVALID_PARAM_LOG( 1619 "aadLenInBytes for AES_GMAC"); 1620 return CPA_STATUS_INVALID_PARAM; 1621 } 1622 } 1623 /* New bit position (13) for SINGLE PASS. 1624 * The FW provides a specific macro to use to set the proto flag 1625 */ 1626 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 1627 pSessionDesc->laCmdFlags, ICP_QAT_FW_LA_SINGLE_PASS_PROTO); 1628 if (isCyGen2x(pService)) { 1629 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 0); 1630 } 1631 1632 pCdInfo = &(pSessionDesc->contentDescInfo); 1633 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 1634 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == 1635 pSessionDesc->cipherDirection) { 1636 if (LAC_CIPHER_IS_GCM(cipher)) 1637 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 1638 LAC_SYM_QAT_CIPHER_GCM_SPC_OFFSET_IN_DRAM); 1639 else if (LAC_CIPHER_IS_CHACHA(cipher)) 1640 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 1641 LAC_SYM_QAT_CIPHER_CHACHA_SPC_OFFSET_IN_DRAM); 1642 } else if (isSpCcm) { 1643 hwBlockOffsetInDRAM = LAC_QUADWORDS_TO_BYTES( 1644 LAC_SYM_QAT_CIPHER_CCM_SPC_OFFSET_IN_DRAM); 1645 } 1646 1647 /* Update cipher slice type */ 1648 pSessionDesc->cipherSliceType = 1649 LacCipher_GetCipherSliceType(pService, 1650 pSessionDesc->cipherAlgorithm, 1651 pSessionDesc->hashAlgorithm); 1652 1653 ICP_QAT_FW_LA_SLICE_TYPE_SET(pSessionDesc->laCmdFlags, 1654 pSessionDesc->cipherSliceType); 1655 1656 /* construct cipherConfig in CD in DRAM */ 1657 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 1658 pHwBlockBaseInDRAM + 1659 hwBlockOffsetInDRAM, 1660 &sizeInBytes); 1661 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 1662 pSessionDesc->reqSpcCacheHdr), 1663 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 1664 laCmdId, 1665 pSessionDesc->cmnRequestFlags, 1666 pSessionDesc->laCmdFlags); 1667 } else if ((SPC == pSessionDesc->singlePassState) && 1668 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) { 1669 pSessionDesc->symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING; 1670 pSessionDesc->singlePassState = LIKELY_SPC; 1671 pSessionDesc->isCipher = CPA_TRUE; 1672 pSessionDesc->isAuthEncryptOp = CPA_TRUE; 1673 pSessionDesc->isAuth = CPA_TRUE; 1674 pCdInfo = &(pSessionDesc->contentDescInfo); 1675 pHwBlockBaseInDRAM = (Cpa8U *)pCdInfo->pData; 1676 1677 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == 1678 pSessionDesc->cipherDirection) { 1679 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_CIPHER_HASH; 1680 } else { 1681 pSessionDesc->laCmdId = ICP_QAT_FW_LA_CMD_HASH_CIPHER; 1682 } 1683 1684 laCmdId = pSessionDesc->laCmdId; 1685 ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( 1686 pSessionDesc->laCmdFlags, 0); 1687 ICP_QAT_FW_LA_PROTO_SET(pSessionDesc->laCmdFlags, 1688 ICP_QAT_FW_LA_GCM_PROTO); 1689 1690 LacSymQat_CipherHwBlockPopulateCfgData(pSessionDesc, 1691 pHwBlockBaseInDRAM + 1692 hwBlockOffsetInDRAM, 1693 &sizeInBytes); 1694 1695 SalQatMsg_CmnHdrWrite((icp_qat_fw_comn_req_t *)&( 1696 pSessionDesc->reqCacheHdr), 1697 ICP_QAT_FW_COMN_REQ_CPM_FW_LA, 1698 laCmdId, 1699 pSessionDesc->cmnRequestFlags, 1700 pSessionDesc->laCmdFlags); 1701 } 1702 1703 else if (LAC_CIPHER_IS_CHACHA(cipher) && 1704 (LAC_CIPHER_SPC_IV_SIZE != pOpData->ivLenInBytes)) { 1705 LAC_INVALID_PARAM_LOG("IV for CHACHA"); 1706 return CPA_STATUS_INVALID_PARAM; 1707 } 1708 1709 if ((CPA_TRUE == pSessionDesc->isAuthEncryptOp) || isSpCcm) { 1710 if (CPA_CY_SYM_HASH_AES_CCM == hash) { 1711 status = LacSymAlgChain_CheckCCMData( 1712 pOpData->pAdditionalAuthData, 1713 pOpData->pIv, 1714 pOpData->messageLenToCipherInBytes, 1715 pOpData->ivLenInBytes); 1716 if (CPA_STATUS_SUCCESS == status) { 1717 LacSymAlgChain_PrepareCCMData( 1718 pSessionDesc, 1719 pOpData->pAdditionalAuthData, 1720 pOpData->pIv, 1721 pOpData->messageLenToCipherInBytes, 1722 pOpData->ivLenInBytes); 1723 } 1724 } else if (CPA_CY_SYM_HASH_AES_GCM == hash) { 1725 if (aadLenInBytes != 0 && 1726 pOpData->pAdditionalAuthData == NULL) { 1727 LAC_INVALID_PARAM_LOG("pAdditionalAuthData"); 1728 status = CPA_STATUS_INVALID_PARAM; 1729 } 1730 if (CPA_STATUS_SUCCESS == status) { 1731 LacSymAlgChain_PrepareGCMData( 1732 pSessionDesc, pOpData->pAdditionalAuthData); 1733 } 1734 } 1735 } 1736 1737 /* allocate cookie (used by callback function) */ 1738 if (CPA_STATUS_SUCCESS == status) { 1739 pSymCookie = (lac_sym_cookie_t *)Lac_MemPoolEntryAlloc( 1740 pService->lac_sym_cookie_pool); 1741 if (pSymCookie == NULL) { 1742 LAC_LOG_ERROR("Cannot allocate cookie - NULL"); 1743 status = CPA_STATUS_RESOURCE; 1744 } else if ((void *)CPA_STATUS_RETRY == pSymCookie) { 1745 pSymCookie = NULL; 1746 status = CPA_STATUS_RETRY; 1747 } else { 1748 pCookie = &(pSymCookie->u.bulkCookie); 1749 } 1750 } 1751 1752 if (CPA_STATUS_SUCCESS == status) { 1753 /* write the buffer descriptors */ 1754 if (IS_ZERO_LENGTH_BUFFER_SUPPORTED(cipher, hash)) { 1755 status = 1756 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer( 1757 (CpaBufferList *)pSrcBuffer, 1758 &srcAddrPhys, 1759 CPA_FALSE, 1760 &(pService->generic_service_info)); 1761 if (CPA_STATUS_SUCCESS != status) { 1762 LAC_LOG_ERROR( 1763 "Unable to write src buffer descriptors"); 1764 } 1765 /* For out of place operations */ 1766 if ((pSrcBuffer != pDstBuffer) && 1767 (CPA_STATUS_SUCCESS == status)) { 1768 status = 1769 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer( 1770 pDstBuffer, 1771 &dstAddrPhys, 1772 CPA_FALSE, 1773 &(pService->generic_service_info)); 1774 if (CPA_STATUS_SUCCESS != status) { 1775 LAC_LOG_ERROR( 1776 "Unable to write dest buffer descriptors"); 1777 } 1778 } 1779 } else { 1780 status = LacBuffDesc_BufferListDescWrite( 1781 (CpaBufferList *)pSrcBuffer, 1782 &srcAddrPhys, 1783 CPA_FALSE, 1784 &(pService->generic_service_info)); 1785 if (CPA_STATUS_SUCCESS != status) { 1786 LAC_LOG_ERROR( 1787 "Unable to write src buffer descriptors in " 1788 "LacBuffDesc_BufferListDescWrite"); 1789 } 1790 /* For out of place operations */ 1791 if ((pSrcBuffer != pDstBuffer) && 1792 (CPA_STATUS_SUCCESS == status)) { 1793 status = LacBuffDesc_BufferListDescWrite( 1794 pDstBuffer, 1795 &dstAddrPhys, 1796 CPA_FALSE, 1797 &(pService->generic_service_info)); 1798 if (CPA_STATUS_SUCCESS != status) { 1799 LAC_LOG_ERROR( 1800 "Unable to write dest buffer descriptors in " 1801 "LacBuffDesc_BufferListDescWrite"); 1802 } 1803 } 1804 } 1805 } 1806 if (CPA_STATUS_SUCCESS == status) { 1807 /* populate the cookie */ 1808 pCookie->pCallbackTag = pCallbackTag; 1809 pCookie->sessionCtx = pOpData->sessionCtx; 1810 pCookie->pOpData = (const CpaCySymOpData *)pOpData; 1811 pCookie->pDstBuffer = pDstBuffer; 1812 pCookie->updateSessionIvOnSend = CPA_FALSE; 1813 pCookie->updateUserIvOnRecieve = CPA_FALSE; 1814 pCookie->updateKeySizeOnRecieve = CPA_FALSE; 1815 pCookie->pNext = NULL; 1816 pCookie->instanceHandle = pService; 1817 1818 /* get the qat packet type for LAC packet type */ 1819 LacSymQat_packetTypeGet(pOpData->packetType, 1820 pSessionDesc->partialState, 1821 &qatPacketType); 1822 /* 1823 * For XTS mode, the key size must be updated after 1824 * the first partial has been sent. Set a flag here so the 1825 * response knows to do this. 1826 */ 1827 if (LAC_CIPHER_IS_XTS_MODE(cipher) && 1828 (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) && 1829 (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) && 1830 (qatPacketType == ICP_QAT_FW_LA_PARTIAL_START)) { 1831 pCookie->updateKeySizeOnRecieve = CPA_TRUE; 1832 } 1833 1834 /* 1835 * Now create the Request. 1836 * Start by populating it from the cache in the session 1837 * descriptor. 1838 */ 1839 pMsg = &(pCookie->qatMsg); 1840 pMsgDummy = (Cpa8U *)pMsg; 1841 1842 if (SPC == pSessionDesc->singlePassState) { 1843 pCacheDummyHdr = 1844 (Cpa8U *)&(pSessionDesc->reqSpcCacheHdr); 1845 pCacheDummyFtr = 1846 (Cpa8U *)&(pSessionDesc->reqSpcCacheFtr); 1847 } else { 1848 /* Normally, we want to use the SHRAM Constants Table if 1849 * possible for best performance (less DRAM accesses 1850 * incurred by CPM). But we can't use it for 1851 * partial-packet hash operations. This is why we build 1852 * 2 versions of the message template at sessionInit, 1853 * one for SHRAM Constants Table usage and the other 1854 * (default) for Content Descriptor h/w setup data in 1855 * DRAM. And we chose between them here on a 1856 * per-request basis, when we know the packetType 1857 */ 1858 if ((!pSessionDesc->useSymConstantsTable) || 1859 (pSessionDesc->isAuth && 1860 (CPA_CY_SYM_PACKET_TYPE_FULL != 1861 pOpData->packetType))) { 1862 pCacheDummyHdr = 1863 (Cpa8U *)&(pSessionDesc->reqCacheHdr); 1864 pCacheDummyFtr = 1865 (Cpa8U *)&(pSessionDesc->reqCacheFtr); 1866 } else { 1867 pCacheDummyHdr = 1868 (Cpa8U *)&(pSessionDesc->shramReqCacheHdr); 1869 pCacheDummyFtr = 1870 (Cpa8U *)&(pSessionDesc->shramReqCacheFtr); 1871 } 1872 } 1873 memcpy(pMsgDummy, 1874 pCacheDummyHdr, 1875 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)); 1876 memset((pMsgDummy + 1877 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_HDR_IN_LW)), 1878 0, 1879 (LAC_LONG_WORD_IN_BYTES * 1880 LAC_SIZE_OF_CACHE_TO_CLEAR_IN_LW)); 1881 memcpy(pMsgDummy + 1882 (LAC_LONG_WORD_IN_BYTES * 1883 LAC_START_OF_CACHE_FTR_IN_LW), 1884 pCacheDummyFtr, 1885 (LAC_LONG_WORD_IN_BYTES * LAC_SIZE_OF_CACHE_FTR_IN_LW)); 1886 /* 1887 * Populate the comn_mid section 1888 */ 1889 SalQatMsg_CmnMidWrite(pMsg, 1890 pCookie, 1891 LAC_SYM_DEFAULT_QAT_PTR_TYPE, 1892 srcAddrPhys, 1893 dstAddrPhys, 1894 0, 1895 0); 1896 1897 /* 1898 * Populate the serv_specif_flags field of the Request header 1899 * Some of the flags are set up here. 1900 * Others are set up later when the RequestParams are set up. 1901 */ 1902 1903 LacSymQat_LaPacketCommandFlagSet( 1904 qatPacketType, 1905 laCmdId, 1906 cipher, 1907 &pMsg->comn_hdr.serv_specif_flags, 1908 pOpData->ivLenInBytes); 1909 1910 if (SPC == pSessionDesc->singlePassState) { 1911 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( 1912 pMsg->comn_hdr.serv_specif_flags, 1913 ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS); 1914 1915 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == 1916 pOpData->packetType) { 1917 ICP_QAT_FW_LA_RET_AUTH_SET( 1918 pMsg->comn_hdr.serv_specif_flags, 1919 ICP_QAT_FW_LA_NO_RET_AUTH_RES); 1920 1921 ICP_QAT_FW_LA_CMP_AUTH_SET( 1922 pMsg->comn_hdr.serv_specif_flags, 1923 ICP_QAT_FW_LA_NO_CMP_AUTH_RES); 1924 } 1925 } 1926 1927 ICP_QAT_FW_LA_SLICE_TYPE_SET(pMsg->comn_hdr.serv_specif_flags, 1928 pSessionDesc->cipherSliceType); 1929 1930 LacBuffDesc_BufferListTotalSizeGet(pSrcBuffer, &srcPktSize); 1931 LacBuffDesc_BufferListTotalSizeGet(pDstBuffer, &dstPktSize); 1932 1933 /* 1934 * Populate the CipherRequestParams section of the Request 1935 */ 1936 if (laCmdId != ICP_QAT_FW_LA_CMD_AUTH) { 1937 1938 Cpa8U *pIvBuffer = NULL; 1939 1940 status = LacCipher_PerformParamCheck(cipher, 1941 pOpData, 1942 srcPktSize); 1943 if (CPA_STATUS_SUCCESS != status) { 1944 /* free the cookie */ 1945 Lac_MemPoolEntryFree(pCookie); 1946 return status; 1947 } 1948 1949 if (CPA_STATUS_SUCCESS == status) { 1950 /* align cipher IV */ 1951 status = LacCipher_PerformIvCheck( 1952 &(pService->generic_service_info), 1953 pCookie, 1954 qatPacketType, 1955 &pIvBuffer); 1956 } 1957 if ((SPC == pSessionDesc->singlePassState) && 1958 ((ICP_QAT_FW_LA_PARTIAL_MID == qatPacketType) || 1959 (ICP_QAT_FW_LA_PARTIAL_END == qatPacketType))) { 1960 /* For SPC stateful cipher state size for mid 1961 * and end partial packet is 48 bytes 1962 */ 1963 pSpcCdCtrlHdr = 1964 (icp_qat_fw_cipher_cd_ctrl_hdr_t *)&( 1965 pMsg->cd_ctrl); 1966 pSpcCdCtrlHdr->cipher_state_sz = 1967 LAC_BYTES_TO_QUADWORDS( 1968 LAC_SYM_QAT_CIPHER_SPC_STATE_SIZE); 1969 } 1970 /*populate the cipher request parameters */ 1971 if (CPA_STATUS_SUCCESS == status) { 1972 Cpa64U ivBufferPhysAddr = 0; 1973 1974 if (pIvBuffer != NULL) { 1975 /* User OpData memory being used for IV 1976 * buffer */ 1977 /* get the physical address */ 1978 ivBufferPhysAddr = 1979 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 1980 pService->generic_service_info, 1981 pIvBuffer); 1982 if (0 == ivBufferPhysAddr) { 1983 LAC_LOG_ERROR( 1984 "Unable to get the physical address " 1985 "of the IV\n"); 1986 status = CPA_STATUS_FAIL; 1987 } 1988 } 1989 1990 if (status == CPA_STATUS_SUCCESS) { 1991 status = 1992 LacSymQat_CipherRequestParamsPopulate( 1993 pSessionDesc, 1994 pMsg, 1995 pOpData 1996 ->cryptoStartSrcOffsetInBytes, 1997 pOpData 1998 ->messageLenToCipherInBytes, 1999 ivBufferPhysAddr, 2000 pIvBuffer); 2001 } 2002 } 2003 2004 if ((SPC == pSessionDesc->singlePassState) && 2005 CPA_STATUS_SUCCESS == status) { 2006 Cpa64U aadBufferPhysAddr = 0; 2007 2008 /* For CHACHA and AES-GCM there is an AAD buffer 2009 * if aadLenInBytes is nonzero In case of 2010 * AES-GMAC, AAD buffer passed in the src 2011 * buffer. 2012 */ 2013 if ((0 != aadLenInBytes && 2014 CPA_CY_SYM_HASH_AES_GMAC != hash) || 2015 isSpCcm) { 2016 LAC_CHECK_NULL_PARAM( 2017 pOpData->pAdditionalAuthData); 2018 Cpa32U aadDataLen = 2019 pSessionDesc->aadLenInBytes; 2020 2021 /* In case of AES_CCM, B0 block size and 2022 * 2 bytes of AAD len encoding need to 2023 * be added to total AAD data len */ 2024 if (isSpCcm) 2025 aadDataLen += 2026 LAC_CIPHER_CCM_AAD_OFFSET; 2027 2028 blockLen = 2029 LacSymQat_CipherBlockSizeBytesGet( 2030 cipher); 2031 if ((aadDataLen % blockLen) != 0) { 2032 paddingLen = blockLen - 2033 (aadDataLen % blockLen); 2034 memset( 2035 &pOpData 2036 ->pAdditionalAuthData 2037 [aadDataLen], 2038 0, 2039 paddingLen); 2040 } 2041 2042 /* User OpData memory being used for AAD 2043 * buffer */ 2044 /* get the physical address */ 2045 aadBufferPhysAddr = 2046 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 2047 pService->generic_service_info, 2048 pOpData->pAdditionalAuthData); 2049 if (0 == aadBufferPhysAddr) { 2050 LAC_LOG_ERROR( 2051 "Unable to get the physical address " 2052 "of the aad\n"); 2053 status = CPA_STATUS_FAIL; 2054 } 2055 } 2056 2057 if (CPA_STATUS_SUCCESS == status) { 2058 icp_qat_fw_la_cipher_req_params_t *pCipherReqParams = 2059 (icp_qat_fw_la_cipher_req_params_t 2060 *)((Cpa8U *)&( 2061 pMsg->serv_specif_rqpars) + 2062 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 2063 2064 icp_qat_fw_la_cipher_20_req_params_t 2065 *pCipher20ReqParams = 2066 (void 2067 *)((Cpa8U *)&( 2068 pMsg->serv_specif_rqpars) + 2069 ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET); 2070 2071 if (isCyGen4x(pService)) { 2072 pCipher20ReqParams 2073 ->spc_aad_addr = 2074 aadBufferPhysAddr; 2075 pCipher20ReqParams->spc_aad_sz = 2076 pSessionDesc->aadLenInBytes; 2077 pCipher20ReqParams 2078 ->spc_aad_offset = 0; 2079 if (isSpCcm) 2080 pCipher20ReqParams 2081 ->spc_aad_sz += 2082 LAC_CIPHER_CCM_AAD_OFFSET; 2083 } else { 2084 pCipherReqParams->spc_aad_addr = 2085 aadBufferPhysAddr; 2086 pCipherReqParams->spc_aad_sz = 2087 (Cpa16U)pSessionDesc 2088 ->aadLenInBytes; 2089 } 2090 2091 if (CPA_TRUE != 2092 pSessionDesc->digestIsAppended) { 2093 Cpa64U digestBufferPhysAddr = 0; 2094 /* User OpData memory being used 2095 * for digest buffer */ 2096 /* get the physical address */ 2097 digestBufferPhysAddr = 2098 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 2099 pService 2100 ->generic_service_info, 2101 pOpData->pDigestResult); 2102 if (0 != digestBufferPhysAddr) { 2103 if (isCyGen4x( 2104 pService)) { 2105 pCipher20ReqParams 2106 ->spc_auth_res_addr = 2107 digestBufferPhysAddr; 2108 pCipher20ReqParams 2109 ->spc_auth_res_sz = 2110 (Cpa8U)pSessionDesc 2111 ->hashResultSize; 2112 } else { 2113 pCipherReqParams 2114 ->spc_auth_res_addr = 2115 digestBufferPhysAddr; 2116 pCipherReqParams 2117 ->spc_auth_res_sz = 2118 (Cpa8U)pSessionDesc 2119 ->hashResultSize; 2120 } 2121 } else { 2122 LAC_LOG_ERROR( 2123 "Unable to get the physical address " 2124 "of the digest\n"); 2125 status = 2126 CPA_STATUS_FAIL; 2127 } 2128 } else { 2129 /* Check if the dest buffer can 2130 * handle the digest, only for 2131 * last packet */ 2132 if (((ICP_QAT_FW_LA_PARTIAL_NONE == 2133 qatPacketType) || 2134 (ICP_QAT_FW_LA_PARTIAL_END == 2135 qatPacketType))) { 2136 if (dstPktSize < 2137 (pOpData 2138 ->cryptoStartSrcOffsetInBytes + 2139 pOpData 2140 ->messageLenToCipherInBytes + 2141 pSessionDesc 2142 ->hashResultSize)) 2143 status = 2144 CPA_STATUS_INVALID_PARAM; 2145 } 2146 if (isCyGen4x(pService)) { 2147 pCipher20ReqParams 2148 ->spc_auth_res_sz = 2149 (Cpa8U)pSessionDesc 2150 ->hashResultSize; 2151 } else { 2152 pCipherReqParams 2153 ->spc_auth_res_sz = 2154 (Cpa8U)pSessionDesc 2155 ->hashResultSize; 2156 } 2157 } 2158 } 2159 } 2160 } 2161 2162 /* 2163 * Set up HashRequestParams part of Request 2164 */ 2165 if ((status == CPA_STATUS_SUCCESS) && 2166 (laCmdId != ICP_QAT_FW_LA_CMD_CIPHER)) { 2167 Cpa32U authOffsetInBytes = 2168 pOpData->hashStartSrcOffsetInBytes; 2169 Cpa32U authLenInBytes = 2170 pOpData->messageLenToHashInBytes; 2171 2172 status = LacHash_PerformParamCheck(instanceHandle, 2173 pSessionDesc, 2174 pOpData, 2175 srcPktSize, 2176 pVerifyResult); 2177 if (CPA_STATUS_SUCCESS != status) { 2178 /* free the cookie */ 2179 Lac_MemPoolEntryFree(pCookie); 2180 return status; 2181 } 2182 if (CPA_STATUS_SUCCESS == status) { 2183 /* Info structure for CCM/GCM */ 2184 lac_sym_qat_hash_state_buffer_info_t 2185 hashStateBufferInfo = { 0 }; 2186 lac_sym_qat_hash_state_buffer_info_t 2187 *pHashStateBufferInfo = 2188 &(pSessionDesc->hashStateBufferInfo); 2189 2190 if (CPA_TRUE == pSessionDesc->isAuthEncryptOp) { 2191 icp_qat_fw_la_auth_req_params_t *pHashReqParams = 2192 (icp_qat_fw_la_auth_req_params_t 2193 *)((Cpa8U *)&( 2194 pMsg->serv_specif_rqpars) + 2195 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 2196 2197 hashStateBufferInfo.pData = 2198 pOpData->pAdditionalAuthData; 2199 if (pOpData->pAdditionalAuthData == 2200 NULL) { 2201 hashStateBufferInfo.pDataPhys = 2202 0; 2203 } else { 2204 hashStateBufferInfo 2205 .pDataPhys = LAC_MEM_CAST_PTR_TO_UINT64( 2206 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 2207 pService 2208 ->generic_service_info, 2209 pOpData 2210 ->pAdditionalAuthData)); 2211 } 2212 2213 hashStateBufferInfo 2214 .stateStorageSzQuadWords = 0; 2215 hashStateBufferInfo 2216 .prefixAadSzQuadWords = 2217 LAC_BYTES_TO_QUADWORDS( 2218 pHashReqParams->u2.aad_sz); 2219 2220 /* Overwrite hash state buffer info 2221 * structure pointer with the one 2222 * created for CCM/GCM */ 2223 pHashStateBufferInfo = 2224 &hashStateBufferInfo; 2225 2226 /* Aad buffer could be null in the GCM 2227 * case */ 2228 if (0 == 2229 hashStateBufferInfo.pDataPhys && 2230 CPA_CY_SYM_HASH_AES_GCM != hash && 2231 CPA_CY_SYM_HASH_AES_GMAC != hash) { 2232 LAC_LOG_ERROR( 2233 "Unable to get the physical address" 2234 "of the AAD\n"); 2235 status = CPA_STATUS_FAIL; 2236 } 2237 2238 /* for CCM/GCM the hash and cipher data 2239 * regions are equal */ 2240 authOffsetInBytes = 2241 pOpData 2242 ->cryptoStartSrcOffsetInBytes; 2243 2244 /* For authenticated encryption, 2245 * authentication length is determined 2246 * by messageLenToCipherInBytes for 2247 * AES-GCM and AES-CCM, and by 2248 * messageLenToHashInBytes for AES-GMAC. 2249 * You don't see the latter here, as 2250 * that is the initial value of 2251 * authLenInBytes. */ 2252 if (hash != CPA_CY_SYM_HASH_AES_GMAC) 2253 authLenInBytes = 2254 pOpData 2255 ->messageLenToCipherInBytes; 2256 } else if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 2257 hash || 2258 CPA_CY_SYM_HASH_ZUC_EIA3 == hash) { 2259 hashStateBufferInfo.pData = 2260 pOpData->pAdditionalAuthData; 2261 hashStateBufferInfo.pDataPhys = 2262 LAC_OS_VIRT_TO_PHYS_EXTERNAL( 2263 pService->generic_service_info, 2264 hashStateBufferInfo.pData); 2265 hashStateBufferInfo 2266 .stateStorageSzQuadWords = 0; 2267 hashStateBufferInfo 2268 .prefixAadSzQuadWords = 2269 LAC_BYTES_TO_QUADWORDS( 2270 aadLenInBytes); 2271 2272 pHashStateBufferInfo = 2273 &hashStateBufferInfo; 2274 2275 if (0 == 2276 hashStateBufferInfo.pDataPhys) { 2277 LAC_LOG_ERROR( 2278 "Unable to get the physical address" 2279 "of the AAD\n"); 2280 status = CPA_STATUS_FAIL; 2281 } 2282 } 2283 if (CPA_CY_SYM_HASH_AES_CCM == hash) { 2284 if (CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT == 2285 pSessionDesc->cipherDirection) { 2286 /* On a decrypt path pSrcBuffer 2287 * is used as this is where 2288 * encrypted digest is located. 2289 * Firmware uses encrypted 2290 * digest for 2291 * compare/verification*/ 2292 pBufferList = 2293 (CpaBufferList *)pSrcBuffer; 2294 } else { 2295 /* On an encrypt path pDstBuffer 2296 * is used as this is where 2297 * encrypted digest will be 2298 * written */ 2299 pBufferList = 2300 (CpaBufferList *)pDstBuffer; 2301 } 2302 status = LacSymAlgChain_PtrFromOffsetGet( 2303 pBufferList, 2304 pOpData->cryptoStartSrcOffsetInBytes + 2305 pOpData 2306 ->messageLenToCipherInBytes, 2307 &pDigestResult); 2308 if (CPA_STATUS_SUCCESS != status) { 2309 LAC_LOG_ERROR( 2310 "Cannot set digest pointer within the" 2311 " buffer list - offset out of bounds"); 2312 } 2313 } else { 2314 pDigestResult = pOpData->pDigestResult; 2315 } 2316 2317 if (CPA_TRUE == 2318 pSessionDesc->useStatefulSha3ContentDesc) { 2319 LacAlgChain_StatefulSha3_SkipStateLoadFlags( 2320 pMsg, 2321 qatPacketType, 2322 pSessionDesc->qatHashMode); 2323 } 2324 2325 if (CPA_CY_SYM_OP_ALGORITHM_CHAINING == 2326 pSessionDesc->symOperation) { 2327 /* In alg chaining mode, packets are not 2328 * seen as partials for hash operations. 2329 * Override to NONE. 2330 */ 2331 qatPacketType = 2332 ICP_QAT_FW_LA_PARTIAL_NONE; 2333 } 2334 digestIsAppended = 2335 pSessionDesc->digestIsAppended; 2336 if (CPA_TRUE == digestIsAppended) { 2337 /*Check if the destination buffer can 2338 * handle the digest if digestIsAppend 2339 * is true*/ 2340 if (srcPktSize < 2341 (authOffsetInBytes + 2342 authLenInBytes + 2343 pSessionDesc->hashResultSize)) { 2344 status = 2345 CPA_STATUS_INVALID_PARAM; 2346 } 2347 } 2348 if (CPA_STATUS_SUCCESS == status) { 2349 /* populate the hash request parameters 2350 */ 2351 status = 2352 LacSymQat_HashRequestParamsPopulate( 2353 pMsg, 2354 authOffsetInBytes, 2355 authLenInBytes, 2356 &(pService 2357 ->generic_service_info), 2358 pHashStateBufferInfo, 2359 qatPacketType, 2360 pSessionDesc->hashResultSize, 2361 pSessionDesc->digestVerify, 2362 digestIsAppended ? 2363 NULL : 2364 pDigestResult, 2365 hash, 2366 NULL); 2367 } 2368 } 2369 } 2370 } 2371 2372 /* 2373 * send the message to the QAT 2374 */ 2375 if (CPA_STATUS_SUCCESS == status) { 2376 qatUtilsAtomicInc(&(pSessionDesc->u.pendingCbCount)); 2377 2378 status = LacSymQueue_RequestSend(instanceHandle, 2379 pCookie, 2380 pSessionDesc); 2381 2382 if (CPA_STATUS_SUCCESS != status) { 2383 /* Decrease pending callback counter on send fail. */ 2384 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount)); 2385 } 2386 } 2387 /* Case that will catch all error status's for this function */ 2388 if (CPA_STATUS_SUCCESS != status) { 2389 /* free the cookie */ 2390 if (NULL != pSymCookie) { 2391 Lac_MemPoolEntryFree(pSymCookie); 2392 } 2393 } 2394 return status; 2395 } 2396