1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 4 /** 5 *************************************************************************** 6 * @file lac_sym_qat_hash.c 7 * 8 * @ingroup LacSymQatHash 9 * 10 * Implementation for populating QAT data structures for hash operation 11 ***************************************************************************/ 12 13 /* 14 ******************************************************************************* 15 * Include public/global header files 16 ******************************************************************************* 17 */ 18 19 #include "cpa.h" 20 #include "cpa_cy_sym.h" 21 #include "icp_accel_devices.h" 22 #include "icp_adf_debug.h" 23 #include "lac_log.h" 24 #include "lac_mem.h" 25 #include "lac_sym.h" 26 #include "lac_common.h" 27 #include "lac_sym_qat.h" 28 #include "lac_list.h" 29 #include "lac_sal_types.h" 30 #include "lac_sal_types_crypto.h" 31 #include "lac_sym_qat_hash.h" 32 #include "lac_sym_qat_hash_defs_lookup.h" 33 #include "sal_hw_gen.h" 34 35 /** 36 * This structure contains pointers into the hash setup block of the 37 * security descriptor. As the hash setup block contains fields that 38 * are of variable length, pointers must be calculated to these fields 39 * and the hash setup block is populated using these pointers. */ 40 typedef struct lac_hash_blk_ptrs_s { 41 icp_qat_hw_auth_setup_t *pInHashSetup; 42 /**< inner hash setup */ 43 Cpa8U *pInHashInitState1; 44 /**< inner initial state 1 */ 45 Cpa8U *pInHashInitState2; 46 /**< inner initial state 2 */ 47 icp_qat_hw_auth_setup_t *pOutHashSetup; 48 /**< outer hash setup */ 49 Cpa8U *pOutHashInitState1; 50 /**< outer hash initial state */ 51 } lac_hash_blk_ptrs_t; 52 53 typedef struct lac_hash_blk_ptrs_optimised_s { 54 Cpa8U *pInHashInitState1; 55 /**< inner initial state 1 */ 56 Cpa8U *pInHashInitState2; 57 /**< inner initial state 2 */ 58 59 } lac_hash_blk_ptrs_optimised_t; 60 61 /** 62 * This function calculates the pointers into the hash setup block 63 * based on the control block 64 * 65 * @param[in] pHashControlBlock Pointer to hash control block 66 * @param[in] pHwBlockBase pointer to base of hardware block 67 * @param[out] pHashBlkPtrs structure containing pointers to 68 * various fields in the hash setup block 69 * 70 * @return void 71 */ 72 static void 73 LacSymQat_HashHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 74 void *pHwBlockBase, 75 lac_hash_blk_ptrs_t *pHashBlkPtrs); 76 77 static void 78 LacSymQat_HashSetupBlockOptimisedFormatInit( 79 const CpaCySymHashSetupData *pHashSetupData, 80 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 81 void *pHwBlockBase, 82 icp_qat_hw_auth_mode_t qatHashMode, 83 lac_sym_qat_hash_precompute_info_t *pPrecompute, 84 lac_sym_qat_hash_defs_t *pHashDefs, 85 lac_sym_qat_hash_defs_t *pOuterHashDefs); 86 87 /** 88 * This function populates the hash setup block 89 * 90 * @param[in] pHashSetupData Pointer to the hash context 91 * @param[in] pHashControlBlock Pointer to hash control block 92 * @param[in] pHwBlockBase pointer to base of hardware block 93 * @param[in] qatHashMode QAT hash mode 94 * @param[in] pPrecompute For auth mode, this is the pointer 95 * to the precompute data. Otherwise this 96 * should be set to NULL 97 * @param[in] pHashDefs Pointer to Hash definitions 98 * @param[in] pOuterHashDefs Pointer to Outer Hash definitions. 99 * Required for nested hash mode only 100 * 101 * @return void 102 */ 103 static void 104 LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, 105 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 106 void *pHwBlockBase, 107 icp_qat_hw_auth_mode_t qatHashMode, 108 lac_sym_qat_hash_precompute_info_t *pPrecompute, 109 lac_sym_qat_hash_defs_t *pHashDefs, 110 lac_sym_qat_hash_defs_t *pOuterHashDefs); 111 112 /** @ingroup LacSymQatHash */ 113 void 114 LacSymQat_HashGetCfgData(CpaInstanceHandle pInstance, 115 icp_qat_hw_auth_mode_t qatHashMode, 116 CpaCySymHashMode apiHashMode, 117 CpaCySymHashAlgorithm apiHashAlgorithm, 118 icp_qat_hw_auth_algo_t *pQatAlgorithm, 119 CpaBoolean *pQatNested) 120 { 121 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 122 123 LacSymQat_HashDefsLookupGet(pInstance, apiHashAlgorithm, &pHashDefs); 124 *pQatAlgorithm = pHashDefs->qatInfo->algoEnc; 125 126 if (IS_HASH_MODE_2(qatHashMode)) { 127 /* set bit for nested hashing */ 128 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED; 129 } 130 /* Nested hash in mode 0. */ 131 else if (CPA_CY_SYM_HASH_MODE_NESTED == apiHashMode) { 132 /* set bit for nested hashing */ 133 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED; 134 } 135 /* mode0 - plain or mode1 - auth */ 136 else { 137 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; 138 } 139 } 140 141 /** @ingroup LacSymQatHash */ 142 void 143 LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, 144 CpaInstanceHandle instanceHandle, 145 const CpaCySymHashSetupData *pHashSetupData, 146 void *pHwBlockBase, 147 Cpa32U hwBlockOffsetInQuadWords, 148 icp_qat_fw_slice_t nextSlice, 149 icp_qat_hw_auth_mode_t qatHashMode, 150 CpaBoolean useSymConstantsTable, 151 CpaBoolean useOptimisedContentDesc, 152 CpaBoolean useStatefulSha3ContentDesc, 153 lac_sym_qat_hash_precompute_info_t *pPrecompute, 154 Cpa32U *pHashBlkSizeInBytes) 155 { 156 icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = 157 (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 158 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 159 lac_sym_qat_hash_defs_t *pOuterHashDefs = NULL; 160 Cpa32U hashSetupBlkSize = 0; 161 162 /* setup the offset in QuadWords into the hw blk */ 163 cd_ctrl->hash_cfg_offset = (Cpa8U)hwBlockOffsetInQuadWords; 164 165 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, nextSlice); 166 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 167 168 LacSymQat_HashDefsLookupGet(instanceHandle, 169 pHashSetupData->hashAlgorithm, 170 &pHashDefs); 171 172 /* Hmac in mode 2 TLS */ 173 if (IS_HASH_MODE_2(qatHashMode)) { 174 if (isCyGen4x((sal_crypto_service_t *)instanceHandle)) { 175 /* CPM2.0 has a dedicated bit for HMAC mode2 */ 176 ICP_QAT_FW_HASH_FLAG_MODE2_SET(cd_ctrl->hash_flags, 177 QAT_FW_LA_MODE2); 178 } else { 179 /* Set bit for nested hashing. 180 * Make sure not to overwrite other flags in hash_flags 181 * byte. 182 */ 183 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 184 cd_ctrl->hash_flags, 185 ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); 186 } 187 } 188 /* Nested hash in mode 0 */ 189 else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) { 190 /* Set bit for nested hashing. 191 * Make sure not to overwrite other flags in hash_flags byte. 192 */ 193 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 194 cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); 195 } 196 /* mode0 - plain or mode1 - auth */ 197 else { 198 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 199 cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED); 200 } 201 202 /* Set skip state load flags */ 203 if (useStatefulSha3ContentDesc) { 204 /* Here both skip state load flags are set. FW reads them based 205 * on partial packet type. */ 206 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( 207 cd_ctrl->hash_flags, QAT_FW_LA_SKIP_INNER_STATE1_LOAD); 208 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( 209 cd_ctrl->hash_flags, QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); 210 } 211 212 /* set the final digest size */ 213 cd_ctrl->final_sz = (Cpa8U)pHashSetupData->digestResultLenInBytes; 214 215 /* set the state1 size */ 216 if (useStatefulSha3ContentDesc) { 217 cd_ctrl->inner_state1_sz = 218 LAC_ALIGN_POW2_ROUNDUP(LAC_HASH_SHA3_STATEFUL_STATE_SIZE, 219 LAC_QUAD_WORD_IN_BYTES); 220 } else { 221 cd_ctrl->inner_state1_sz = 222 LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length, 223 LAC_QUAD_WORD_IN_BYTES); 224 } 225 226 /* set the inner result size to the digest length */ 227 cd_ctrl->inner_res_sz = (Cpa8U)pHashDefs->algInfo->digestLength; 228 229 /* set the state2 size - only for mode 1 Auth algos and AES CBC MAC */ 230 if (IS_HASH_MODE_1(qatHashMode) || 231 pHashSetupData->hashAlgorithm == CPA_CY_SYM_HASH_AES_CBC_MAC || 232 pHashSetupData->hashAlgorithm == CPA_CY_SYM_HASH_ZUC_EIA3) { 233 cd_ctrl->inner_state2_sz = 234 LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state2Length, 235 LAC_QUAD_WORD_IN_BYTES); 236 } else { 237 cd_ctrl->inner_state2_sz = 0; 238 } 239 240 if (useSymConstantsTable) { 241 cd_ctrl->inner_state2_offset = 242 LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state1_sz); 243 244 /* size of inner part of hash setup block */ 245 hashSetupBlkSize = 246 cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; 247 } else { 248 cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset + 249 LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) + 250 cd_ctrl->inner_state1_sz); 251 252 /* size of inner part of hash setup block */ 253 hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) + 254 cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; 255 } 256 257 /* For nested hashing - Fill in the outer fields */ 258 if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode || 259 IS_HASH_MODE_2(qatHashMode)) { 260 /* For nested - use the outer algorithm. This covers TLS and 261 * nested hash. For HMAC mode2 use inner algorithm again */ 262 CpaCySymHashAlgorithm outerAlg = 263 (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) ? 264 pHashSetupData->nestedModeSetupData.outerHashAlgorithm : 265 pHashSetupData->hashAlgorithm; 266 267 LacSymQat_HashDefsLookupGet(instanceHandle, 268 outerAlg, 269 &pOuterHashDefs); 270 271 /* outer config offset */ 272 cd_ctrl->outer_config_offset = cd_ctrl->inner_state2_offset + 273 LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state2_sz); 274 275 if (useStatefulSha3ContentDesc) { 276 cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( 277 LAC_HASH_SHA3_STATEFUL_STATE_SIZE, 278 LAC_QUAD_WORD_IN_BYTES); 279 } else { 280 cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( 281 pOuterHashDefs->algInfo->stateSize, 282 LAC_QUAD_WORD_IN_BYTES); 283 } 284 285 /* outer result size */ 286 cd_ctrl->outer_res_sz = 287 (Cpa8U)pOuterHashDefs->algInfo->digestLength; 288 289 /* outer_prefix_offset will be the size of the inner prefix data 290 * plus the hash state storage size. */ 291 /* The prefix buffer is part of the ReqParams, so this param 292 * will be setup where ReqParams are set up */ 293 294 /* add on size of outer part of hash block */ 295 hashSetupBlkSize += 296 sizeof(icp_qat_hw_auth_setup_t) + cd_ctrl->outer_state1_sz; 297 } else { 298 cd_ctrl->outer_config_offset = 0; 299 cd_ctrl->outer_state1_sz = 0; 300 cd_ctrl->outer_res_sz = 0; 301 } 302 303 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pHashSetupData->hashAlgorithm) { 304 /* add the size for the cipher config word, the key and the IV*/ 305 hashSetupBlkSize += sizeof(icp_qat_hw_cipher_config_t) + 306 pHashSetupData->authModeSetupData.authKeyLenInBytes + 307 ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; 308 } 309 310 *pHashBlkSizeInBytes = hashSetupBlkSize; 311 312 if (useOptimisedContentDesc) { 313 LacSymQat_HashSetupBlockOptimisedFormatInit(pHashSetupData, 314 cd_ctrl, 315 pHwBlockBase, 316 qatHashMode, 317 pPrecompute, 318 pHashDefs, 319 pOuterHashDefs); 320 } else if (!useSymConstantsTable) { 321 /***************************************************************************** 322 * Populate Hash Setup block * 323 *****************************************************************************/ 324 LacSymQat_HashSetupBlockInit(pHashSetupData, 325 cd_ctrl, 326 pHwBlockBase, 327 qatHashMode, 328 pPrecompute, 329 pHashDefs, 330 pOuterHashDefs); 331 } 332 } 333 334 /* This fn populates fields in both the CD ctrl block and the ReqParams block 335 * which describe the Hash ReqParams: 336 * cd_ctrl.outer_prefix_offset 337 * cd_ctrl.outer_prefix_sz 338 * req_params.inner_prefix_sz/aad_sz 339 * req_params.hash_state_sz 340 * req_params.auth_res_sz 341 * 342 */ 343 void 344 LacSymQat_HashSetupReqParamsMetaData( 345 icp_qat_la_bulk_req_ftr_t *pMsg, 346 CpaInstanceHandle instanceHandle, 347 const CpaCySymHashSetupData *pHashSetupData, 348 CpaBoolean hashStateBuffer, 349 icp_qat_hw_auth_mode_t qatHashMode, 350 CpaBoolean digestVerify) 351 { 352 icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = NULL; 353 icp_qat_la_auth_req_params_t *pHashReqParams = NULL; 354 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 355 356 cd_ctrl = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 357 pHashReqParams = 358 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 359 360 LacSymQat_HashDefsLookupGet(instanceHandle, 361 pHashSetupData->hashAlgorithm, 362 &pHashDefs); 363 364 /* Hmac in mode 2 TLS */ 365 if (IS_HASH_MODE_2(qatHashMode)) { 366 /* Inner and outer prefixes are the block length */ 367 pHashReqParams->u2.inner_prefix_sz = 368 (Cpa8U)pHashDefs->algInfo->blockLength; 369 cd_ctrl->outer_prefix_sz = 370 (Cpa8U)pHashDefs->algInfo->blockLength; 371 cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( 372 LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), 373 LAC_QUAD_WORD_IN_BYTES)); 374 } 375 /* Nested hash in mode 0 */ 376 else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) { 377 378 /* set inner and outer prefixes */ 379 pHashReqParams->u2.inner_prefix_sz = 380 (Cpa8U)pHashSetupData->nestedModeSetupData 381 .innerPrefixLenInBytes; 382 cd_ctrl->outer_prefix_sz = 383 (Cpa8U)pHashSetupData->nestedModeSetupData 384 .outerPrefixLenInBytes; 385 cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( 386 LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), 387 LAC_QUAD_WORD_IN_BYTES)); 388 } 389 /* mode0 - plain or mode1 - auth */ 390 else { 391 Cpa16U aadDataSize = 0; 392 393 /* For Auth Encrypt set the aad size */ 394 if (CPA_CY_SYM_HASH_AES_CCM == pHashSetupData->hashAlgorithm) { 395 /* at the beginning of the buffer there is B0 block */ 396 aadDataSize = LAC_HASH_AES_CCM_BLOCK_SIZE; 397 398 /* then, if there is some 'a' data, the buffer will 399 * store encoded 400 * length of 'a' and 'a' itself */ 401 if (pHashSetupData->authModeSetupData.aadLenInBytes > 402 0) { 403 /* as the QAT API puts the requirement on the 404 * pAdditionalAuthData not to be bigger than 240 405 * bytes then we 406 * just need 2 bytes to store encoded length of 407 * 'a' */ 408 aadDataSize += sizeof(Cpa16U); 409 aadDataSize += 410 (Cpa16U)pHashSetupData->authModeSetupData 411 .aadLenInBytes; 412 } 413 414 /* round the aad size to the multiple of CCM block 415 * size.*/ 416 pHashReqParams->u2.aad_sz = 417 LAC_ALIGN_POW2_ROUNDUP(aadDataSize, 418 LAC_HASH_AES_CCM_BLOCK_SIZE); 419 } else if (CPA_CY_SYM_HASH_AES_GCM == 420 pHashSetupData->hashAlgorithm) { 421 aadDataSize = 422 (Cpa16U) 423 pHashSetupData->authModeSetupData.aadLenInBytes; 424 425 /* round the aad size to the multiple of GCM hash block 426 * size. */ 427 pHashReqParams->u2.aad_sz = 428 LAC_ALIGN_POW2_ROUNDUP(aadDataSize, 429 LAC_HASH_AES_GCM_BLOCK_SIZE); 430 } else { 431 pHashReqParams->u2.aad_sz = 0; 432 } 433 434 cd_ctrl->outer_prefix_sz = 0; 435 cd_ctrl->outer_prefix_offset = 0; 436 } 437 438 /* If there is a hash state prefix buffer */ 439 if (CPA_TRUE == hashStateBuffer) { 440 /* Note, this sets up size for both aad and non-aad cases */ 441 pHashReqParams->hash_state_sz = LAC_BYTES_TO_QUADWORDS( 442 LAC_ALIGN_POW2_ROUNDUP(pHashReqParams->u2.inner_prefix_sz, 443 LAC_QUAD_WORD_IN_BYTES) + 444 LAC_ALIGN_POW2_ROUNDUP(cd_ctrl->outer_prefix_sz, 445 LAC_QUAD_WORD_IN_BYTES)); 446 } else { 447 pHashReqParams->hash_state_sz = 0; 448 } 449 450 if (CPA_TRUE == digestVerify) { 451 /* auth result size in bytes to be read in for a verify 452 * operation */ 453 pHashReqParams->auth_res_sz = 454 (Cpa8U)pHashSetupData->digestResultLenInBytes; 455 } else { 456 pHashReqParams->auth_res_sz = 0; 457 } 458 459 pHashReqParams->resrvd1 = 0; 460 } 461 462 void 463 LacSymQat_HashHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl, 464 void *pHwBlockBase, 465 lac_hash_blk_ptrs_t *pHashBlkPtrs) 466 { 467 /* encoded offset for inner config is converted to a byte offset. */ 468 pHashBlkPtrs->pInHashSetup = 469 (icp_qat_hw_auth_setup_t *)((Cpa8U *)pHwBlockBase + 470 (cd_ctrl->hash_cfg_offset * 471 LAC_QUAD_WORD_IN_BYTES)); 472 473 pHashBlkPtrs->pInHashInitState1 = (Cpa8U *)pHashBlkPtrs->pInHashSetup + 474 sizeof(icp_qat_hw_auth_setup_t); 475 476 pHashBlkPtrs->pInHashInitState2 = 477 (Cpa8U *)(pHashBlkPtrs->pInHashInitState1) + 478 cd_ctrl->inner_state1_sz; 479 480 pHashBlkPtrs->pOutHashSetup = 481 (icp_qat_hw_auth_setup_t *)((Cpa8U *)(pHashBlkPtrs 482 ->pInHashInitState2) + 483 cd_ctrl->inner_state2_sz); 484 485 pHashBlkPtrs->pOutHashInitState1 = 486 (Cpa8U *)(pHashBlkPtrs->pOutHashSetup) + 487 sizeof(icp_qat_hw_auth_setup_t); 488 } 489 490 static void 491 LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, 492 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 493 void *pHwBlockBase, 494 icp_qat_hw_auth_mode_t qatHashMode, 495 lac_sym_qat_hash_precompute_info_t *pPrecompute, 496 lac_sym_qat_hash_defs_t *pHashDefs, 497 lac_sym_qat_hash_defs_t *pOuterHashDefs) 498 { 499 Cpa32U innerConfig = 0; 500 lac_hash_blk_ptrs_t hashBlkPtrs = { 0 }; 501 Cpa32U aedHashCmpLength = 0; 502 503 LacSymQat_HashHwBlockPtrsInit(pHashControlBlock, 504 pHwBlockBase, 505 &hashBlkPtrs); 506 507 innerConfig = ICP_QAT_HW_AUTH_CONFIG_BUILD( 508 qatHashMode, 509 pHashDefs->qatInfo->algoEnc, 510 pHashSetupData->digestResultLenInBytes); 511 512 /* Set the Inner hash configuration */ 513 hashBlkPtrs.pInHashSetup->auth_config.config = innerConfig; 514 hashBlkPtrs.pInHashSetup->auth_config.reserved = 0; 515 516 /* For mode 1 pre-computes for auth algorithms */ 517 if (IS_HASH_MODE_1(qatHashMode) || 518 CPA_CY_SYM_HASH_AES_CBC_MAC == pHashSetupData->hashAlgorithm || 519 CPA_CY_SYM_HASH_ZUC_EIA3 == pHashSetupData->hashAlgorithm) { 520 /* for HMAC in mode 1 authCounter is the block size 521 * else the authCounter is 0. The firmware expects the counter 522 * to be 523 * big endian */ 524 LAC_MEM_SHARED_WRITE_SWAP( 525 hashBlkPtrs.pInHashSetup->auth_counter.counter, 526 pHashDefs->qatInfo->authCounter); 527 528 /* state 1 is set to 0 for the following algorithms */ 529 if ((CPA_CY_SYM_HASH_AES_XCBC == 530 pHashSetupData->hashAlgorithm) || 531 (CPA_CY_SYM_HASH_AES_CMAC == 532 pHashSetupData->hashAlgorithm) || 533 (CPA_CY_SYM_HASH_AES_CBC_MAC == 534 pHashSetupData->hashAlgorithm) || 535 (CPA_CY_SYM_HASH_KASUMI_F9 == 536 pHashSetupData->hashAlgorithm) || 537 (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 538 pHashSetupData->hashAlgorithm) || 539 (CPA_CY_SYM_HASH_AES_CCM == 540 pHashSetupData->hashAlgorithm) || 541 (CPA_CY_SYM_HASH_AES_GMAC == 542 pHashSetupData->hashAlgorithm) || 543 (CPA_CY_SYM_HASH_AES_GCM == 544 pHashSetupData->hashAlgorithm) || 545 (CPA_CY_SYM_HASH_ZUC_EIA3 == 546 pHashSetupData->hashAlgorithm)) { 547 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1, 548 pHashDefs->qatInfo->state1Length); 549 } 550 551 /* Pad remaining bytes of sha1 precomputes */ 552 if (CPA_CY_SYM_HASH_SHA1 == pHashSetupData->hashAlgorithm) { 553 Cpa32U state1PadLen = 0; 554 Cpa32U state2PadLen = 0; 555 556 if (pHashControlBlock->inner_state1_sz > 557 pHashDefs->algInfo->stateSize) { 558 state1PadLen = 559 pHashControlBlock->inner_state1_sz - 560 pHashDefs->algInfo->stateSize; 561 } 562 563 if (pHashControlBlock->inner_state2_sz > 564 pHashDefs->algInfo->stateSize) { 565 state2PadLen = 566 pHashControlBlock->inner_state2_sz - 567 pHashDefs->algInfo->stateSize; 568 } 569 570 if (state1PadLen > 0) { 571 572 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1 + 573 pHashDefs->algInfo->stateSize, 574 state1PadLen); 575 } 576 577 if (state2PadLen > 0) { 578 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState2 + 579 pHashDefs->algInfo->stateSize, 580 state2PadLen); 581 } 582 } 583 584 pPrecompute->state1Size = pHashDefs->qatInfo->state1Length; 585 pPrecompute->state2Size = pHashDefs->qatInfo->state2Length; 586 587 /* Set the destination for pre-compute state1 data to be written 588 */ 589 pPrecompute->pState1 = hashBlkPtrs.pInHashInitState1; 590 591 /* Set the destination for pre-compute state1 data to be written 592 */ 593 pPrecompute->pState2 = hashBlkPtrs.pInHashInitState2; 594 } 595 /* For digest and nested digest */ 596 else { 597 Cpa32U padLen = pHashControlBlock->inner_state1_sz - 598 pHashDefs->algInfo->stateSize; 599 600 /* counter set to 0 */ 601 hashBlkPtrs.pInHashSetup->auth_counter.counter = 0; 602 603 /* set the inner hash state 1 */ 604 memcpy(hashBlkPtrs.pInHashInitState1, 605 pHashDefs->algInfo->initState, 606 pHashDefs->algInfo->stateSize); 607 608 if (padLen > 0) { 609 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1 + 610 pHashDefs->algInfo->stateSize, 611 padLen); 612 } 613 } 614 615 hashBlkPtrs.pInHashSetup->auth_counter.reserved = 0; 616 617 /* Fill in the outer part of the hash setup block */ 618 if ((CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode || 619 IS_HASH_MODE_2(qatHashMode)) && 620 (NULL != pOuterHashDefs)) { 621 Cpa32U outerConfig = ICP_QAT_HW_AUTH_CONFIG_BUILD( 622 qatHashMode, 623 pOuterHashDefs->qatInfo->algoEnc, 624 pHashSetupData->digestResultLenInBytes); 625 626 Cpa32U padLen = pHashControlBlock->outer_state1_sz - 627 pOuterHashDefs->algInfo->stateSize; 628 629 /* populate the auth config */ 630 hashBlkPtrs.pOutHashSetup->auth_config.config = outerConfig; 631 hashBlkPtrs.pOutHashSetup->auth_config.reserved = 0; 632 633 /* outer Counter set to 0 */ 634 hashBlkPtrs.pOutHashSetup->auth_counter.counter = 0; 635 hashBlkPtrs.pOutHashSetup->auth_counter.reserved = 0; 636 637 /* set outer hash state 1 */ 638 memcpy(hashBlkPtrs.pOutHashInitState1, 639 pOuterHashDefs->algInfo->initState, 640 pOuterHashDefs->algInfo->stateSize); 641 642 if (padLen > 0) { 643 LAC_OS_BZERO(hashBlkPtrs.pOutHashInitState1 + 644 pOuterHashDefs->algInfo->stateSize, 645 padLen); 646 } 647 } 648 649 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pHashSetupData->hashAlgorithm) { 650 icp_qat_hw_cipher_config_t *pCipherConfig = 651 (icp_qat_hw_cipher_config_t *)hashBlkPtrs.pOutHashSetup; 652 653 pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( 654 ICP_QAT_HW_CIPHER_ECB_MODE, 655 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, 656 ICP_QAT_HW_CIPHER_KEY_CONVERT, 657 ICP_QAT_HW_CIPHER_ENCRYPT, 658 aedHashCmpLength); 659 660 pCipherConfig->reserved = 0; 661 662 memcpy((Cpa8U *)pCipherConfig + 663 sizeof(icp_qat_hw_cipher_config_t), 664 pHashSetupData->authModeSetupData.authKey, 665 pHashSetupData->authModeSetupData.authKeyLenInBytes); 666 667 LAC_OS_BZERO( 668 (Cpa8U *)pCipherConfig + 669 sizeof(icp_qat_hw_cipher_config_t) + 670 pHashSetupData->authModeSetupData.authKeyLenInBytes, 671 ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); 672 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pHashSetupData->hashAlgorithm) { 673 icp_qat_hw_cipher_config_t *pCipherConfig = 674 (icp_qat_hw_cipher_config_t *)hashBlkPtrs.pOutHashSetup; 675 676 pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( 677 ICP_QAT_HW_CIPHER_ECB_MODE, 678 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3, 679 ICP_QAT_HW_CIPHER_KEY_CONVERT, 680 ICP_QAT_HW_CIPHER_ENCRYPT, 681 aedHashCmpLength); 682 683 pCipherConfig->reserved = 0; 684 685 memcpy((Cpa8U *)pCipherConfig + 686 sizeof(icp_qat_hw_cipher_config_t), 687 pHashSetupData->authModeSetupData.authKey, 688 pHashSetupData->authModeSetupData.authKeyLenInBytes); 689 690 LAC_OS_BZERO( 691 (Cpa8U *)pCipherConfig + 692 sizeof(icp_qat_hw_cipher_config_t) + 693 pHashSetupData->authModeSetupData.authKeyLenInBytes, 694 ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); 695 } 696 } 697 698 static void 699 LacSymQat_HashOpHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl, 700 void *pHwBlockBase, 701 lac_hash_blk_ptrs_optimised_t *pHashBlkPtrs) 702 { 703 pHashBlkPtrs->pInHashInitState1 = (((Cpa8U *)pHwBlockBase) + 16); 704 pHashBlkPtrs->pInHashInitState2 = 705 (Cpa8U *)(pHashBlkPtrs->pInHashInitState1) + 706 cd_ctrl->inner_state1_sz; 707 } 708 709 static void 710 LacSymQat_HashSetupBlockOptimisedFormatInit( 711 const CpaCySymHashSetupData *pHashSetupData, 712 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 713 void *pHwBlockBase, 714 icp_qat_hw_auth_mode_t qatHashMode, 715 lac_sym_qat_hash_precompute_info_t *pPrecompute, 716 lac_sym_qat_hash_defs_t *pHashDefs, 717 lac_sym_qat_hash_defs_t *pOuterHashDefs) 718 { 719 720 Cpa32U state1PadLen = 0; 721 Cpa32U state2PadLen = 0; 722 723 lac_hash_blk_ptrs_optimised_t pHashBlkPtrs = { 0 }; 724 725 LacSymQat_HashOpHwBlockPtrsInit(pHashControlBlock, 726 pHwBlockBase, 727 &pHashBlkPtrs); 728 729 if (pHashControlBlock->inner_state1_sz > 730 pHashDefs->algInfo->stateSize) { 731 state1PadLen = pHashControlBlock->inner_state1_sz - 732 pHashDefs->algInfo->stateSize; 733 } 734 735 if (pHashControlBlock->inner_state2_sz > 736 pHashDefs->algInfo->stateSize) { 737 state2PadLen = pHashControlBlock->inner_state2_sz - 738 pHashDefs->algInfo->stateSize; 739 } 740 741 if (state1PadLen > 0) { 742 743 LAC_OS_BZERO(pHashBlkPtrs.pInHashInitState1 + 744 pHashDefs->algInfo->stateSize, 745 state1PadLen); 746 } 747 748 if (state2PadLen > 0) { 749 750 LAC_OS_BZERO(pHashBlkPtrs.pInHashInitState2 + 751 pHashDefs->algInfo->stateSize, 752 state2PadLen); 753 } 754 pPrecompute->state1Size = pHashDefs->qatInfo->state1Length; 755 pPrecompute->state2Size = pHashDefs->qatInfo->state2Length; 756 757 /* Set the destination for pre-compute state1 data to be written */ 758 pPrecompute->pState1 = pHashBlkPtrs.pInHashInitState1; 759 760 /* Set the destination for pre-compute state1 data to be written */ 761 pPrecompute->pState2 = pHashBlkPtrs.pInHashInitState2; 762 } 763 764 void 765 LacSymQat_HashStatePrefixAadBufferSizeGet( 766 icp_qat_la_bulk_req_ftr_t *pMsg, 767 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf) 768 { 769 const icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl; 770 icp_qat_la_auth_req_params_t *pHashReqParams; 771 772 cd_ctrl = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 773 pHashReqParams = 774 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 775 776 /* hash state storage needed to support partial packets. Space reserved 777 * for this in all cases */ 778 pHashStateBuf->stateStorageSzQuadWords = LAC_BYTES_TO_QUADWORDS( 779 sizeof(icp_qat_hw_auth_counter_t) + cd_ctrl->inner_state1_sz); 780 781 pHashStateBuf->prefixAadSzQuadWords = pHashReqParams->hash_state_sz; 782 } 783 784 void 785 LacSymQat_HashStatePrefixAadBufferPopulate( 786 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf, 787 icp_qat_la_bulk_req_ftr_t *pMsg, 788 Cpa8U *pInnerPrefixAad, 789 Cpa8U innerPrefixSize, 790 Cpa8U *pOuterPrefix, 791 Cpa8U outerPrefixSize) 792 { 793 const icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = 794 (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 795 796 icp_qat_la_auth_req_params_t *pHashReqParams = 797 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 798 799 /* 800 * Let S be the supplied secret 801 * S1 = S/2 if S is even and (S/2 + 1) if S is odd. 802 * Set length S2 (inner prefix) = S1 and the start address 803 * of S2 is S[S1/2] i.e. if S is odd then S2 starts at the last byte of 804 * S1 805 * _____________________________________________________________ 806 * | outer prefix | padding | 807 * |________________| | 808 * | | 809 * |____________________________________________________________| 810 * | inner prefix | padding | 811 * |________________| | 812 * | | 813 * |____________________________________________________________| 814 * 815 */ 816 if (NULL != pInnerPrefixAad) { 817 Cpa8U *pLocalInnerPrefix = 818 (Cpa8U *)(pHashStateBuf->pData) + 819 LAC_QUADWORDS_TO_BYTES( 820 pHashStateBuf->stateStorageSzQuadWords); 821 Cpa8U padding = 822 pHashReqParams->u2.inner_prefix_sz - innerPrefixSize; 823 /* copy the inner prefix or aad data */ 824 memcpy(pLocalInnerPrefix, pInnerPrefixAad, innerPrefixSize); 825 826 /* Reset with zeroes any area reserved for padding in this block 827 */ 828 if (0 < padding) { 829 LAC_OS_BZERO(pLocalInnerPrefix + innerPrefixSize, 830 padding); 831 } 832 } 833 834 if (NULL != pOuterPrefix) { 835 Cpa8U *pLocalOuterPrefix = 836 (Cpa8U *)pHashStateBuf->pData + 837 LAC_QUADWORDS_TO_BYTES( 838 pHashStateBuf->stateStorageSzQuadWords + 839 cd_ctrl->outer_prefix_offset); 840 Cpa8U padding = LAC_QUADWORDS_TO_BYTES( 841 pHashStateBuf->prefixAadSzQuadWords) - 842 pHashReqParams->u2.inner_prefix_sz - outerPrefixSize; 843 844 /* copy the outer prefix */ 845 memcpy(pLocalOuterPrefix, pOuterPrefix, outerPrefixSize); 846 847 /* Reset with zeroes any area reserved for padding in this block 848 */ 849 if (0 < padding) { 850 LAC_OS_BZERO(pLocalOuterPrefix + outerPrefixSize, 851 padding); 852 } 853 } 854 } 855 856 inline CpaStatus 857 LacSymQat_HashRequestParamsPopulate( 858 icp_qat_fw_la_bulk_req_t *pReq, 859 Cpa32U authOffsetInBytes, 860 Cpa32U authLenInBytes, 861 sal_service_t *pService, 862 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf, 863 Cpa32U packetType, 864 Cpa32U hashResultSize, 865 CpaBoolean digestVerify, 866 Cpa8U *pAuthResult, 867 CpaCySymHashAlgorithm alg, 868 void *pHKDFSecret) 869 { 870 Cpa64U authResultPhys = 0; 871 icp_qat_fw_la_auth_req_params_t *pHashReqParams; 872 873 pHashReqParams = (icp_qat_fw_la_auth_req_params_t 874 *)((Cpa8U *)&(pReq->serv_specif_rqpars) + 875 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 876 877 pHashReqParams->auth_off = authOffsetInBytes; 878 pHashReqParams->auth_len = authLenInBytes; 879 880 /* Set the physical location of secret for HKDF */ 881 if (NULL != pHKDFSecret) { 882 LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL( 883 (*pService), pHashReqParams->u1.aad_adr, pHKDFSecret); 884 885 if (0 == pHashReqParams->u1.aad_adr) { 886 LAC_LOG_ERROR( 887 "Unable to get the physical address of the" 888 " HKDF secret\n"); 889 return CPA_STATUS_FAIL; 890 } 891 } 892 893 /* For a Full packet or last partial need to set the digest result 894 * pointer 895 * and the auth result field */ 896 if (NULL != pAuthResult) { 897 authResultPhys = 898 LAC_OS_VIRT_TO_PHYS_EXTERNAL((*pService), 899 (void *)pAuthResult); 900 901 if (authResultPhys == 0) { 902 LAC_LOG_ERROR( 903 "Unable to get the physical address of the" 904 " auth result\n"); 905 return CPA_STATUS_FAIL; 906 } 907 908 pHashReqParams->auth_res_addr = authResultPhys; 909 } else { 910 pHashReqParams->auth_res_addr = 0; 911 } 912 913 if (CPA_TRUE == digestVerify) { 914 /* auth result size in bytes to be read in for a verify 915 * operation */ 916 pHashReqParams->auth_res_sz = (Cpa8U)hashResultSize; 917 } else { 918 pHashReqParams->auth_res_sz = 0; 919 } 920 921 /* If there is a hash state prefix buffer */ 922 if (NULL != pHashStateBuf) { 923 /* Only write the pointer to the buffer if the size is greater 924 * than 0 925 * this will be the case for plain and auth mode due to the 926 * state storage required for partial packets and for nested 927 * mode (when 928 * the prefix data is > 0) */ 929 if ((pHashStateBuf->stateStorageSzQuadWords + 930 pHashStateBuf->prefixAadSzQuadWords) > 0) { 931 /* For the first partial packet, the QAT expects the 932 * pointer to the 933 * inner prefix even if there is no memory allocated for 934 * this. The 935 * QAT will internally calculate where to write the 936 * state back. */ 937 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || 938 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { 939 // prefix_addr changed to auth_partial_st_prefix 940 pHashReqParams->u1.auth_partial_st_prefix = 941 ((pHashStateBuf->pDataPhys) + 942 LAC_QUADWORDS_TO_BYTES( 943 pHashStateBuf 944 ->stateStorageSzQuadWords)); 945 } else { 946 pHashReqParams->u1.auth_partial_st_prefix = 947 pHashStateBuf->pDataPhys; 948 } 949 } 950 /* nested mode when the prefix data is 0 */ 951 else { 952 pHashReqParams->u1.auth_partial_st_prefix = 0; 953 } 954 955 /* For middle & last partial, state size is the hash state 956 * storage 957 * if hash mode 2 this will include the prefix data */ 958 if ((ICP_QAT_FW_LA_PARTIAL_MID == packetType) || 959 (ICP_QAT_FW_LA_PARTIAL_END == packetType)) { 960 pHashReqParams->hash_state_sz = 961 (pHashStateBuf->stateStorageSzQuadWords + 962 pHashStateBuf->prefixAadSzQuadWords); 963 } 964 /* For full packets and first partials set the state size to 965 * that of 966 * the prefix/aad. prefix includes both the inner and outer 967 * prefix */ 968 else { 969 pHashReqParams->hash_state_sz = 970 pHashStateBuf->prefixAadSzQuadWords; 971 } 972 } else { 973 pHashReqParams->u1.auth_partial_st_prefix = 0; 974 pHashReqParams->hash_state_sz = 0; 975 } 976 977 /* GMAC only */ 978 if (CPA_CY_SYM_HASH_AES_GMAC == alg) { 979 pHashReqParams->hash_state_sz = 0; 980 pHashReqParams->u1.aad_adr = 0; 981 } 982 983 /* This field is only used by TLS requests */ 984 /* In TLS case this is set after this function is called */ 985 pHashReqParams->resrvd1 = 0; 986 return CPA_STATUS_SUCCESS; 987 } 988