1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 5 /** 6 *************************************************************************** 7 * @file lac_sym_qat_hash.c 8 * 9 * @ingroup LacSymQatHash 10 * 11 * Implementation for populating QAT data structures for hash operation 12 ***************************************************************************/ 13 14 /* 15 ******************************************************************************* 16 * Include public/global header files 17 ******************************************************************************* 18 */ 19 20 #include "cpa.h" 21 #include "cpa_cy_sym.h" 22 #include "icp_accel_devices.h" 23 #include "icp_adf_debug.h" 24 #include "lac_log.h" 25 #include "lac_mem.h" 26 #include "lac_sym.h" 27 #include "lac_common.h" 28 #include "lac_sym_qat.h" 29 #include "lac_list.h" 30 #include "lac_sal_types.h" 31 #include "lac_sal_types_crypto.h" 32 #include "lac_sym_qat_hash.h" 33 #include "lac_sym_qat_hash_defs_lookup.h" 34 #include "sal_hw_gen.h" 35 36 /** 37 * This structure contains pointers into the hash setup block of the 38 * security descriptor. As the hash setup block contains fields that 39 * are of variable length, pointers must be calculated to these fields 40 * and the hash setup block is populated using these pointers. */ 41 typedef struct lac_hash_blk_ptrs_s { 42 icp_qat_hw_auth_setup_t *pInHashSetup; 43 /**< inner hash setup */ 44 Cpa8U *pInHashInitState1; 45 /**< inner initial state 1 */ 46 Cpa8U *pInHashInitState2; 47 /**< inner initial state 2 */ 48 icp_qat_hw_auth_setup_t *pOutHashSetup; 49 /**< outer hash setup */ 50 Cpa8U *pOutHashInitState1; 51 /**< outer hash initial state */ 52 } lac_hash_blk_ptrs_t; 53 54 typedef struct lac_hash_blk_ptrs_optimised_s { 55 Cpa8U *pInHashInitState1; 56 /**< inner initial state 1 */ 57 Cpa8U *pInHashInitState2; 58 /**< inner initial state 2 */ 59 60 } lac_hash_blk_ptrs_optimised_t; 61 62 /** 63 * This function calculates the pointers into the hash setup block 64 * based on the control block 65 * 66 * @param[in] pHashControlBlock Pointer to hash control block 67 * @param[in] pHwBlockBase pointer to base of hardware block 68 * @param[out] pHashBlkPtrs structure containing pointers to 69 * various fields in the hash setup block 70 * 71 * @return void 72 */ 73 static void 74 LacSymQat_HashHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 75 void *pHwBlockBase, 76 lac_hash_blk_ptrs_t *pHashBlkPtrs); 77 78 static void 79 LacSymQat_HashSetupBlockOptimisedFormatInit( 80 const CpaCySymHashSetupData *pHashSetupData, 81 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 82 void *pHwBlockBase, 83 icp_qat_hw_auth_mode_t qatHashMode, 84 lac_sym_qat_hash_precompute_info_t *pPrecompute, 85 lac_sym_qat_hash_defs_t *pHashDefs, 86 lac_sym_qat_hash_defs_t *pOuterHashDefs); 87 88 /** 89 * This function populates the hash setup block 90 * 91 * @param[in] pHashSetupData Pointer to the hash context 92 * @param[in] pHashControlBlock Pointer to hash control block 93 * @param[in] pHwBlockBase pointer to base of hardware block 94 * @param[in] qatHashMode QAT hash mode 95 * @param[in] pPrecompute For auth mode, this is the pointer 96 * to the precompute data. Otherwise this 97 * should be set to NULL 98 * @param[in] pHashDefs Pointer to Hash definitions 99 * @param[in] pOuterHashDefs Pointer to Outer Hash definitions. 100 * Required for nested hash mode only 101 * 102 * @return void 103 */ 104 static void 105 LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, 106 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 107 void *pHwBlockBase, 108 icp_qat_hw_auth_mode_t qatHashMode, 109 lac_sym_qat_hash_precompute_info_t *pPrecompute, 110 lac_sym_qat_hash_defs_t *pHashDefs, 111 lac_sym_qat_hash_defs_t *pOuterHashDefs); 112 113 /** @ingroup LacSymQatHash */ 114 void 115 LacSymQat_HashGetCfgData(CpaInstanceHandle pInstance, 116 icp_qat_hw_auth_mode_t qatHashMode, 117 CpaCySymHashMode apiHashMode, 118 CpaCySymHashAlgorithm apiHashAlgorithm, 119 icp_qat_hw_auth_algo_t *pQatAlgorithm, 120 CpaBoolean *pQatNested) 121 { 122 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 123 124 LacSymQat_HashDefsLookupGet(pInstance, apiHashAlgorithm, &pHashDefs); 125 *pQatAlgorithm = pHashDefs->qatInfo->algoEnc; 126 127 if (IS_HASH_MODE_2(qatHashMode)) { 128 /* set bit for nested hashing */ 129 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED; 130 } 131 /* Nested hash in mode 0. */ 132 else if (CPA_CY_SYM_HASH_MODE_NESTED == apiHashMode) { 133 /* set bit for nested hashing */ 134 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED; 135 } 136 /* mode0 - plain or mode1 - auth */ 137 else { 138 *pQatNested = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; 139 } 140 } 141 142 /** @ingroup LacSymQatHash */ 143 void 144 LacSymQat_HashContentDescInit(icp_qat_la_bulk_req_ftr_t *pMsg, 145 CpaInstanceHandle instanceHandle, 146 const CpaCySymHashSetupData *pHashSetupData, 147 void *pHwBlockBase, 148 Cpa32U hwBlockOffsetInQuadWords, 149 icp_qat_fw_slice_t nextSlice, 150 icp_qat_hw_auth_mode_t qatHashMode, 151 CpaBoolean useSymConstantsTable, 152 CpaBoolean useOptimisedContentDesc, 153 CpaBoolean useStatefulSha3ContentDesc, 154 lac_sym_qat_hash_precompute_info_t *pPrecompute, 155 Cpa32U *pHashBlkSizeInBytes) 156 { 157 icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = 158 (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 159 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 160 lac_sym_qat_hash_defs_t *pOuterHashDefs = NULL; 161 Cpa32U hashSetupBlkSize = 0; 162 163 /* setup the offset in QuadWords into the hw blk */ 164 cd_ctrl->hash_cfg_offset = (Cpa8U)hwBlockOffsetInQuadWords; 165 166 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, nextSlice); 167 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 168 169 LacSymQat_HashDefsLookupGet(instanceHandle, 170 pHashSetupData->hashAlgorithm, 171 &pHashDefs); 172 173 /* Hmac in mode 2 TLS */ 174 if (IS_HASH_MODE_2(qatHashMode)) { 175 if (isCyGen4x((sal_crypto_service_t *)instanceHandle)) { 176 /* CPM2.0 has a dedicated bit for HMAC mode2 */ 177 ICP_QAT_FW_HASH_FLAG_MODE2_SET(cd_ctrl->hash_flags, 178 QAT_FW_LA_MODE2); 179 } else { 180 /* Set bit for nested hashing. 181 * Make sure not to overwrite other flags in hash_flags 182 * byte. 183 */ 184 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 185 cd_ctrl->hash_flags, 186 ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); 187 } 188 } 189 /* Nested hash in mode 0 */ 190 else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) { 191 /* Set bit for nested hashing. 192 * Make sure not to overwrite other flags in hash_flags byte. 193 */ 194 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 195 cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED); 196 } 197 /* mode0 - plain or mode1 - auth */ 198 else { 199 ICP_QAT_FW_HASH_FLAG_AUTH_HDR_NESTED_SET( 200 cd_ctrl->hash_flags, ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED); 201 } 202 203 /* Set skip state load flags */ 204 if (useStatefulSha3ContentDesc) { 205 /* Here both skip state load flags are set. FW reads them based 206 * on partial packet type. */ 207 ICP_QAT_FW_HASH_FLAG_SKIP_INNER_STATE1_LOAD_SET( 208 cd_ctrl->hash_flags, QAT_FW_LA_SKIP_INNER_STATE1_LOAD); 209 ICP_QAT_FW_HASH_FLAG_SKIP_OUTER_STATE1_LOAD_SET( 210 cd_ctrl->hash_flags, QAT_FW_LA_SKIP_OUTER_STATE1_LOAD); 211 } 212 213 /* set the final digest size */ 214 cd_ctrl->final_sz = (Cpa8U)pHashSetupData->digestResultLenInBytes; 215 216 /* set the state1 size */ 217 if (useStatefulSha3ContentDesc) { 218 cd_ctrl->inner_state1_sz = 219 LAC_ALIGN_POW2_ROUNDUP(LAC_HASH_SHA3_STATEFUL_STATE_SIZE, 220 LAC_QUAD_WORD_IN_BYTES); 221 } else { 222 cd_ctrl->inner_state1_sz = 223 LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state1Length, 224 LAC_QUAD_WORD_IN_BYTES); 225 } 226 227 /* set the inner result size to the digest length */ 228 cd_ctrl->inner_res_sz = (Cpa8U)pHashDefs->algInfo->digestLength; 229 230 /* set the state2 size - only for mode 1 Auth algos and AES CBC MAC */ 231 if (IS_HASH_MODE_1(qatHashMode) || 232 pHashSetupData->hashAlgorithm == CPA_CY_SYM_HASH_AES_CBC_MAC || 233 pHashSetupData->hashAlgorithm == CPA_CY_SYM_HASH_ZUC_EIA3) { 234 cd_ctrl->inner_state2_sz = 235 LAC_ALIGN_POW2_ROUNDUP(pHashDefs->qatInfo->state2Length, 236 LAC_QUAD_WORD_IN_BYTES); 237 } else { 238 cd_ctrl->inner_state2_sz = 0; 239 } 240 241 if (useSymConstantsTable) { 242 cd_ctrl->inner_state2_offset = 243 LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state1_sz); 244 245 /* size of inner part of hash setup block */ 246 hashSetupBlkSize = 247 cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; 248 } else { 249 cd_ctrl->inner_state2_offset = cd_ctrl->hash_cfg_offset + 250 LAC_BYTES_TO_QUADWORDS(sizeof(icp_qat_hw_auth_setup_t) + 251 cd_ctrl->inner_state1_sz); 252 253 /* size of inner part of hash setup block */ 254 hashSetupBlkSize = sizeof(icp_qat_hw_auth_setup_t) + 255 cd_ctrl->inner_state1_sz + cd_ctrl->inner_state2_sz; 256 } 257 258 /* For nested hashing - Fill in the outer fields */ 259 if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode || 260 IS_HASH_MODE_2(qatHashMode)) { 261 /* For nested - use the outer algorithm. This covers TLS and 262 * nested hash. For HMAC mode2 use inner algorithm again */ 263 CpaCySymHashAlgorithm outerAlg = 264 (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) ? 265 pHashSetupData->nestedModeSetupData.outerHashAlgorithm : 266 pHashSetupData->hashAlgorithm; 267 268 LacSymQat_HashDefsLookupGet(instanceHandle, 269 outerAlg, 270 &pOuterHashDefs); 271 272 /* outer config offset */ 273 cd_ctrl->outer_config_offset = cd_ctrl->inner_state2_offset + 274 LAC_BYTES_TO_QUADWORDS(cd_ctrl->inner_state2_sz); 275 276 if (useStatefulSha3ContentDesc) { 277 cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( 278 LAC_HASH_SHA3_STATEFUL_STATE_SIZE, 279 LAC_QUAD_WORD_IN_BYTES); 280 } else { 281 cd_ctrl->outer_state1_sz = LAC_ALIGN_POW2_ROUNDUP( 282 pOuterHashDefs->algInfo->stateSize, 283 LAC_QUAD_WORD_IN_BYTES); 284 } 285 286 /* outer result size */ 287 cd_ctrl->outer_res_sz = 288 (Cpa8U)pOuterHashDefs->algInfo->digestLength; 289 290 /* outer_prefix_offset will be the size of the inner prefix data 291 * plus the hash state storage size. */ 292 /* The prefix buffer is part of the ReqParams, so this param 293 * will be setup where ReqParams are set up */ 294 295 /* add on size of outer part of hash block */ 296 hashSetupBlkSize += 297 sizeof(icp_qat_hw_auth_setup_t) + cd_ctrl->outer_state1_sz; 298 } else { 299 cd_ctrl->outer_config_offset = 0; 300 cd_ctrl->outer_state1_sz = 0; 301 cd_ctrl->outer_res_sz = 0; 302 } 303 304 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pHashSetupData->hashAlgorithm) { 305 /* add the size for the cipher config word, the key and the IV*/ 306 hashSetupBlkSize += sizeof(icp_qat_hw_cipher_config_t) + 307 pHashSetupData->authModeSetupData.authKeyLenInBytes + 308 ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; 309 } 310 311 *pHashBlkSizeInBytes = hashSetupBlkSize; 312 313 if (useOptimisedContentDesc) { 314 LacSymQat_HashSetupBlockOptimisedFormatInit(pHashSetupData, 315 cd_ctrl, 316 pHwBlockBase, 317 qatHashMode, 318 pPrecompute, 319 pHashDefs, 320 pOuterHashDefs); 321 } else if (!useSymConstantsTable) { 322 /***************************************************************************** 323 * Populate Hash Setup block * 324 *****************************************************************************/ 325 LacSymQat_HashSetupBlockInit(pHashSetupData, 326 cd_ctrl, 327 pHwBlockBase, 328 qatHashMode, 329 pPrecompute, 330 pHashDefs, 331 pOuterHashDefs); 332 } 333 } 334 335 /* This fn populates fields in both the CD ctrl block and the ReqParams block 336 * which describe the Hash ReqParams: 337 * cd_ctrl.outer_prefix_offset 338 * cd_ctrl.outer_prefix_sz 339 * req_params.inner_prefix_sz/aad_sz 340 * req_params.hash_state_sz 341 * req_params.auth_res_sz 342 * 343 */ 344 void 345 LacSymQat_HashSetupReqParamsMetaData( 346 icp_qat_la_bulk_req_ftr_t *pMsg, 347 CpaInstanceHandle instanceHandle, 348 const CpaCySymHashSetupData *pHashSetupData, 349 CpaBoolean hashStateBuffer, 350 icp_qat_hw_auth_mode_t qatHashMode, 351 CpaBoolean digestVerify) 352 { 353 icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = NULL; 354 icp_qat_la_auth_req_params_t *pHashReqParams = NULL; 355 lac_sym_qat_hash_defs_t *pHashDefs = NULL; 356 357 cd_ctrl = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 358 pHashReqParams = 359 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 360 361 LacSymQat_HashDefsLookupGet(instanceHandle, 362 pHashSetupData->hashAlgorithm, 363 &pHashDefs); 364 365 /* Hmac in mode 2 TLS */ 366 if (IS_HASH_MODE_2(qatHashMode)) { 367 /* Inner and outer prefixes are the block length */ 368 pHashReqParams->u2.inner_prefix_sz = 369 (Cpa8U)pHashDefs->algInfo->blockLength; 370 cd_ctrl->outer_prefix_sz = 371 (Cpa8U)pHashDefs->algInfo->blockLength; 372 cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( 373 LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), 374 LAC_QUAD_WORD_IN_BYTES)); 375 } 376 /* Nested hash in mode 0 */ 377 else if (CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode) { 378 379 /* set inner and outer prefixes */ 380 pHashReqParams->u2.inner_prefix_sz = 381 (Cpa8U)pHashSetupData->nestedModeSetupData 382 .innerPrefixLenInBytes; 383 cd_ctrl->outer_prefix_sz = 384 (Cpa8U)pHashSetupData->nestedModeSetupData 385 .outerPrefixLenInBytes; 386 cd_ctrl->outer_prefix_offset = LAC_BYTES_TO_QUADWORDS( 387 LAC_ALIGN_POW2_ROUNDUP((pHashReqParams->u2.inner_prefix_sz), 388 LAC_QUAD_WORD_IN_BYTES)); 389 } 390 /* mode0 - plain or mode1 - auth */ 391 else { 392 Cpa16U aadDataSize = 0; 393 394 /* For Auth Encrypt set the aad size */ 395 if (CPA_CY_SYM_HASH_AES_CCM == pHashSetupData->hashAlgorithm) { 396 /* at the beginning of the buffer there is B0 block */ 397 aadDataSize = LAC_HASH_AES_CCM_BLOCK_SIZE; 398 399 /* then, if there is some 'a' data, the buffer will 400 * store encoded 401 * length of 'a' and 'a' itself */ 402 if (pHashSetupData->authModeSetupData.aadLenInBytes > 403 0) { 404 /* as the QAT API puts the requirement on the 405 * pAdditionalAuthData not to be bigger than 240 406 * bytes then we 407 * just need 2 bytes to store encoded length of 408 * 'a' */ 409 aadDataSize += sizeof(Cpa16U); 410 aadDataSize += 411 (Cpa16U)pHashSetupData->authModeSetupData 412 .aadLenInBytes; 413 } 414 415 /* round the aad size to the multiple of CCM block 416 * size.*/ 417 pHashReqParams->u2.aad_sz = 418 LAC_ALIGN_POW2_ROUNDUP(aadDataSize, 419 LAC_HASH_AES_CCM_BLOCK_SIZE); 420 } else if (CPA_CY_SYM_HASH_AES_GCM == 421 pHashSetupData->hashAlgorithm) { 422 aadDataSize = 423 (Cpa16U) 424 pHashSetupData->authModeSetupData.aadLenInBytes; 425 426 /* round the aad size to the multiple of GCM hash block 427 * size. */ 428 pHashReqParams->u2.aad_sz = 429 LAC_ALIGN_POW2_ROUNDUP(aadDataSize, 430 LAC_HASH_AES_GCM_BLOCK_SIZE); 431 } else { 432 pHashReqParams->u2.aad_sz = 0; 433 } 434 435 cd_ctrl->outer_prefix_sz = 0; 436 cd_ctrl->outer_prefix_offset = 0; 437 } 438 439 /* If there is a hash state prefix buffer */ 440 if (CPA_TRUE == hashStateBuffer) { 441 /* Note, this sets up size for both aad and non-aad cases */ 442 pHashReqParams->hash_state_sz = LAC_BYTES_TO_QUADWORDS( 443 LAC_ALIGN_POW2_ROUNDUP(pHashReqParams->u2.inner_prefix_sz, 444 LAC_QUAD_WORD_IN_BYTES) + 445 LAC_ALIGN_POW2_ROUNDUP(cd_ctrl->outer_prefix_sz, 446 LAC_QUAD_WORD_IN_BYTES)); 447 } else { 448 pHashReqParams->hash_state_sz = 0; 449 } 450 451 if (CPA_TRUE == digestVerify) { 452 /* auth result size in bytes to be read in for a verify 453 * operation */ 454 pHashReqParams->auth_res_sz = 455 (Cpa8U)pHashSetupData->digestResultLenInBytes; 456 } else { 457 pHashReqParams->auth_res_sz = 0; 458 } 459 460 pHashReqParams->resrvd1 = 0; 461 } 462 463 void 464 LacSymQat_HashHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl, 465 void *pHwBlockBase, 466 lac_hash_blk_ptrs_t *pHashBlkPtrs) 467 { 468 /* encoded offset for inner config is converted to a byte offset. */ 469 pHashBlkPtrs->pInHashSetup = 470 (icp_qat_hw_auth_setup_t *)((Cpa8U *)pHwBlockBase + 471 (cd_ctrl->hash_cfg_offset * 472 LAC_QUAD_WORD_IN_BYTES)); 473 474 pHashBlkPtrs->pInHashInitState1 = (Cpa8U *)pHashBlkPtrs->pInHashSetup + 475 sizeof(icp_qat_hw_auth_setup_t); 476 477 pHashBlkPtrs->pInHashInitState2 = 478 (Cpa8U *)(pHashBlkPtrs->pInHashInitState1) + 479 cd_ctrl->inner_state1_sz; 480 481 pHashBlkPtrs->pOutHashSetup = 482 (icp_qat_hw_auth_setup_t *)((Cpa8U *)(pHashBlkPtrs 483 ->pInHashInitState2) + 484 cd_ctrl->inner_state2_sz); 485 486 pHashBlkPtrs->pOutHashInitState1 = 487 (Cpa8U *)(pHashBlkPtrs->pOutHashSetup) + 488 sizeof(icp_qat_hw_auth_setup_t); 489 } 490 491 static void 492 LacSymQat_HashSetupBlockInit(const CpaCySymHashSetupData *pHashSetupData, 493 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 494 void *pHwBlockBase, 495 icp_qat_hw_auth_mode_t qatHashMode, 496 lac_sym_qat_hash_precompute_info_t *pPrecompute, 497 lac_sym_qat_hash_defs_t *pHashDefs, 498 lac_sym_qat_hash_defs_t *pOuterHashDefs) 499 { 500 Cpa32U innerConfig = 0; 501 lac_hash_blk_ptrs_t hashBlkPtrs = { 0 }; 502 Cpa32U aedHashCmpLength = 0; 503 504 LacSymQat_HashHwBlockPtrsInit(pHashControlBlock, 505 pHwBlockBase, 506 &hashBlkPtrs); 507 508 innerConfig = ICP_QAT_HW_AUTH_CONFIG_BUILD( 509 qatHashMode, 510 pHashDefs->qatInfo->algoEnc, 511 pHashSetupData->digestResultLenInBytes); 512 513 /* Set the Inner hash configuration */ 514 hashBlkPtrs.pInHashSetup->auth_config.config = innerConfig; 515 hashBlkPtrs.pInHashSetup->auth_config.reserved = 0; 516 517 /* For mode 1 pre-computes for auth algorithms */ 518 if (IS_HASH_MODE_1(qatHashMode) || 519 CPA_CY_SYM_HASH_AES_CBC_MAC == pHashSetupData->hashAlgorithm || 520 CPA_CY_SYM_HASH_ZUC_EIA3 == pHashSetupData->hashAlgorithm) { 521 /* for HMAC in mode 1 authCounter is the block size 522 * else the authCounter is 0. The firmware expects the counter 523 * to be 524 * big endian */ 525 LAC_MEM_SHARED_WRITE_SWAP( 526 hashBlkPtrs.pInHashSetup->auth_counter.counter, 527 pHashDefs->qatInfo->authCounter); 528 529 /* state 1 is set to 0 for the following algorithms */ 530 if ((CPA_CY_SYM_HASH_AES_XCBC == 531 pHashSetupData->hashAlgorithm) || 532 (CPA_CY_SYM_HASH_AES_CMAC == 533 pHashSetupData->hashAlgorithm) || 534 (CPA_CY_SYM_HASH_AES_CBC_MAC == 535 pHashSetupData->hashAlgorithm) || 536 (CPA_CY_SYM_HASH_KASUMI_F9 == 537 pHashSetupData->hashAlgorithm) || 538 (CPA_CY_SYM_HASH_SNOW3G_UIA2 == 539 pHashSetupData->hashAlgorithm) || 540 (CPA_CY_SYM_HASH_AES_CCM == 541 pHashSetupData->hashAlgorithm) || 542 (CPA_CY_SYM_HASH_AES_GMAC == 543 pHashSetupData->hashAlgorithm) || 544 (CPA_CY_SYM_HASH_AES_GCM == 545 pHashSetupData->hashAlgorithm) || 546 (CPA_CY_SYM_HASH_ZUC_EIA3 == 547 pHashSetupData->hashAlgorithm)) { 548 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1, 549 pHashDefs->qatInfo->state1Length); 550 } 551 552 /* Pad remaining bytes of sha1 precomputes */ 553 if (CPA_CY_SYM_HASH_SHA1 == pHashSetupData->hashAlgorithm) { 554 Cpa32U state1PadLen = 0; 555 Cpa32U state2PadLen = 0; 556 557 if (pHashControlBlock->inner_state1_sz > 558 pHashDefs->algInfo->stateSize) { 559 state1PadLen = 560 pHashControlBlock->inner_state1_sz - 561 pHashDefs->algInfo->stateSize; 562 } 563 564 if (pHashControlBlock->inner_state2_sz > 565 pHashDefs->algInfo->stateSize) { 566 state2PadLen = 567 pHashControlBlock->inner_state2_sz - 568 pHashDefs->algInfo->stateSize; 569 } 570 571 if (state1PadLen > 0) { 572 573 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1 + 574 pHashDefs->algInfo->stateSize, 575 state1PadLen); 576 } 577 578 if (state2PadLen > 0) { 579 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState2 + 580 pHashDefs->algInfo->stateSize, 581 state2PadLen); 582 } 583 } 584 585 pPrecompute->state1Size = pHashDefs->qatInfo->state1Length; 586 pPrecompute->state2Size = pHashDefs->qatInfo->state2Length; 587 588 /* Set the destination for pre-compute state1 data to be written 589 */ 590 pPrecompute->pState1 = hashBlkPtrs.pInHashInitState1; 591 592 /* Set the destination for pre-compute state1 data to be written 593 */ 594 pPrecompute->pState2 = hashBlkPtrs.pInHashInitState2; 595 } 596 /* For digest and nested digest */ 597 else { 598 Cpa32U padLen = pHashControlBlock->inner_state1_sz - 599 pHashDefs->algInfo->stateSize; 600 601 /* counter set to 0 */ 602 hashBlkPtrs.pInHashSetup->auth_counter.counter = 0; 603 604 /* set the inner hash state 1 */ 605 memcpy(hashBlkPtrs.pInHashInitState1, 606 pHashDefs->algInfo->initState, 607 pHashDefs->algInfo->stateSize); 608 609 if (padLen > 0) { 610 LAC_OS_BZERO(hashBlkPtrs.pInHashInitState1 + 611 pHashDefs->algInfo->stateSize, 612 padLen); 613 } 614 } 615 616 hashBlkPtrs.pInHashSetup->auth_counter.reserved = 0; 617 618 /* Fill in the outer part of the hash setup block */ 619 if ((CPA_CY_SYM_HASH_MODE_NESTED == pHashSetupData->hashMode || 620 IS_HASH_MODE_2(qatHashMode)) && 621 (NULL != pOuterHashDefs)) { 622 Cpa32U outerConfig = ICP_QAT_HW_AUTH_CONFIG_BUILD( 623 qatHashMode, 624 pOuterHashDefs->qatInfo->algoEnc, 625 pHashSetupData->digestResultLenInBytes); 626 627 Cpa32U padLen = pHashControlBlock->outer_state1_sz - 628 pOuterHashDefs->algInfo->stateSize; 629 630 /* populate the auth config */ 631 hashBlkPtrs.pOutHashSetup->auth_config.config = outerConfig; 632 hashBlkPtrs.pOutHashSetup->auth_config.reserved = 0; 633 634 /* outer Counter set to 0 */ 635 hashBlkPtrs.pOutHashSetup->auth_counter.counter = 0; 636 hashBlkPtrs.pOutHashSetup->auth_counter.reserved = 0; 637 638 /* set outer hash state 1 */ 639 memcpy(hashBlkPtrs.pOutHashInitState1, 640 pOuterHashDefs->algInfo->initState, 641 pOuterHashDefs->algInfo->stateSize); 642 643 if (padLen > 0) { 644 LAC_OS_BZERO(hashBlkPtrs.pOutHashInitState1 + 645 pOuterHashDefs->algInfo->stateSize, 646 padLen); 647 } 648 } 649 650 if (CPA_CY_SYM_HASH_SNOW3G_UIA2 == pHashSetupData->hashAlgorithm) { 651 icp_qat_hw_cipher_config_t *pCipherConfig = 652 (icp_qat_hw_cipher_config_t *)hashBlkPtrs.pOutHashSetup; 653 654 pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( 655 ICP_QAT_HW_CIPHER_ECB_MODE, 656 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, 657 ICP_QAT_HW_CIPHER_KEY_CONVERT, 658 ICP_QAT_HW_CIPHER_ENCRYPT, 659 aedHashCmpLength); 660 661 pCipherConfig->reserved = 0; 662 663 memcpy((Cpa8U *)pCipherConfig + 664 sizeof(icp_qat_hw_cipher_config_t), 665 pHashSetupData->authModeSetupData.authKey, 666 pHashSetupData->authModeSetupData.authKeyLenInBytes); 667 668 LAC_OS_BZERO( 669 (Cpa8U *)pCipherConfig + 670 sizeof(icp_qat_hw_cipher_config_t) + 671 pHashSetupData->authModeSetupData.authKeyLenInBytes, 672 ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); 673 } else if (CPA_CY_SYM_HASH_ZUC_EIA3 == pHashSetupData->hashAlgorithm) { 674 icp_qat_hw_cipher_config_t *pCipherConfig = 675 (icp_qat_hw_cipher_config_t *)hashBlkPtrs.pOutHashSetup; 676 677 pCipherConfig->val = ICP_QAT_HW_CIPHER_CONFIG_BUILD( 678 ICP_QAT_HW_CIPHER_ECB_MODE, 679 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3, 680 ICP_QAT_HW_CIPHER_KEY_CONVERT, 681 ICP_QAT_HW_CIPHER_ENCRYPT, 682 aedHashCmpLength); 683 684 pCipherConfig->reserved = 0; 685 686 memcpy((Cpa8U *)pCipherConfig + 687 sizeof(icp_qat_hw_cipher_config_t), 688 pHashSetupData->authModeSetupData.authKey, 689 pHashSetupData->authModeSetupData.authKeyLenInBytes); 690 691 LAC_OS_BZERO( 692 (Cpa8U *)pCipherConfig + 693 sizeof(icp_qat_hw_cipher_config_t) + 694 pHashSetupData->authModeSetupData.authKeyLenInBytes, 695 ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); 696 } 697 } 698 699 static void 700 LacSymQat_HashOpHwBlockPtrsInit(icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl, 701 void *pHwBlockBase, 702 lac_hash_blk_ptrs_optimised_t *pHashBlkPtrs) 703 { 704 pHashBlkPtrs->pInHashInitState1 = (((Cpa8U *)pHwBlockBase) + 16); 705 pHashBlkPtrs->pInHashInitState2 = 706 (Cpa8U *)(pHashBlkPtrs->pInHashInitState1) + 707 cd_ctrl->inner_state1_sz; 708 } 709 710 static void 711 LacSymQat_HashSetupBlockOptimisedFormatInit( 712 const CpaCySymHashSetupData *pHashSetupData, 713 icp_qat_fw_auth_cd_ctrl_hdr_t *pHashControlBlock, 714 void *pHwBlockBase, 715 icp_qat_hw_auth_mode_t qatHashMode, 716 lac_sym_qat_hash_precompute_info_t *pPrecompute, 717 lac_sym_qat_hash_defs_t *pHashDefs, 718 lac_sym_qat_hash_defs_t *pOuterHashDefs) 719 { 720 721 Cpa32U state1PadLen = 0; 722 Cpa32U state2PadLen = 0; 723 724 lac_hash_blk_ptrs_optimised_t pHashBlkPtrs = { 0 }; 725 726 LacSymQat_HashOpHwBlockPtrsInit(pHashControlBlock, 727 pHwBlockBase, 728 &pHashBlkPtrs); 729 730 if (pHashControlBlock->inner_state1_sz > 731 pHashDefs->algInfo->stateSize) { 732 state1PadLen = pHashControlBlock->inner_state1_sz - 733 pHashDefs->algInfo->stateSize; 734 } 735 736 if (pHashControlBlock->inner_state2_sz > 737 pHashDefs->algInfo->stateSize) { 738 state2PadLen = pHashControlBlock->inner_state2_sz - 739 pHashDefs->algInfo->stateSize; 740 } 741 742 if (state1PadLen > 0) { 743 744 LAC_OS_BZERO(pHashBlkPtrs.pInHashInitState1 + 745 pHashDefs->algInfo->stateSize, 746 state1PadLen); 747 } 748 749 if (state2PadLen > 0) { 750 751 LAC_OS_BZERO(pHashBlkPtrs.pInHashInitState2 + 752 pHashDefs->algInfo->stateSize, 753 state2PadLen); 754 } 755 pPrecompute->state1Size = pHashDefs->qatInfo->state1Length; 756 pPrecompute->state2Size = pHashDefs->qatInfo->state2Length; 757 758 /* Set the destination for pre-compute state1 data to be written */ 759 pPrecompute->pState1 = pHashBlkPtrs.pInHashInitState1; 760 761 /* Set the destination for pre-compute state1 data to be written */ 762 pPrecompute->pState2 = pHashBlkPtrs.pInHashInitState2; 763 } 764 765 void 766 LacSymQat_HashStatePrefixAadBufferSizeGet( 767 icp_qat_la_bulk_req_ftr_t *pMsg, 768 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf) 769 { 770 const icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl; 771 icp_qat_la_auth_req_params_t *pHashReqParams; 772 773 cd_ctrl = (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 774 pHashReqParams = 775 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 776 777 /* hash state storage needed to support partial packets. Space reserved 778 * for this in all cases */ 779 pHashStateBuf->stateStorageSzQuadWords = LAC_BYTES_TO_QUADWORDS( 780 sizeof(icp_qat_hw_auth_counter_t) + cd_ctrl->inner_state1_sz); 781 782 pHashStateBuf->prefixAadSzQuadWords = pHashReqParams->hash_state_sz; 783 } 784 785 void 786 LacSymQat_HashStatePrefixAadBufferPopulate( 787 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf, 788 icp_qat_la_bulk_req_ftr_t *pMsg, 789 Cpa8U *pInnerPrefixAad, 790 Cpa8U innerPrefixSize, 791 Cpa8U *pOuterPrefix, 792 Cpa8U outerPrefixSize) 793 { 794 const icp_qat_fw_auth_cd_ctrl_hdr_t *cd_ctrl = 795 (icp_qat_fw_auth_cd_ctrl_hdr_t *)&(pMsg->cd_ctrl); 796 797 icp_qat_la_auth_req_params_t *pHashReqParams = 798 (icp_qat_la_auth_req_params_t *)(&(pMsg->serv_specif_rqpars)); 799 800 /* 801 * Let S be the supplied secret 802 * S1 = S/2 if S is even and (S/2 + 1) if S is odd. 803 * Set length S2 (inner prefix) = S1 and the start address 804 * of S2 is S[S1/2] i.e. if S is odd then S2 starts at the last byte of 805 * S1 806 * _____________________________________________________________ 807 * | outer prefix | padding | 808 * |________________| | 809 * | | 810 * |____________________________________________________________| 811 * | inner prefix | padding | 812 * |________________| | 813 * | | 814 * |____________________________________________________________| 815 * 816 */ 817 if (NULL != pInnerPrefixAad) { 818 Cpa8U *pLocalInnerPrefix = 819 (Cpa8U *)(pHashStateBuf->pData) + 820 LAC_QUADWORDS_TO_BYTES( 821 pHashStateBuf->stateStorageSzQuadWords); 822 Cpa8U padding = 823 pHashReqParams->u2.inner_prefix_sz - innerPrefixSize; 824 /* copy the inner prefix or aad data */ 825 memcpy(pLocalInnerPrefix, pInnerPrefixAad, innerPrefixSize); 826 827 /* Reset with zeroes any area reserved for padding in this block 828 */ 829 if (0 < padding) { 830 LAC_OS_BZERO(pLocalInnerPrefix + innerPrefixSize, 831 padding); 832 } 833 } 834 835 if (NULL != pOuterPrefix) { 836 Cpa8U *pLocalOuterPrefix = 837 (Cpa8U *)pHashStateBuf->pData + 838 LAC_QUADWORDS_TO_BYTES( 839 pHashStateBuf->stateStorageSzQuadWords + 840 cd_ctrl->outer_prefix_offset); 841 Cpa8U padding = LAC_QUADWORDS_TO_BYTES( 842 pHashStateBuf->prefixAadSzQuadWords) - 843 pHashReqParams->u2.inner_prefix_sz - outerPrefixSize; 844 845 /* copy the outer prefix */ 846 memcpy(pLocalOuterPrefix, pOuterPrefix, outerPrefixSize); 847 848 /* Reset with zeroes any area reserved for padding in this block 849 */ 850 if (0 < padding) { 851 LAC_OS_BZERO(pLocalOuterPrefix + outerPrefixSize, 852 padding); 853 } 854 } 855 } 856 857 inline CpaStatus 858 LacSymQat_HashRequestParamsPopulate( 859 icp_qat_fw_la_bulk_req_t *pReq, 860 Cpa32U authOffsetInBytes, 861 Cpa32U authLenInBytes, 862 sal_service_t *pService, 863 lac_sym_qat_hash_state_buffer_info_t *pHashStateBuf, 864 Cpa32U packetType, 865 Cpa32U hashResultSize, 866 CpaBoolean digestVerify, 867 Cpa8U *pAuthResult, 868 CpaCySymHashAlgorithm alg, 869 void *pHKDFSecret) 870 { 871 Cpa64U authResultPhys = 0; 872 icp_qat_fw_la_auth_req_params_t *pHashReqParams; 873 874 pHashReqParams = (icp_qat_fw_la_auth_req_params_t 875 *)((Cpa8U *)&(pReq->serv_specif_rqpars) + 876 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 877 878 pHashReqParams->auth_off = authOffsetInBytes; 879 pHashReqParams->auth_len = authLenInBytes; 880 881 /* Set the physical location of secret for HKDF */ 882 if (NULL != pHKDFSecret) { 883 LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL( 884 (*pService), pHashReqParams->u1.aad_adr, pHKDFSecret); 885 886 if (0 == pHashReqParams->u1.aad_adr) { 887 LAC_LOG_ERROR( 888 "Unable to get the physical address of the" 889 " HKDF secret\n"); 890 return CPA_STATUS_FAIL; 891 } 892 } 893 894 /* For a Full packet or last partial need to set the digest result 895 * pointer 896 * and the auth result field */ 897 if (NULL != pAuthResult) { 898 authResultPhys = 899 LAC_OS_VIRT_TO_PHYS_EXTERNAL((*pService), 900 (void *)pAuthResult); 901 902 if (authResultPhys == 0) { 903 LAC_LOG_ERROR( 904 "Unable to get the physical address of the" 905 " auth result\n"); 906 return CPA_STATUS_FAIL; 907 } 908 909 pHashReqParams->auth_res_addr = authResultPhys; 910 } else { 911 pHashReqParams->auth_res_addr = 0; 912 } 913 914 if (CPA_TRUE == digestVerify) { 915 /* auth result size in bytes to be read in for a verify 916 * operation */ 917 pHashReqParams->auth_res_sz = (Cpa8U)hashResultSize; 918 } else { 919 pHashReqParams->auth_res_sz = 0; 920 } 921 922 /* If there is a hash state prefix buffer */ 923 if (NULL != pHashStateBuf) { 924 /* Only write the pointer to the buffer if the size is greater 925 * than 0 926 * this will be the case for plain and auth mode due to the 927 * state storage required for partial packets and for nested 928 * mode (when 929 * the prefix data is > 0) */ 930 if ((pHashStateBuf->stateStorageSzQuadWords + 931 pHashStateBuf->prefixAadSzQuadWords) > 0) { 932 /* For the first partial packet, the QAT expects the 933 * pointer to the 934 * inner prefix even if there is no memory allocated for 935 * this. The 936 * QAT will internally calculate where to write the 937 * state back. */ 938 if ((ICP_QAT_FW_LA_PARTIAL_START == packetType) || 939 (ICP_QAT_FW_LA_PARTIAL_NONE == packetType)) { 940 // prefix_addr changed to auth_partial_st_prefix 941 pHashReqParams->u1.auth_partial_st_prefix = 942 ((pHashStateBuf->pDataPhys) + 943 LAC_QUADWORDS_TO_BYTES( 944 pHashStateBuf 945 ->stateStorageSzQuadWords)); 946 } else { 947 pHashReqParams->u1.auth_partial_st_prefix = 948 pHashStateBuf->pDataPhys; 949 } 950 } 951 /* nested mode when the prefix data is 0 */ 952 else { 953 pHashReqParams->u1.auth_partial_st_prefix = 0; 954 } 955 956 /* For middle & last partial, state size is the hash state 957 * storage 958 * if hash mode 2 this will include the prefix data */ 959 if ((ICP_QAT_FW_LA_PARTIAL_MID == packetType) || 960 (ICP_QAT_FW_LA_PARTIAL_END == packetType)) { 961 pHashReqParams->hash_state_sz = 962 (pHashStateBuf->stateStorageSzQuadWords + 963 pHashStateBuf->prefixAadSzQuadWords); 964 } 965 /* For full packets and first partials set the state size to 966 * that of 967 * the prefix/aad. prefix includes both the inner and outer 968 * prefix */ 969 else { 970 pHashReqParams->hash_state_sz = 971 pHashStateBuf->prefixAadSzQuadWords; 972 } 973 } else { 974 pHashReqParams->u1.auth_partial_st_prefix = 0; 975 pHashReqParams->hash_state_sz = 0; 976 } 977 978 /* GMAC only */ 979 if (CPA_CY_SYM_HASH_AES_GMAC == alg) { 980 pHashReqParams->hash_state_sz = 0; 981 pHashReqParams->u1.aad_adr = 0; 982 } 983 984 /* This field is only used by TLS requests */ 985 /* In TLS case this is set after this function is called */ 986 pHashReqParams->resrvd1 = 0; 987 return CPA_STATUS_SUCCESS; 988 } 989