1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 4 /** 5 *************************************************************************** 6 * @file lac_sym_cb.c Callback handler functions for symmetric components 7 * 8 * @ingroup LacSym 9 * 10 ***************************************************************************/ 11 12 /* 13 ******************************************************************************* 14 * Include public/global header files 15 ******************************************************************************* 16 */ 17 18 #include "cpa.h" 19 #include "cpa_cy_sym.h" 20 21 #include "icp_accel_devices.h" 22 #include "icp_adf_init.h" 23 #include "icp_qat_fw_la.h" 24 #include "icp_adf_transport.h" 25 #include "icp_adf_debug.h" 26 27 #include "lac_sym.h" 28 #include "lac_sym_cipher.h" 29 #include "lac_common.h" 30 #include "lac_list.h" 31 #include "lac_sal_types_crypto.h" 32 #include "lac_sal.h" 33 #include "lac_sal_ctrl.h" 34 #include "lac_session.h" 35 #include "lac_sym_stats.h" 36 #include "lac_log.h" 37 #include "lac_sym_cb.h" 38 #include "lac_sym_hash.h" 39 #include "lac_sym_qat_cipher.h" 40 #include "lac_sym_qat.h" 41 42 #define DEQUEUE_MSGPUT_MAX_RETRIES 10000 43 44 /* 45 ******************************************************************************* 46 * Define static function definitions 47 ******************************************************************************* 48 */ 49 50 /** 51 ***************************************************************************** 52 * @ingroup LacSymCb 53 * Function to clean computed data. 54 * 55 * @description 56 * This function cleans GCM or CCM data in the case of a failure. 57 * 58 * @param[in] pSessionDesc pointer to the session descriptor 59 * @param[out] pBufferList pointer to the bufferlist to clean 60 * @param[in] pOpData pointer to operation data 61 * @param[in] isCCM is it a CCM operation boolean 62 * 63 * @return None 64 *****************************************************************************/ 65 static void 66 LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc, 67 CpaBufferList *pBufferList, 68 const CpaCySymOpData *pOpData, 69 CpaBoolean isCCM) 70 { 71 Cpa32U authTagLen = 0; 72 73 /* Retrieve authTagLen */ 74 authTagLen = pSessionDesc->hashResultSize; 75 76 /* Cleaning */ 77 if (isCCM) { 78 /* for CCM the digest is inside the buffer list */ 79 LacBuffDesc_BufferListZeroFromOffset( 80 pBufferList, 81 pOpData->cryptoStartSrcOffsetInBytes, 82 pOpData->messageLenToCipherInBytes + authTagLen); 83 } else { 84 /* clean buffer list */ 85 LacBuffDesc_BufferListZeroFromOffset( 86 pBufferList, 87 pOpData->cryptoStartSrcOffsetInBytes, 88 pOpData->messageLenToCipherInBytes); 89 } 90 if ((CPA_TRUE != pSessionDesc->digestIsAppended) && 91 (NULL != pOpData->pDigestResult)) { 92 /* clean digest */ 93 memset(pOpData->pDigestResult, 0, authTagLen); 94 } 95 } 96 97 /** 98 ***************************************************************************** 99 * @ingroup LacSymCb 100 * Definition of callback function for processing symmetric responses 101 * 102 * @description 103 * This callback is invoked to process symmetric response messages from 104 * the QAT. It will extract some details from the message and invoke 105 * the user's callback to complete a symmetric operation. 106 * 107 * @param[in] pCookie Pointer to cookie associated with this request 108 * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT 109 * @param[in] status Status variable indicating an error occurred 110 * in sending the message (e.g. when dequeueing) 111 * @param[in] pSessionDesc Session descriptor 112 * 113 * @return None 114 *****************************************************************************/ 115 static void 116 LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, 117 CpaBoolean qatRespStatusOkFlag, 118 CpaStatus status, 119 lac_session_desc_t *pSessionDesc) 120 { 121 CpaCySymCbFunc pSymCb = NULL; 122 void *pCallbackTag = NULL; 123 CpaCySymOpData *pOpData = NULL; 124 CpaBufferList *pDstBuffer = NULL; 125 CpaCySymOp operationType = CPA_CY_SYM_OP_NONE; 126 CpaStatus dequeueStatus = CPA_STATUS_SUCCESS; 127 128 CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE; 129 /* NOTE: cookie pointer validated in previous function */ 130 instanceHandle = pCookie->instanceHandle; 131 132 pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData); 133 operationType = pSessionDesc->symOperation; 134 135 /* Set the destination pointer to the one supplied in the cookie. */ 136 pDstBuffer = pCookie->pDstBuffer; 137 138 /* For a digest verify operation - for full packet and final partial 139 * only, perform a comparison with the digest generated and with the one 140 * supplied in the packet. In case of AES_GCM in SPC mode, destination 141 * buffer needs to be cleared if digest verify operation fails */ 142 143 if (((SPC == pSessionDesc->singlePassState) || 144 (CPA_CY_SYM_OP_CIPHER != operationType)) && 145 (CPA_TRUE == pSessionDesc->digestVerify) && 146 ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) || 147 (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) { 148 if (CPA_FALSE == qatRespStatusOkFlag) { 149 LAC_SYM_STAT_INC(numSymOpVerifyFailures, 150 instanceHandle); 151 152 /* The comparison has failed at this point (status is 153 * fail), need to clean any sensitive calculated data up 154 * to this point. The data calculated is no longer 155 * useful to the end result and does not need to be 156 * returned to the user so setting buffers to zero. 157 */ 158 if (pSessionDesc->cipherAlgorithm == 159 CPA_CY_SYM_CIPHER_AES_CCM) { 160 LacSymCb_CleanUserData(pSessionDesc, 161 pDstBuffer, 162 pOpData, 163 CPA_TRUE); 164 } else if (pSessionDesc->cipherAlgorithm == 165 CPA_CY_SYM_CIPHER_AES_GCM) { 166 LacSymCb_CleanUserData(pSessionDesc, 167 pDstBuffer, 168 pOpData, 169 CPA_FALSE); 170 } 171 } 172 } else { 173 /* Most commands have no point of failure and always return 174 * success. This is the default response from the QAT. 175 * If status is already set to an error value, don't overwrite 176 * it 177 */ 178 if ((CPA_STATUS_SUCCESS == status) && 179 (CPA_TRUE != qatRespStatusOkFlag)) { 180 LAC_LOG_ERROR("Response status value not as expected"); 181 status = CPA_STATUS_FAIL; 182 } 183 } 184 185 pSymCb = pSessionDesc->pSymCb; 186 pCallbackTag = pCookie->pCallbackTag; 187 188 /* State returned to the client for intermediate partials packets 189 * for hash only and cipher only partial packets. Cipher update 190 * allow next partial through */ 191 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) { 192 if ((CPA_CY_SYM_OP_CIPHER == operationType) || 193 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { 194 if (CPA_TRUE == pCookie->updateUserIvOnRecieve) { 195 /* Update the user's IV buffer 196 * Very important to do this BEFORE dequeuing 197 * subsequent partial requests, as the state 198 * buffer may get overwritten 199 */ 200 memcpy(pCookie->pOpData->pIv, 201 pSessionDesc->cipherPartialOpState, 202 pCookie->pOpData->ivLenInBytes); 203 } 204 if (CPA_TRUE == pCookie->updateKeySizeOnRecieve && 205 LAC_CIPHER_IS_XTS_MODE( 206 pSessionDesc->cipherAlgorithm)) { 207 LacSymQat_CipherXTSModeUpdateKeyLen( 208 pSessionDesc, 209 pSessionDesc->cipherKeyLenInBytes / 2); 210 } 211 } 212 } else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) { 213 if ((CPA_CY_SYM_OP_CIPHER == operationType) || 214 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { 215 if (CPA_TRUE == 216 LAC_CIPHER_IS_XTS_MODE( 217 pSessionDesc->cipherAlgorithm)) { 218 /* 219 * For XTS mode, we replace the updated key with 220 * the original key - for subsequent partial 221 * requests 222 * 223 */ 224 LacSymQat_CipherXTSModeUpdateKeyLen( 225 pSessionDesc, 226 pSessionDesc->cipherKeyLenInBytes); 227 } 228 } 229 } 230 231 if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) && 232 (qatRespStatusOkFlag != CPA_FALSE)) { 233 /* There may be requests blocked pending the completion of this 234 * operation 235 */ 236 237 dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc); 238 if (CPA_STATUS_SUCCESS != dequeueStatus) { 239 LAC_SYM_STAT_INC(numSymOpCompletedErrors, 240 instanceHandle); 241 qatRespStatusOkFlag = CPA_FALSE; 242 if (CPA_STATUS_SUCCESS == status) { 243 status = dequeueStatus; 244 } 245 } 246 } 247 248 if (CPA_STATUS_SUCCESS == status) { 249 /* update stats */ 250 if (pSessionDesc->internalSession == CPA_FALSE) { 251 LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle); 252 if (CPA_STATUS_SUCCESS != status) { 253 LAC_SYM_STAT_INC(numSymOpCompletedErrors, 254 instanceHandle); 255 } 256 } 257 } 258 259 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount)); 260 261 /* deallocate the memory for the internal callback cookie */ 262 Lac_MemPoolEntryFree(pCookie); 263 264 /* user callback function is the last thing to be called */ 265 pSymCb(pCallbackTag, 266 status, 267 operationType, 268 pOpData, 269 pDstBuffer, 270 qatRespStatusOkFlag); 271 } 272 273 /** 274 ****************************************************************************** 275 * @ingroup LacSymCb 276 * Definition of callback function for processing symmetric Data Plane 277 * responses 278 * 279 * @description 280 * This callback checks the status, decrements the number of operations 281 * pending and calls the user callback 282 * 283 * @param[in/out] pResponse pointer to the response structure 284 * @param[in] qatRespStatusOkFlag status 285 * @param[in] pSessionDesc pointer to the session descriptor 286 * 287 * @return None 288 ******************************************************************************/ 289 static void 290 LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse, 291 CpaBoolean qatRespStatusOkFlag, 292 CpaStatus status, 293 lac_session_desc_t *pSessionDesc) 294 { 295 CpaCySymDpCbFunc pSymDpCb = NULL; 296 297 /* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be 298 * cleaned as stated in RFC 3610; in DP mode, it is the user 299 * responsability to do so */ 300 301 if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) && 302 SPC != pSessionDesc->singlePassState) || 303 (CPA_FALSE == pSessionDesc->digestVerify)) { 304 /* If not doing digest compare and qatRespStatusOkFlag != 305 CPA_TRUE then there is something very wrong */ 306 if ((CPA_FALSE == qatRespStatusOkFlag) && 307 (status != CPA_STATUS_UNSUPPORTED)) { 308 LAC_LOG_ERROR("Response status value not as expected"); 309 status = CPA_STATUS_FAIL; 310 } 311 } 312 313 pSymDpCb = 314 ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb; 315 316 pSymDpCb(pResponse, status, qatRespStatusOkFlag); 317 318 /* 319 * Decrement the number of pending CB. 320 * 321 * If the @pendingDpCbCount becomes zero, we may remove the session, 322 * please read more information in the cpaCySymRemoveSession(). 323 * 324 * But there is a field in the @pResponse to store the session, 325 * the "sessionCtx". In another word, in the above @->pSymDpCb() 326 * callback, it may use the session again. If we decrease the 327 * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the 328 * @->pSymDpCb() may reference to a deleted session. 329 * 330 * So in order to avoid the risk, we decrease the @pendingDpCbCount 331 * after the @->pSymDpCb() callback. 332 */ 333 qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount); 334 } 335 336 /** 337 ****************************************************************************** 338 * @ingroup LacSymCb 339 * Definition of callback function for processing symmetric responses 340 * 341 * @description 342 * This callback, which is registered with the common symmetric response 343 * message handler, is invoked to process symmetric response messages from 344 * the QAT. It will extract the response status from the cmnRespFlags set 345 * by the QAT, and then will pass it to @ref 346 * LacSymCb_ProcessCallbackInternal to complete the response processing. 347 * 348 * @param[in] lacCmdId ID of the symmetric QAT command of the request 349 * message 350 * @param[in] pOpaqueData pointer to opaque data in the request message 351 * @param[in] cmnRespFlags Flags set by QAT to indicate response status 352 * 353 * @return None 354 ******************************************************************************/ 355 static void 356 LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId, 357 void *pOpaqueData, 358 icp_qat_fw_comn_flags cmnRespFlags) 359 { 360 CpaStatus status = CPA_STATUS_SUCCESS; 361 CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData; 362 lac_session_desc_t *pSessionDesc = 363 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx); 364 CpaBoolean qatRespStatusOkFlag = 365 (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK == 366 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags)); 367 368 if (CPA_TRUE == pSessionDesc->isDPSession) { 369 /* DP session */ 370 if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET( 371 cmnRespFlags)) { 372 status = CPA_STATUS_UNSUPPORTED; 373 } 374 LacSymCb_ProcessDpCallback(pDpOpData, 375 qatRespStatusOkFlag, 376 status, 377 pSessionDesc); 378 } else { 379 /* Trad session */ 380 LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *) 381 pOpaqueData, 382 qatRespStatusOkFlag, 383 CPA_STATUS_SUCCESS, 384 pSessionDesc); 385 } 386 } 387 388 /* 389 ******************************************************************************* 390 * Define public/global function definitions 391 ******************************************************************************* 392 */ 393 394 /** 395 * @ingroup LacSymCb 396 * 397 * @return CpaStatus 398 * value returned will be the result of icp_adf_transPutMsg 399 */ 400 CpaStatus 401 LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) 402 { 403 CpaStatus status = CPA_STATUS_SUCCESS; 404 sal_crypto_service_t *pService = NULL; 405 Cpa32U retries = 0; 406 407 pService = (sal_crypto_service_t *)pSessionDesc->pInstance; 408 409 /* Need to protect access to queue head and tail pointers, which may 410 * be accessed by multiple contexts simultaneously for enqueue and 411 * dequeue operations 412 */ 413 LAC_SPINLOCK(&pSessionDesc->requestQueueLock); 414 415 /* Clear the blocking flag in the session descriptor */ 416 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE; 417 418 while ((NULL != pSessionDesc->pRequestQueueHead) && 419 (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) { 420 421 /* If we send a partial packet request, set the 422 * blockingOpsInProgress flag for the session to indicate that 423 * subsequent requests must be queued up until this request 424 * completes 425 */ 426 if (CPA_CY_SYM_PACKET_TYPE_FULL != 427 pSessionDesc->pRequestQueueHead->pOpData->packetType) { 428 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE; 429 } 430 431 /* At this point, we're clear to send the request. For cipher 432 * requests, we need to check if the session IV needs to be 433 * updated. This can only be done when no other partials are in 434 * flight for this session, to ensure the cipherPartialOpState 435 * buffer in the session descriptor is not currently in use 436 */ 437 if (CPA_TRUE == 438 pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) { 439 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { 440 memcpy(pSessionDesc->cipherPartialOpState, 441 pSessionDesc->cipherARC4InitialState, 442 LAC_CIPHER_ARC4_STATE_LEN_BYTES); 443 } else { 444 memcpy(pSessionDesc->cipherPartialOpState, 445 pSessionDesc->pRequestQueueHead->pOpData 446 ->pIv, 447 pSessionDesc->pRequestQueueHead->pOpData 448 ->ivLenInBytes); 449 } 450 } 451 452 /* 453 * Now we'll attempt to send the message directly to QAT. We'll 454 * keep looing until it succeeds (or at least a very high number 455 * of retries), as the failure only happens when the ring is 456 * full, and this is only a temporary situation. After a few 457 * retries, space will become availble, allowing the putMsg to 458 * succeed. 459 */ 460 retries = 0; 461 do { 462 /* Send to QAT */ 463 status = icp_adf_transPutMsg( 464 pService->trans_handle_sym_tx, 465 (void *)&(pSessionDesc->pRequestQueueHead->qatMsg), 466 LAC_QAT_SYM_REQ_SZ_LW); 467 468 retries++; 469 /* 470 * Yield to allow other threads that may be on this 471 * session to poll and make some space on the ring 472 */ 473 if (CPA_STATUS_SUCCESS != status) { 474 qatUtilsYield(); 475 } 476 } while ((CPA_STATUS_SUCCESS != status) && 477 (retries < DEQUEUE_MSGPUT_MAX_RETRIES)); 478 479 if ((CPA_STATUS_SUCCESS != status) || 480 (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) { 481 LAC_LOG_ERROR( 482 "Failed to SalQatMsg_transPutMsg, maximum retries exceeded."); 483 goto cleanup; 484 } 485 486 pSessionDesc->pRequestQueueHead = 487 pSessionDesc->pRequestQueueHead->pNext; 488 } 489 490 /* If we've drained the queue, ensure the tail pointer is set to NULL */ 491 if (NULL == pSessionDesc->pRequestQueueHead) { 492 pSessionDesc->pRequestQueueTail = NULL; 493 } 494 495 cleanup: 496 LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock); 497 return status; 498 } 499 500 /** 501 * @ingroup LacSymCb 502 */ 503 void 504 LacSymCb_CallbacksRegister(void) 505 { 506 /*** HASH ***/ 507 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH, 508 LacSymCb_ProcessCallback); 509 510 /*** ALGORITHM-CHAINING CIPHER_HASH***/ 511 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH, 512 LacSymCb_ProcessCallback); 513 514 /*** ALGORITHM-CHAINING HASH_CIPHER***/ 515 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER, 516 LacSymCb_ProcessCallback); 517 518 /*** CIPHER ***/ 519 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER, 520 LacSymCb_ProcessCallback); 521 522 /* Call compile time param check function to ensure it is included 523 in the build by the compiler - this compile time check 524 ensures callbacks run as expected */ 525 LacSym_CompileTimeAssertions(); 526 } 527