1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 
4 /**
5  ***************************************************************************
6  * @file lac_sym_cb.c      Callback handler functions for symmetric components
7  *
8  * @ingroup LacSym
9  *
10  ***************************************************************************/
11 
12 /*
13 *******************************************************************************
14 * Include public/global header files
15 *******************************************************************************
16 */
17 
18 #include "cpa.h"
19 #include "cpa_cy_sym.h"
20 
21 #include "icp_accel_devices.h"
22 #include "icp_adf_init.h"
23 #include "icp_qat_fw_la.h"
24 #include "icp_adf_transport.h"
25 #include "icp_adf_debug.h"
26 
27 #include "lac_sym.h"
28 #include "lac_sym_cipher.h"
29 #include "lac_common.h"
30 #include "lac_list.h"
31 #include "lac_sal_types_crypto.h"
32 #include "lac_sal.h"
33 #include "lac_sal_ctrl.h"
34 #include "lac_session.h"
35 #include "lac_sym_stats.h"
36 #include "lac_log.h"
37 #include "lac_sym_cb.h"
38 #include "lac_sym_hash.h"
39 #include "lac_sym_qat_cipher.h"
40 #include "lac_sym_qat.h"
41 
42 #define DEQUEUE_MSGPUT_MAX_RETRIES 10000
43 
44 /*
45 *******************************************************************************
46 * Define static function definitions
47 *******************************************************************************
48 */
49 
50 /**
51  *****************************************************************************
52  * @ingroup LacSymCb
53  *      Function to clean computed data.
54  *
55  * @description
56  *      This function cleans GCM or CCM data in the case of a failure.
57  *
58  * @param[in]  pSessionDesc pointer to the session descriptor
59  * @param[out] pBufferList  pointer to the bufferlist to clean
60  * @param[in]  pOpData      pointer to operation data
61  * @param[in]  isCCM        is it a CCM operation boolean
62  *
63  * @return  None
64  *****************************************************************************/
65 static void
66 LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc,
67 		       CpaBufferList *pBufferList,
68 		       const CpaCySymOpData *pOpData,
69 		       CpaBoolean isCCM)
70 {
71 	Cpa32U authTagLen = 0;
72 
73 	/* Retrieve authTagLen */
74 	authTagLen = pSessionDesc->hashResultSize;
75 
76 	/* Cleaning */
77 	if (isCCM) {
78 		/* for CCM the digest is inside the buffer list */
79 		LacBuffDesc_BufferListZeroFromOffset(
80 		    pBufferList,
81 		    pOpData->cryptoStartSrcOffsetInBytes,
82 		    pOpData->messageLenToCipherInBytes + authTagLen);
83 	} else {
84 		/* clean buffer list */
85 		LacBuffDesc_BufferListZeroFromOffset(
86 		    pBufferList,
87 		    pOpData->cryptoStartSrcOffsetInBytes,
88 		    pOpData->messageLenToCipherInBytes);
89 	}
90 	if ((CPA_TRUE != pSessionDesc->digestIsAppended) &&
91 	    (NULL != pOpData->pDigestResult)) {
92 		/* clean digest */
93 		memset(pOpData->pDigestResult, 0, authTagLen);
94 	}
95 }
96 
97 /**
98  *****************************************************************************
99  * @ingroup LacSymCb
100  *      Definition of callback function for processing symmetric responses
101  *
102  * @description
103  *      This callback is invoked to process symmetric response messages from
104  *      the QAT.  It will extract some details from the message and invoke
105  *      the user's callback to complete a symmetric operation.
106  *
107  * @param[in] pCookie             Pointer to cookie associated with this request
108  * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT
109  * @param[in] status              Status variable indicating an error occurred
110  *                                in sending the message (e.g. when dequeueing)
111  * @param[in] pSessionDesc        Session descriptor
112  *
113  * @return  None
114  *****************************************************************************/
115 static void
116 LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie,
117 				 CpaBoolean qatRespStatusOkFlag,
118 				 CpaStatus status,
119 				 lac_session_desc_t *pSessionDesc)
120 {
121 	CpaCySymCbFunc pSymCb = NULL;
122 	void *pCallbackTag = NULL;
123 	CpaCySymOpData *pOpData = NULL;
124 	CpaBufferList *pDstBuffer = NULL;
125 	CpaCySymOp operationType = CPA_CY_SYM_OP_NONE;
126 	CpaStatus dequeueStatus = CPA_STATUS_SUCCESS;
127 
128 	CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE;
129 	/* NOTE: cookie pointer validated in previous function */
130 	instanceHandle = pCookie->instanceHandle;
131 
132 	pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData);
133 	operationType = pSessionDesc->symOperation;
134 
135 	/* Set the destination pointer to the one supplied in the cookie. */
136 	pDstBuffer = pCookie->pDstBuffer;
137 
138 	/* For a digest verify operation - for full packet and final partial
139 	 * only, perform a comparison with the digest generated and with the one
140 	 * supplied in the packet. In case of AES_GCM in SPC mode, destination
141 	 * buffer needs to be cleared if digest verify operation fails */
142 
143 	if (((SPC == pSessionDesc->singlePassState) ||
144 	     (CPA_CY_SYM_OP_CIPHER != operationType)) &&
145 	    (CPA_TRUE == pSessionDesc->digestVerify) &&
146 	    ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) ||
147 	     (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) {
148 		if (CPA_FALSE == qatRespStatusOkFlag) {
149 			LAC_SYM_STAT_INC(numSymOpVerifyFailures,
150 					 instanceHandle);
151 
152 			/* The comparison has failed at this point (status is
153 			 * fail), need to clean any sensitive calculated data up
154 			 * to this point. The data calculated is no longer
155 			 * useful to the end result and does not need to be
156 			 * returned to the user so setting buffers to zero.
157 			 */
158 			if (pSessionDesc->cipherAlgorithm ==
159 			    CPA_CY_SYM_CIPHER_AES_CCM) {
160 				LacSymCb_CleanUserData(pSessionDesc,
161 						       pDstBuffer,
162 						       pOpData,
163 						       CPA_TRUE);
164 			} else if (pSessionDesc->cipherAlgorithm ==
165 				   CPA_CY_SYM_CIPHER_AES_GCM) {
166 				LacSymCb_CleanUserData(pSessionDesc,
167 						       pDstBuffer,
168 						       pOpData,
169 						       CPA_FALSE);
170 			}
171 		}
172 	} else {
173 		/* Most commands have no point of failure and always return
174 		 * success. This is the default response from the QAT.
175 		 * If status is already set to an error value, don't overwrite
176 		 * it
177 		 */
178 		if ((CPA_STATUS_SUCCESS == status) &&
179 		    (CPA_TRUE != qatRespStatusOkFlag)) {
180 			LAC_LOG_ERROR("Response status value not as expected");
181 			status = CPA_STATUS_FAIL;
182 		}
183 	}
184 
185 	pSymCb = pSessionDesc->pSymCb;
186 	pCallbackTag = pCookie->pCallbackTag;
187 
188 	/* State returned to the client for intermediate partials packets
189 	 * for hash only and cipher only partial packets. Cipher update
190 	 * allow next partial through */
191 	if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) {
192 		if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
193 		    (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
194 			if (CPA_TRUE == pCookie->updateUserIvOnRecieve) {
195 				/* Update the user's IV buffer
196 				 * Very important to do this BEFORE dequeuing
197 				 * subsequent partial requests, as the state
198 				 * buffer may get overwritten
199 				 */
200 				memcpy(pCookie->pOpData->pIv,
201 				       pSessionDesc->cipherPartialOpState,
202 				       pCookie->pOpData->ivLenInBytes);
203 			}
204 			if (CPA_TRUE == pCookie->updateKeySizeOnRecieve &&
205 			    LAC_CIPHER_IS_XTS_MODE(
206 				pSessionDesc->cipherAlgorithm)) {
207 				LacSymQat_CipherXTSModeUpdateKeyLen(
208 				    pSessionDesc,
209 				    pSessionDesc->cipherKeyLenInBytes / 2);
210 			}
211 		}
212 	} else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) {
213 		if ((CPA_CY_SYM_OP_CIPHER == operationType) ||
214 		    (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) {
215 			if (CPA_TRUE ==
216 			    LAC_CIPHER_IS_XTS_MODE(
217 				pSessionDesc->cipherAlgorithm)) {
218 				/*
219 				 * For XTS mode, we replace the updated key with
220 				 * the original key - for subsequent partial
221 				 * requests
222 				 *
223 				 */
224 				LacSymQat_CipherXTSModeUpdateKeyLen(
225 				    pSessionDesc,
226 				    pSessionDesc->cipherKeyLenInBytes);
227 			}
228 		}
229 	}
230 
231 	if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) &&
232 	    (qatRespStatusOkFlag != CPA_FALSE)) {
233 		/* There may be requests blocked pending the completion of this
234 		 * operation
235 		 */
236 
237 		dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc);
238 		if (CPA_STATUS_SUCCESS != dequeueStatus) {
239 			LAC_SYM_STAT_INC(numSymOpCompletedErrors,
240 					 instanceHandle);
241 			qatRespStatusOkFlag = CPA_FALSE;
242 			if (CPA_STATUS_SUCCESS == status) {
243 				status = dequeueStatus;
244 			}
245 		}
246 	}
247 
248 	if (CPA_STATUS_SUCCESS == status) {
249 		/* update stats */
250 		if (pSessionDesc->internalSession == CPA_FALSE) {
251 			LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle);
252 			if (CPA_STATUS_SUCCESS != status) {
253 				LAC_SYM_STAT_INC(numSymOpCompletedErrors,
254 						 instanceHandle);
255 			}
256 		}
257 	}
258 
259 	qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount));
260 
261 	/* deallocate the memory for the internal callback cookie */
262 	Lac_MemPoolEntryFree(pCookie);
263 
264 	/* user callback function is the last thing to be called */
265 	pSymCb(pCallbackTag,
266 	       status,
267 	       operationType,
268 	       pOpData,
269 	       pDstBuffer,
270 	       qatRespStatusOkFlag);
271 }
272 
273 /**
274  ******************************************************************************
275  * @ingroup LacSymCb
276  *      Definition of callback function for processing symmetric Data Plane
277  *       responses
278  *
279  * @description
280  *      This callback checks the status, decrements the number of operations
281  *      pending and calls the user callback
282  *
283  * @param[in/out] pResponse          pointer to the response structure
284  * @param[in] qatRespStatusOkFlag    status
285  * @param[in] pSessionDesc           pointer to the session descriptor
286  *
287  * @return  None
288  ******************************************************************************/
289 static void
290 LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse,
291 			   CpaBoolean qatRespStatusOkFlag,
292 			   CpaStatus status,
293 			   lac_session_desc_t *pSessionDesc)
294 {
295 	CpaCySymDpCbFunc pSymDpCb = NULL;
296 
297 	/* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be
298 	 * cleaned as stated in RFC 3610; in DP mode, it is the user
299 	 * responsability to do so */
300 
301 	if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) &&
302 	     SPC != pSessionDesc->singlePassState) ||
303 	    (CPA_FALSE == pSessionDesc->digestVerify)) {
304 		/* If not doing digest compare and qatRespStatusOkFlag !=
305 		   CPA_TRUE then there is something very wrong */
306 		if ((CPA_FALSE == qatRespStatusOkFlag) &&
307 		    (status != CPA_STATUS_UNSUPPORTED)) {
308 			LAC_LOG_ERROR("Response status value not as expected");
309 			status = CPA_STATUS_FAIL;
310 		}
311 	}
312 
313 	pSymDpCb =
314 	    ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb;
315 
316 	pSymDpCb(pResponse, status, qatRespStatusOkFlag);
317 
318 	/*
319 	 * Decrement the number of pending CB.
320 	 *
321 	 * If the @pendingDpCbCount becomes zero, we may remove the session,
322 	 * please read more information in the cpaCySymRemoveSession().
323 	 *
324 	 * But there is a field in the @pResponse to store the session,
325 	 * the "sessionCtx". In another word, in the above @->pSymDpCb()
326 	 * callback, it may use the session again. If we decrease the
327 	 * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the
328 	 * @->pSymDpCb() may reference to a deleted session.
329 	 *
330 	 * So in order to avoid the risk, we decrease the @pendingDpCbCount
331 	 * after the @->pSymDpCb() callback.
332 	 */
333 	qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount);
334 }
335 
336 /**
337  ******************************************************************************
338  * @ingroup LacSymCb
339  *      Definition of callback function for processing symmetric responses
340  *
341  * @description
342  *      This callback, which is registered with the common symmetric response
343  *      message handler,  is invoked to process symmetric response messages from
344  *      the QAT.  It will extract the response status from the cmnRespFlags set
345  *      by the QAT, and then will pass it to @ref
346  *      LacSymCb_ProcessCallbackInternal to complete the response processing.
347  *
348  * @param[in] lacCmdId          ID of the symmetric QAT command of the request
349  *                              message
350  * @param[in] pOpaqueData       pointer to opaque data in the request message
351  * @param[in] cmnRespFlags      Flags set by QAT to indicate response status
352  *
353  * @return  None
354  ******************************************************************************/
355 static void
356 LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId,
357 			 void *pOpaqueData,
358 			 icp_qat_fw_comn_flags cmnRespFlags)
359 {
360 	CpaStatus status = CPA_STATUS_SUCCESS;
361 	CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData;
362 	lac_session_desc_t *pSessionDesc =
363 	    LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx);
364 	CpaBoolean qatRespStatusOkFlag =
365 	    (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
366 			 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags));
367 
368 	if (CPA_TRUE == pSessionDesc->isDPSession) {
369 		/* DP session */
370 		if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(
371 			cmnRespFlags)) {
372 			status = CPA_STATUS_UNSUPPORTED;
373 		}
374 		LacSymCb_ProcessDpCallback(pDpOpData,
375 					   qatRespStatusOkFlag,
376 					   status,
377 					   pSessionDesc);
378 	} else {
379 		/* Trad session */
380 		LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *)
381 						     pOpaqueData,
382 						 qatRespStatusOkFlag,
383 						 CPA_STATUS_SUCCESS,
384 						 pSessionDesc);
385 	}
386 }
387 
388 /*
389 *******************************************************************************
390 * Define public/global function definitions
391 *******************************************************************************
392 */
393 
394 /**
395  * @ingroup LacSymCb
396  *
397  * @return CpaStatus
398  *      value returned will be the result of icp_adf_transPutMsg
399  */
400 CpaStatus
401 LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc)
402 {
403 	CpaStatus status = CPA_STATUS_SUCCESS;
404 	sal_crypto_service_t *pService = NULL;
405 	Cpa32U retries = 0;
406 
407 	pService = (sal_crypto_service_t *)pSessionDesc->pInstance;
408 
409 	/* Need to protect access to queue head and tail pointers, which may
410 	 * be accessed by multiple contexts simultaneously for enqueue and
411 	 * dequeue operations
412 	 */
413 	LAC_SPINLOCK(&pSessionDesc->requestQueueLock);
414 
415 	/* Clear the blocking flag in the session descriptor */
416 	pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE;
417 
418 	while ((NULL != pSessionDesc->pRequestQueueHead) &&
419 	       (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) {
420 
421 		/* If we send a partial packet request, set the
422 		 * blockingOpsInProgress flag for the session to indicate that
423 		 * subsequent requests must be queued up until this request
424 		 * completes
425 		 */
426 		if (CPA_CY_SYM_PACKET_TYPE_FULL !=
427 		    pSessionDesc->pRequestQueueHead->pOpData->packetType) {
428 			pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE;
429 		}
430 
431 		/* At this point, we're clear to send the request.  For cipher
432 		 * requests, we need to check if the session IV needs to be
433 		 * updated.  This can only be done when no other partials are in
434 		 * flight for this session, to ensure the cipherPartialOpState
435 		 * buffer in the session descriptor is not currently in use
436 		 */
437 		if (CPA_TRUE ==
438 		    pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) {
439 			if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) {
440 				memcpy(pSessionDesc->cipherPartialOpState,
441 				       pSessionDesc->cipherARC4InitialState,
442 				       LAC_CIPHER_ARC4_STATE_LEN_BYTES);
443 			} else {
444 				memcpy(pSessionDesc->cipherPartialOpState,
445 				       pSessionDesc->pRequestQueueHead->pOpData
446 					   ->pIv,
447 				       pSessionDesc->pRequestQueueHead->pOpData
448 					   ->ivLenInBytes);
449 			}
450 		}
451 
452 		/*
453 		 * Now we'll attempt to send the message directly to QAT. We'll
454 		 * keep looing until it succeeds (or at least a very high number
455 		 * of retries), as the failure only happens when the ring is
456 		 * full, and this is only a temporary situation. After a few
457 		 * retries, space will become availble, allowing the putMsg to
458 		 * succeed.
459 		 */
460 		retries = 0;
461 		do {
462 			/* Send to QAT */
463 			status = icp_adf_transPutMsg(
464 			    pService->trans_handle_sym_tx,
465 			    (void *)&(pSessionDesc->pRequestQueueHead->qatMsg),
466 			    LAC_QAT_SYM_REQ_SZ_LW);
467 
468 			retries++;
469 			/*
470 			 * Yield to allow other threads that may be on this
471 			 * session to poll and make some space on the ring
472 			 */
473 			if (CPA_STATUS_SUCCESS != status) {
474 				qatUtilsYield();
475 			}
476 		} while ((CPA_STATUS_SUCCESS != status) &&
477 			 (retries < DEQUEUE_MSGPUT_MAX_RETRIES));
478 
479 		if ((CPA_STATUS_SUCCESS != status) ||
480 		    (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) {
481 			LAC_LOG_ERROR(
482 			    "Failed to SalQatMsg_transPutMsg, maximum retries exceeded.");
483 			goto cleanup;
484 		}
485 
486 		pSessionDesc->pRequestQueueHead =
487 		    pSessionDesc->pRequestQueueHead->pNext;
488 	}
489 
490 	/* If we've drained the queue, ensure the tail pointer is set to NULL */
491 	if (NULL == pSessionDesc->pRequestQueueHead) {
492 		pSessionDesc->pRequestQueueTail = NULL;
493 	}
494 
495 cleanup:
496 	LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock);
497 	return status;
498 }
499 
500 /**
501  * @ingroup LacSymCb
502  */
503 void
504 LacSymCb_CallbacksRegister(void)
505 {
506 	/*** HASH ***/
507 	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH,
508 				      LacSymCb_ProcessCallback);
509 
510 	/*** ALGORITHM-CHAINING CIPHER_HASH***/
511 	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH,
512 				      LacSymCb_ProcessCallback);
513 
514 	/*** ALGORITHM-CHAINING HASH_CIPHER***/
515 	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER,
516 				      LacSymCb_ProcessCallback);
517 
518 	/*** CIPHER ***/
519 	LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER,
520 				      LacSymCb_ProcessCallback);
521 
522 	/* Call compile time param check function to ensure it is included
523 	   in the build by the compiler - this compile time check
524 	   ensures callbacks run as expected */
525 	LacSym_CompileTimeAssertions();
526 }
527