1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /**
4  *****************************************************************************
5  * @file lac_buffer_desc.c  Utility functions for setting buffer descriptors
6  *
7  * @ingroup LacBufferDesc
8  *
9  *****************************************************************************/
10 
11 /*
12 *******************************************************************************
13 * Include header files
14 *******************************************************************************
15 */
16 #include "qat_utils.h"
17 #include "icp_accel_devices.h"
18 #include "icp_adf_debug.h"
19 #include "icp_adf_init.h"
20 #include "lac_list.h"
21 #include "lac_sal_types.h"
22 #include "lac_buffer_desc.h"
23 #include "lac_mem.h"
24 #include "cpa_cy_common.h"
25 
26 /*
27 *******************************************************************************
28 * Define public/global function definitions
29 *******************************************************************************
30 */
31 /* Invalid physical address value */
32 #define INVALID_PHYSICAL_ADDRESS 0
33 
34 /* Indicates what type of buffer writes need to be perfomed */
35 typedef enum lac_buff_write_op_e {
36 	WRITE_NORMAL = 0,
37 	WRITE_AND_GET_SIZE,
38 	WRITE_AND_ALLOW_ZERO_BUFFER,
39 } lac_buff_write_op_t;
40 
41 /* This function implements the buffer description writes for the traditional
42  * APIs */
43 static CpaStatus
44 LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList,
45 				      Cpa64U *pBufListAlignedPhyAddr,
46 				      CpaBoolean isPhysicalAddress,
47 				      Cpa64U *totalDataLenInBytes,
48 				      sal_service_t *pService,
49 				      lac_buff_write_op_t operationType)
50 {
51 	Cpa32U numBuffers = 0;
52 	icp_qat_addr_width_t bufListDescPhyAddr = 0;
53 	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
54 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
55 	icp_buffer_list_desc_t *pBufferListDesc = NULL;
56 	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
57 
58 	if (WRITE_AND_GET_SIZE == operationType) {
59 		*totalDataLenInBytes = 0;
60 	}
61 
62 	numBuffers = pUserBufferList->numBuffers;
63 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
64 
65 	/*
66 	 * Get the physical address of this descriptor - need to offset by the
67 	 * alignment restrictions on the buffer descriptors
68 	 */
69 	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
70 	    (*pService), pUserBufferList->pPrivateMetaData);
71 
72 	if (bufListDescPhyAddr == 0) {
73 		QAT_UTILS_LOG(
74 		    "Unable to get the physical address of the metadata.\n");
75 		return CPA_STATUS_FAIL;
76 	}
77 
78 	bufListAlignedPhyAddr =
79 	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
80 				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
81 
82 	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
83 	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
84 	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
85 	     (LAC_ARCH_UINT)bufListDescPhyAddr));
86 
87 	/* Go past the Buffer List descriptor to the list of buffer descriptors
88 	 */
89 	pCurrFlatBufDesc =
90 	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
91 
92 	pBufferListDesc->numBuffers = numBuffers;
93 
94 	if (WRITE_AND_GET_SIZE != operationType) {
95 		/* Defining zero buffers is useful for example if running zero
96 		 * length
97 		 * hash */
98 		if (0 == numBuffers) {
99 			/* In the case where there are zero buffers within the
100 			 * BufList
101 			 * it is required by firmware that the number is set to
102 			 * 1
103 			 * but the phyBuffer and dataLenInBytes are set to
104 			 * NULL.*/
105 			pBufferListDesc->numBuffers = 1;
106 			pCurrFlatBufDesc->dataLenInBytes = 0;
107 			pCurrFlatBufDesc->phyBuffer = 0;
108 		}
109 	}
110 
111 	while (0 != numBuffers) {
112 		pCurrFlatBufDesc->dataLenInBytes =
113 		    pCurrClientFlatBuffer->dataLenInBytes;
114 
115 		if (WRITE_AND_GET_SIZE == operationType) {
116 			/* Calculate the total data length in bytes */
117 			*totalDataLenInBytes +=
118 			    pCurrClientFlatBuffer->dataLenInBytes;
119 		}
120 
121 		/* Check if providing a physical address in the function. If not
122 		 * we
123 		 * need to convert it to a physical one */
124 		if (CPA_TRUE == isPhysicalAddress) {
125 			pCurrFlatBufDesc->phyBuffer =
126 			    LAC_MEM_CAST_PTR_TO_UINT64(
127 				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
128 		} else {
129 			pCurrFlatBufDesc->phyBuffer =
130 			    LAC_MEM_CAST_PTR_TO_UINT64(
131 				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
132 				    (*pService), pCurrClientFlatBuffer->pData));
133 
134 			if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) {
135 				if (INVALID_PHYSICAL_ADDRESS ==
136 				    pCurrFlatBufDesc->phyBuffer) {
137 					QAT_UTILS_LOG(
138 					    "Unable to get the physical address of the client buffer.\n");
139 					return CPA_STATUS_FAIL;
140 				}
141 			}
142 		}
143 
144 		pCurrFlatBufDesc++;
145 		pCurrClientFlatBuffer++;
146 
147 		numBuffers--;
148 	}
149 
150 	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
151 	return CPA_STATUS_SUCCESS;
152 }
153 
154 /* This function implements the buffer description writes for the traditional
155  * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and
156  * GCM aglorithms */
157 CpaStatus
158 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
159     const CpaBufferList *pUserBufferList,
160     Cpa64U *pBufListAlignedPhyAddr,
161     CpaBoolean isPhysicalAddress,
162     sal_service_t *pService)
163 {
164 	return LacBuffDesc_CommonBufferListDescWrite(
165 	    pUserBufferList,
166 	    pBufListAlignedPhyAddr,
167 	    isPhysicalAddress,
168 	    NULL,
169 	    pService,
170 	    WRITE_AND_ALLOW_ZERO_BUFFER);
171 }
172 
173 /* This function implements the buffer description writes for the traditional
174  * APIs */
175 CpaStatus
176 LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList,
177 				Cpa64U *pBufListAlignedPhyAddr,
178 				CpaBoolean isPhysicalAddress,
179 				sal_service_t *pService)
180 {
181 	return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList,
182 						     pBufListAlignedPhyAddr,
183 						     isPhysicalAddress,
184 						     NULL,
185 						     pService,
186 						     WRITE_NORMAL);
187 }
188 
189 /* This function does the same processing as LacBuffDesc_BufferListDescWrite
190  * but calculate as well the total length in bytes of the buffer list. */
191 CpaStatus
192 LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList,
193 					  Cpa64U *pBufListAlignedPhyAddr,
194 					  CpaBoolean isPhysicalAddress,
195 					  Cpa64U *totalDataLenInBytes,
196 					  sal_service_t *pService)
197 {
198 	Cpa32U numBuffers = 0;
199 	icp_qat_addr_width_t bufListDescPhyAddr = 0;
200 	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
201 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
202 	icp_buffer_list_desc_t *pBufferListDesc = NULL;
203 	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
204 	*totalDataLenInBytes = 0;
205 
206 	numBuffers = pUserBufferList->numBuffers;
207 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
208 
209 	/*
210 	 * Get the physical address of this descriptor - need to offset by the
211 	 * alignment restrictions on the buffer descriptors
212 	 */
213 	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
214 	    (*pService), pUserBufferList->pPrivateMetaData);
215 
216 	if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) {
217 		QAT_UTILS_LOG(
218 		    "Unable to get the physical address of the metadata.\n");
219 		return CPA_STATUS_FAIL;
220 	}
221 
222 	bufListAlignedPhyAddr =
223 	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
224 				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
225 
226 	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
227 	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
228 	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
229 	     (LAC_ARCH_UINT)bufListDescPhyAddr));
230 
231 	/* Go past the Buffer List descriptor to the list of buffer descriptors
232 	 */
233 	pCurrFlatBufDesc =
234 	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
235 
236 	pBufferListDesc->numBuffers = numBuffers;
237 
238 	while (0 != numBuffers) {
239 		pCurrFlatBufDesc->dataLenInBytes =
240 		    pCurrClientFlatBuffer->dataLenInBytes;
241 
242 		/* Calculate the total data length in bytes */
243 		*totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes;
244 
245 		if (isPhysicalAddress == CPA_TRUE) {
246 			pCurrFlatBufDesc->phyBuffer =
247 			    LAC_MEM_CAST_PTR_TO_UINT64(
248 				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
249 		} else {
250 			pCurrFlatBufDesc->phyBuffer =
251 			    LAC_MEM_CAST_PTR_TO_UINT64(
252 				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
253 				    (*pService), pCurrClientFlatBuffer->pData));
254 
255 			if (pCurrFlatBufDesc->phyBuffer == 0) {
256 				QAT_UTILS_LOG(
257 				    "Unable to get the physical address of the client buffer.\n");
258 				return CPA_STATUS_FAIL;
259 			}
260 		}
261 
262 		pCurrFlatBufDesc++;
263 		pCurrClientFlatBuffer++;
264 
265 		numBuffers--;
266 	}
267 
268 	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
269 	return CPA_STATUS_SUCCESS;
270 }
271 
272 CpaStatus
273 LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer,
274 			     Cpa64U *pPktSize,
275 			     lac_aligment_shift_t alignmentShiftExpected)
276 {
277 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
278 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
279 
280 	if (0 == pUserFlatBuffer->dataLenInBytes) {
281 		QAT_UTILS_LOG("FlatBuffer empty\n");
282 		return CPA_STATUS_INVALID_PARAM;
283 	}
284 
285 	/* Expected alignment */
286 	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
287 		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
288 					 alignmentShiftExpected)) {
289 			QAT_UTILS_LOG(
290 			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
291 			    1 << alignmentShiftExpected);
292 			return CPA_STATUS_INVALID_PARAM;
293 		}
294 	}
295 
296 	/* Update the total size of the packet. This function being called in a
297 	 * loop
298 	 * for an entire buffer list we need to increment the value */
299 	*pPktSize += pUserFlatBuffer->dataLenInBytes;
300 
301 	return CPA_STATUS_SUCCESS;
302 }
303 
304 CpaStatus
305 LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer,
306 				 Cpa64U *pPktSize,
307 				 lac_aligment_shift_t alignmentShiftExpected)
308 {
309 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
310 
311 	if (0 != pUserFlatBuffer->dataLenInBytes) {
312 		LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
313 	}
314 
315 	/* Expected alignment */
316 	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
317 		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
318 					 alignmentShiftExpected)) {
319 			QAT_UTILS_LOG(
320 			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
321 			    1 << alignmentShiftExpected);
322 			return CPA_STATUS_INVALID_PARAM;
323 		}
324 	}
325 
326 	/* Update the total size of the packet. This function being called in a
327 	 * loop
328 	 * for an entire buffer list we need to increment the value */
329 	*pPktSize += pUserFlatBuffer->dataLenInBytes;
330 
331 	return CPA_STATUS_SUCCESS;
332 }
333 
334 CpaStatus
335 LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList,
336 			     Cpa64U *pPktSize,
337 			     lac_aligment_shift_t alignmentShiftExpected)
338 {
339 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
340 	Cpa32U numBuffers = 0;
341 	CpaStatus status = CPA_STATUS_SUCCESS;
342 
343 	LAC_CHECK_NULL_PARAM(pUserBufferList);
344 	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
345 	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
346 
347 	numBuffers = pUserBufferList->numBuffers;
348 
349 	if (0 == pUserBufferList->numBuffers) {
350 		QAT_UTILS_LOG("Number of buffers is 0.\n");
351 		return CPA_STATUS_INVALID_PARAM;
352 	}
353 
354 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
355 
356 	*pPktSize = 0;
357 	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
358 		status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer,
359 						      pPktSize,
360 						      alignmentShiftExpected);
361 
362 		pCurrClientFlatBuffer++;
363 		numBuffers--;
364 	}
365 	return status;
366 }
367 
368 CpaStatus
369 LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList,
370 				 Cpa64U *pPktSize,
371 				 lac_aligment_shift_t alignmentShiftExpected)
372 {
373 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
374 	Cpa32U numBuffers = 0;
375 	CpaStatus status = CPA_STATUS_SUCCESS;
376 
377 	LAC_CHECK_NULL_PARAM(pUserBufferList);
378 	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
379 	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
380 
381 	numBuffers = pUserBufferList->numBuffers;
382 
383 	if (0 == pUserBufferList->numBuffers) {
384 		QAT_UTILS_LOG("Number of buffers is 0.\n");
385 		return CPA_STATUS_INVALID_PARAM;
386 	}
387 
388 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
389 
390 	*pPktSize = 0;
391 	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
392 		status =
393 		    LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer,
394 						     pPktSize,
395 						     alignmentShiftExpected);
396 
397 		pCurrClientFlatBuffer++;
398 		numBuffers--;
399 	}
400 	return status;
401 }
402 
403 /**
404  ******************************************************************************
405  * @ingroup LacBufferDesc
406  *****************************************************************************/
407 void
408 LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList,
409 				   Cpa64U *pPktSize)
410 {
411 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
412 	Cpa32U numBuffers = 0;
413 
414 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
415 	numBuffers = pUserBufferList->numBuffers;
416 
417 	*pPktSize = 0;
418 	while (0 != numBuffers) {
419 		*pPktSize += pCurrClientFlatBuffer->dataLenInBytes;
420 		pCurrClientFlatBuffer++;
421 		numBuffers--;
422 	}
423 }
424 
425 void
426 LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList,
427 				     Cpa32U offset,
428 				     Cpa32U lenToZero)
429 {
430 	Cpa32U zeroLen = 0, sizeLeftToZero = 0;
431 	Cpa64U currentBufferSize = 0;
432 	CpaFlatBuffer *pBuffer = NULL;
433 	Cpa8U *pZero = NULL;
434 	pBuffer = pBuffList->pBuffers;
435 
436 	/* Take a copy of total length to zero. */
437 	sizeLeftToZero = lenToZero;
438 
439 	while (sizeLeftToZero > 0) {
440 		currentBufferSize = pBuffer->dataLenInBytes;
441 		/* check where to start zeroing */
442 		if (offset >= currentBufferSize) {
443 			/* Need to get to next buffer and reduce
444 			 * offset size by data len of buffer */
445 			offset = offset - pBuffer->dataLenInBytes;
446 			pBuffer++;
447 		} else {
448 			/* Start to Zero from this position */
449 			pZero = (Cpa8U *)pBuffer->pData + offset;
450 
451 			/* Need to calculate the correct number of bytes to zero
452 			 * for this iteration and for this location.
453 			 */
454 			if (sizeLeftToZero >= pBuffer->dataLenInBytes) {
455 				/* The size to zero is spanning buffers, zeroLen
456 				 * in
457 				 * this case is from pZero (position) to end of
458 				 * buffer.
459 				 */
460 				zeroLen = pBuffer->dataLenInBytes - offset;
461 			} else {
462 				/* zeroLen is set to sizeLeftToZero, then check
463 				 * if zeroLen and
464 				 * the offset is greater or equal to the size of
465 				 * the buffer, if
466 				 * yes, adjust the zeroLen to zero out the
467 				 * remainder of this
468 				 * buffer.
469 				 */
470 				zeroLen = sizeLeftToZero;
471 				if ((zeroLen + offset) >=
472 				    pBuffer->dataLenInBytes) {
473 					zeroLen =
474 					    pBuffer->dataLenInBytes - offset;
475 				}
476 			} /* end inner else */
477 			memset((void *)pZero, 0, zeroLen);
478 			sizeLeftToZero = sizeLeftToZero - zeroLen;
479 			/* offset is no longer required as any data left to zero
480 			 * is now
481 			 * at the start of the next buffer. set offset to zero
482 			 * and move on
483 			 * the buffer pointer to the next buffer.
484 			 */
485 			offset = 0;
486 			pBuffer++;
487 
488 		} /* end outer else */
489 
490 	} /* end while */
491 }
492