1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /* $FreeBSD$ */
4 /**
5  *****************************************************************************
6  * @file lac_buffer_desc.c  Utility functions for setting buffer descriptors
7  *
8  * @ingroup LacBufferDesc
9  *
10  *****************************************************************************/
11 
12 /*
13 *******************************************************************************
14 * Include header files
15 *******************************************************************************
16 */
17 #include "qat_utils.h"
18 #include "icp_accel_devices.h"
19 #include "icp_adf_debug.h"
20 #include "icp_adf_init.h"
21 #include "lac_list.h"
22 #include "lac_sal_types.h"
23 #include "lac_buffer_desc.h"
24 #include "lac_mem.h"
25 #include "cpa_cy_common.h"
26 
27 /*
28 *******************************************************************************
29 * Define public/global function definitions
30 *******************************************************************************
31 */
32 /* Invalid physical address value */
33 #define INVALID_PHYSICAL_ADDRESS 0
34 
35 /* Indicates what type of buffer writes need to be perfomed */
36 typedef enum lac_buff_write_op_e {
37 	WRITE_NORMAL = 0,
38 	WRITE_AND_GET_SIZE,
39 	WRITE_AND_ALLOW_ZERO_BUFFER,
40 } lac_buff_write_op_t;
41 
42 /* This function implements the buffer description writes for the traditional
43  * APIs */
44 static CpaStatus
45 LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList,
46 				      Cpa64U *pBufListAlignedPhyAddr,
47 				      CpaBoolean isPhysicalAddress,
48 				      Cpa64U *totalDataLenInBytes,
49 				      sal_service_t *pService,
50 				      lac_buff_write_op_t operationType)
51 {
52 	Cpa32U numBuffers = 0;
53 	icp_qat_addr_width_t bufListDescPhyAddr = 0;
54 	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
55 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
56 	icp_buffer_list_desc_t *pBufferListDesc = NULL;
57 	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
58 
59 	if (WRITE_AND_GET_SIZE == operationType) {
60 		*totalDataLenInBytes = 0;
61 	}
62 
63 	numBuffers = pUserBufferList->numBuffers;
64 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
65 
66 	/*
67 	 * Get the physical address of this descriptor - need to offset by the
68 	 * alignment restrictions on the buffer descriptors
69 	 */
70 	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
71 	    (*pService), pUserBufferList->pPrivateMetaData);
72 
73 	if (bufListDescPhyAddr == 0) {
74 		QAT_UTILS_LOG(
75 		    "Unable to get the physical address of the metadata.\n");
76 		return CPA_STATUS_FAIL;
77 	}
78 
79 	bufListAlignedPhyAddr =
80 	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
81 				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
82 
83 	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
84 	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
85 	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
86 	     (LAC_ARCH_UINT)bufListDescPhyAddr));
87 
88 	/* Go past the Buffer List descriptor to the list of buffer descriptors
89 	 */
90 	pCurrFlatBufDesc =
91 	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
92 
93 	pBufferListDesc->numBuffers = numBuffers;
94 
95 	if (WRITE_AND_GET_SIZE != operationType) {
96 		/* Defining zero buffers is useful for example if running zero
97 		 * length
98 		 * hash */
99 		if (0 == numBuffers) {
100 			/* In the case where there are zero buffers within the
101 			 * BufList
102 			 * it is required by firmware that the number is set to
103 			 * 1
104 			 * but the phyBuffer and dataLenInBytes are set to
105 			 * NULL.*/
106 			pBufferListDesc->numBuffers = 1;
107 			pCurrFlatBufDesc->dataLenInBytes = 0;
108 			pCurrFlatBufDesc->phyBuffer = 0;
109 		}
110 	}
111 
112 	while (0 != numBuffers) {
113 		pCurrFlatBufDesc->dataLenInBytes =
114 		    pCurrClientFlatBuffer->dataLenInBytes;
115 
116 		if (WRITE_AND_GET_SIZE == operationType) {
117 			/* Calculate the total data length in bytes */
118 			*totalDataLenInBytes +=
119 			    pCurrClientFlatBuffer->dataLenInBytes;
120 		}
121 
122 		/* Check if providing a physical address in the function. If not
123 		 * we
124 		 * need to convert it to a physical one */
125 		if (CPA_TRUE == isPhysicalAddress) {
126 			pCurrFlatBufDesc->phyBuffer =
127 			    LAC_MEM_CAST_PTR_TO_UINT64(
128 				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
129 		} else {
130 			pCurrFlatBufDesc->phyBuffer =
131 			    LAC_MEM_CAST_PTR_TO_UINT64(
132 				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
133 				    (*pService), pCurrClientFlatBuffer->pData));
134 
135 			if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) {
136 				if (INVALID_PHYSICAL_ADDRESS ==
137 				    pCurrFlatBufDesc->phyBuffer) {
138 					QAT_UTILS_LOG(
139 					    "Unable to get the physical address of the client buffer.\n");
140 					return CPA_STATUS_FAIL;
141 				}
142 			}
143 		}
144 
145 		pCurrFlatBufDesc++;
146 		pCurrClientFlatBuffer++;
147 
148 		numBuffers--;
149 	}
150 
151 	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
152 	return CPA_STATUS_SUCCESS;
153 }
154 
155 /* This function implements the buffer description writes for the traditional
156  * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and
157  * GCM aglorithms */
158 CpaStatus
159 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
160     const CpaBufferList *pUserBufferList,
161     Cpa64U *pBufListAlignedPhyAddr,
162     CpaBoolean isPhysicalAddress,
163     sal_service_t *pService)
164 {
165 	return LacBuffDesc_CommonBufferListDescWrite(
166 	    pUserBufferList,
167 	    pBufListAlignedPhyAddr,
168 	    isPhysicalAddress,
169 	    NULL,
170 	    pService,
171 	    WRITE_AND_ALLOW_ZERO_BUFFER);
172 }
173 
174 /* This function implements the buffer description writes for the traditional
175  * APIs */
176 CpaStatus
177 LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList,
178 				Cpa64U *pBufListAlignedPhyAddr,
179 				CpaBoolean isPhysicalAddress,
180 				sal_service_t *pService)
181 {
182 	return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList,
183 						     pBufListAlignedPhyAddr,
184 						     isPhysicalAddress,
185 						     NULL,
186 						     pService,
187 						     WRITE_NORMAL);
188 }
189 
190 /* This function does the same processing as LacBuffDesc_BufferListDescWrite
191  * but calculate as well the total length in bytes of the buffer list. */
192 CpaStatus
193 LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList,
194 					  Cpa64U *pBufListAlignedPhyAddr,
195 					  CpaBoolean isPhysicalAddress,
196 					  Cpa64U *totalDataLenInBytes,
197 					  sal_service_t *pService)
198 {
199 	Cpa32U numBuffers = 0;
200 	icp_qat_addr_width_t bufListDescPhyAddr = 0;
201 	icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
202 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
203 	icp_buffer_list_desc_t *pBufferListDesc = NULL;
204 	icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
205 	*totalDataLenInBytes = 0;
206 
207 	numBuffers = pUserBufferList->numBuffers;
208 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
209 
210 	/*
211 	 * Get the physical address of this descriptor - need to offset by the
212 	 * alignment restrictions on the buffer descriptors
213 	 */
214 	bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
215 	    (*pService), pUserBufferList->pPrivateMetaData);
216 
217 	if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) {
218 		QAT_UTILS_LOG(
219 		    "Unable to get the physical address of the metadata.\n");
220 		return CPA_STATUS_FAIL;
221 	}
222 
223 	bufListAlignedPhyAddr =
224 	    LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
225 				   ICP_DESCRIPTOR_ALIGNMENT_BYTES);
226 
227 	pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
228 	    (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
229 	    ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
230 	     (LAC_ARCH_UINT)bufListDescPhyAddr));
231 
232 	/* Go past the Buffer List descriptor to the list of buffer descriptors
233 	 */
234 	pCurrFlatBufDesc =
235 	    (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
236 
237 	pBufferListDesc->numBuffers = numBuffers;
238 
239 	while (0 != numBuffers) {
240 		pCurrFlatBufDesc->dataLenInBytes =
241 		    pCurrClientFlatBuffer->dataLenInBytes;
242 
243 		/* Calculate the total data length in bytes */
244 		*totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes;
245 
246 		if (isPhysicalAddress == CPA_TRUE) {
247 			pCurrFlatBufDesc->phyBuffer =
248 			    LAC_MEM_CAST_PTR_TO_UINT64(
249 				(LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
250 		} else {
251 			pCurrFlatBufDesc->phyBuffer =
252 			    LAC_MEM_CAST_PTR_TO_UINT64(
253 				LAC_OS_VIRT_TO_PHYS_EXTERNAL(
254 				    (*pService), pCurrClientFlatBuffer->pData));
255 
256 			if (pCurrFlatBufDesc->phyBuffer == 0) {
257 				QAT_UTILS_LOG(
258 				    "Unable to get the physical address of the client buffer.\n");
259 				return CPA_STATUS_FAIL;
260 			}
261 		}
262 
263 		pCurrFlatBufDesc++;
264 		pCurrClientFlatBuffer++;
265 
266 		numBuffers--;
267 	}
268 
269 	*pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
270 	return CPA_STATUS_SUCCESS;
271 }
272 
273 CpaStatus
274 LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer,
275 			     Cpa64U *pPktSize,
276 			     lac_aligment_shift_t alignmentShiftExpected)
277 {
278 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
279 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
280 
281 	if (0 == pUserFlatBuffer->dataLenInBytes) {
282 		QAT_UTILS_LOG("FlatBuffer empty\n");
283 		return CPA_STATUS_INVALID_PARAM;
284 	}
285 
286 	/* Expected alignment */
287 	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
288 		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
289 					 alignmentShiftExpected)) {
290 			QAT_UTILS_LOG(
291 			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
292 			    1 << alignmentShiftExpected);
293 			return CPA_STATUS_INVALID_PARAM;
294 		}
295 	}
296 
297 	/* Update the total size of the packet. This function being called in a
298 	 * loop
299 	 * for an entire buffer list we need to increment the value */
300 	*pPktSize += pUserFlatBuffer->dataLenInBytes;
301 
302 	return CPA_STATUS_SUCCESS;
303 }
304 
305 CpaStatus
306 LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer,
307 				 Cpa64U *pPktSize,
308 				 lac_aligment_shift_t alignmentShiftExpected)
309 {
310 	LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
311 
312 	if (0 != pUserFlatBuffer->dataLenInBytes) {
313 		LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
314 	}
315 
316 	/* Expected alignment */
317 	if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
318 		if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
319 					 alignmentShiftExpected)) {
320 			QAT_UTILS_LOG(
321 			    "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
322 			    1 << alignmentShiftExpected);
323 			return CPA_STATUS_INVALID_PARAM;
324 		}
325 	}
326 
327 	/* Update the total size of the packet. This function being called in a
328 	 * loop
329 	 * for an entire buffer list we need to increment the value */
330 	*pPktSize += pUserFlatBuffer->dataLenInBytes;
331 
332 	return CPA_STATUS_SUCCESS;
333 }
334 
335 CpaStatus
336 LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList,
337 			     Cpa64U *pPktSize,
338 			     lac_aligment_shift_t alignmentShiftExpected)
339 {
340 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
341 	Cpa32U numBuffers = 0;
342 	CpaStatus status = CPA_STATUS_SUCCESS;
343 
344 	LAC_CHECK_NULL_PARAM(pUserBufferList);
345 	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
346 	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
347 
348 	numBuffers = pUserBufferList->numBuffers;
349 
350 	if (0 == pUserBufferList->numBuffers) {
351 		QAT_UTILS_LOG("Number of buffers is 0.\n");
352 		return CPA_STATUS_INVALID_PARAM;
353 	}
354 
355 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
356 
357 	*pPktSize = 0;
358 	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
359 		status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer,
360 						      pPktSize,
361 						      alignmentShiftExpected);
362 
363 		pCurrClientFlatBuffer++;
364 		numBuffers--;
365 	}
366 	return status;
367 }
368 
369 CpaStatus
370 LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList,
371 				 Cpa64U *pPktSize,
372 				 lac_aligment_shift_t alignmentShiftExpected)
373 {
374 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
375 	Cpa32U numBuffers = 0;
376 	CpaStatus status = CPA_STATUS_SUCCESS;
377 
378 	LAC_CHECK_NULL_PARAM(pUserBufferList);
379 	LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
380 	LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
381 
382 	numBuffers = pUserBufferList->numBuffers;
383 
384 	if (0 == pUserBufferList->numBuffers) {
385 		QAT_UTILS_LOG("Number of buffers is 0.\n");
386 		return CPA_STATUS_INVALID_PARAM;
387 	}
388 
389 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
390 
391 	*pPktSize = 0;
392 	while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
393 		status =
394 		    LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer,
395 						     pPktSize,
396 						     alignmentShiftExpected);
397 
398 		pCurrClientFlatBuffer++;
399 		numBuffers--;
400 	}
401 	return status;
402 }
403 
404 /**
405  ******************************************************************************
406  * @ingroup LacBufferDesc
407  *****************************************************************************/
408 void
409 LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList,
410 				   Cpa64U *pPktSize)
411 {
412 	CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
413 	Cpa32U numBuffers = 0;
414 
415 	pCurrClientFlatBuffer = pUserBufferList->pBuffers;
416 	numBuffers = pUserBufferList->numBuffers;
417 
418 	*pPktSize = 0;
419 	while (0 != numBuffers) {
420 		*pPktSize += pCurrClientFlatBuffer->dataLenInBytes;
421 		pCurrClientFlatBuffer++;
422 		numBuffers--;
423 	}
424 }
425 
426 void
427 LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList,
428 				     Cpa32U offset,
429 				     Cpa32U lenToZero)
430 {
431 	Cpa32U zeroLen = 0, sizeLeftToZero = 0;
432 	Cpa64U currentBufferSize = 0;
433 	CpaFlatBuffer *pBuffer = NULL;
434 	Cpa8U *pZero = NULL;
435 	pBuffer = pBuffList->pBuffers;
436 
437 	/* Take a copy of total length to zero. */
438 	sizeLeftToZero = lenToZero;
439 
440 	while (sizeLeftToZero > 0) {
441 		currentBufferSize = pBuffer->dataLenInBytes;
442 		/* check where to start zeroing */
443 		if (offset >= currentBufferSize) {
444 			/* Need to get to next buffer and reduce
445 			 * offset size by data len of buffer */
446 			offset = offset - pBuffer->dataLenInBytes;
447 			pBuffer++;
448 		} else {
449 			/* Start to Zero from this position */
450 			pZero = (Cpa8U *)pBuffer->pData + offset;
451 
452 			/* Need to calculate the correct number of bytes to zero
453 			 * for this iteration and for this location.
454 			 */
455 			if (sizeLeftToZero >= pBuffer->dataLenInBytes) {
456 				/* The size to zero is spanning buffers, zeroLen
457 				 * in
458 				 * this case is from pZero (position) to end of
459 				 * buffer.
460 				 */
461 				zeroLen = pBuffer->dataLenInBytes - offset;
462 			} else {
463 				/* zeroLen is set to sizeLeftToZero, then check
464 				 * if zeroLen and
465 				 * the offset is greater or equal to the size of
466 				 * the buffer, if
467 				 * yes, adjust the zeroLen to zero out the
468 				 * remainder of this
469 				 * buffer.
470 				 */
471 				zeroLen = sizeLeftToZero;
472 				if ((zeroLen + offset) >=
473 				    pBuffer->dataLenInBytes) {
474 					zeroLen =
475 					    pBuffer->dataLenInBytes - offset;
476 				}
477 			} /* end inner else */
478 			memset((void *)pZero, 0, zeroLen);
479 			sizeLeftToZero = sizeLeftToZero - zeroLen;
480 			/* offset is no longer required as any data left to zero
481 			 * is now
482 			 * at the start of the next buffer. set offset to zero
483 			 * and move on
484 			 * the buffer pointer to the next buffer.
485 			 */
486 			offset = 0;
487 			pBuffer++;
488 
489 		} /* end outer else */
490 
491 	} /* end while */
492 }
493