1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /**
4 *****************************************************************************
5 * @file lac_buffer_desc.c Utility functions for setting buffer descriptors
6 *
7 * @ingroup LacBufferDesc
8 *
9 *****************************************************************************/
10
11 /*
12 *******************************************************************************
13 * Include header files
14 *******************************************************************************
15 */
16 #include "qat_utils.h"
17 #include "icp_accel_devices.h"
18 #include "icp_adf_debug.h"
19 #include "icp_adf_init.h"
20 #include "lac_list.h"
21 #include "lac_sal_types.h"
22 #include "lac_buffer_desc.h"
23 #include "lac_mem.h"
24 #include "cpa_cy_common.h"
25
26 /*
27 *******************************************************************************
28 * Define public/global function definitions
29 *******************************************************************************
30 */
31 /* Invalid physical address value */
32 #define INVALID_PHYSICAL_ADDRESS 0
33
34 /* Indicates what type of buffer writes need to be perfomed */
35 typedef enum lac_buff_write_op_e {
36 WRITE_NORMAL = 0,
37 WRITE_AND_GET_SIZE,
38 WRITE_AND_ALLOW_ZERO_BUFFER,
39 } lac_buff_write_op_t;
40
41 /* This function implements the buffer description writes for the traditional
42 * APIs */
43 static CpaStatus
LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList * pUserBufferList,Cpa64U * pBufListAlignedPhyAddr,CpaBoolean isPhysicalAddress,Cpa64U * totalDataLenInBytes,sal_service_t * pService,lac_buff_write_op_t operationType)44 LacBuffDesc_CommonBufferListDescWrite(const CpaBufferList *pUserBufferList,
45 Cpa64U *pBufListAlignedPhyAddr,
46 CpaBoolean isPhysicalAddress,
47 Cpa64U *totalDataLenInBytes,
48 sal_service_t *pService,
49 lac_buff_write_op_t operationType)
50 {
51 Cpa32U numBuffers = 0;
52 icp_qat_addr_width_t bufListDescPhyAddr = 0;
53 icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
54 CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
55 icp_buffer_list_desc_t *pBufferListDesc = NULL;
56 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
57
58 if (WRITE_AND_GET_SIZE == operationType) {
59 *totalDataLenInBytes = 0;
60 }
61
62 numBuffers = pUserBufferList->numBuffers;
63 pCurrClientFlatBuffer = pUserBufferList->pBuffers;
64
65 /*
66 * Get the physical address of this descriptor - need to offset by the
67 * alignment restrictions on the buffer descriptors
68 */
69 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
70 (*pService), pUserBufferList->pPrivateMetaData);
71
72 if (bufListDescPhyAddr == 0) {
73 QAT_UTILS_LOG(
74 "Unable to get the physical address of the metadata.\n");
75 return CPA_STATUS_FAIL;
76 }
77
78 bufListAlignedPhyAddr =
79 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
80 ICP_DESCRIPTOR_ALIGNMENT_BYTES);
81
82 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
83 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
84 ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
85 (LAC_ARCH_UINT)bufListDescPhyAddr));
86
87 /* Go past the Buffer List descriptor to the list of buffer descriptors
88 */
89 pCurrFlatBufDesc =
90 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
91
92 pBufferListDesc->numBuffers = numBuffers;
93
94 if (WRITE_AND_GET_SIZE != operationType) {
95 /* Defining zero buffers is useful for example if running zero
96 * length
97 * hash */
98 if (0 == numBuffers) {
99 /* In the case where there are zero buffers within the
100 * BufList
101 * it is required by firmware that the number is set to
102 * 1
103 * but the phyBuffer and dataLenInBytes are set to
104 * NULL.*/
105 pBufferListDesc->numBuffers = 1;
106 pCurrFlatBufDesc->dataLenInBytes = 0;
107 pCurrFlatBufDesc->phyBuffer = 0;
108 }
109 }
110
111 while (0 != numBuffers) {
112 pCurrFlatBufDesc->dataLenInBytes =
113 pCurrClientFlatBuffer->dataLenInBytes;
114
115 if (WRITE_AND_GET_SIZE == operationType) {
116 /* Calculate the total data length in bytes */
117 *totalDataLenInBytes +=
118 pCurrClientFlatBuffer->dataLenInBytes;
119 }
120
121 /* Check if providing a physical address in the function. If not
122 * we
123 * need to convert it to a physical one */
124 if (CPA_TRUE == isPhysicalAddress) {
125 pCurrFlatBufDesc->phyBuffer =
126 LAC_MEM_CAST_PTR_TO_UINT64(
127 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
128 } else {
129 pCurrFlatBufDesc->phyBuffer =
130 LAC_MEM_CAST_PTR_TO_UINT64(
131 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
132 (*pService), pCurrClientFlatBuffer->pData));
133
134 if (WRITE_AND_ALLOW_ZERO_BUFFER != operationType) {
135 if (INVALID_PHYSICAL_ADDRESS ==
136 pCurrFlatBufDesc->phyBuffer) {
137 QAT_UTILS_LOG(
138 "Unable to get the physical address of the client buffer.\n");
139 return CPA_STATUS_FAIL;
140 }
141 }
142 }
143
144 pCurrFlatBufDesc++;
145 pCurrClientFlatBuffer++;
146
147 numBuffers--;
148 }
149
150 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
151 return CPA_STATUS_SUCCESS;
152 }
153
154 /* This function implements the buffer description writes for the traditional
155 * APIs Zero length buffers are allowed, should be used for CHA-CHA-POLY and
156 * GCM aglorithms */
157 CpaStatus
LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(const CpaBufferList * pUserBufferList,Cpa64U * pBufListAlignedPhyAddr,CpaBoolean isPhysicalAddress,sal_service_t * pService)158 LacBuffDesc_BufferListDescWriteAndAllowZeroBuffer(
159 const CpaBufferList *pUserBufferList,
160 Cpa64U *pBufListAlignedPhyAddr,
161 CpaBoolean isPhysicalAddress,
162 sal_service_t *pService)
163 {
164 return LacBuffDesc_CommonBufferListDescWrite(
165 pUserBufferList,
166 pBufListAlignedPhyAddr,
167 isPhysicalAddress,
168 NULL,
169 pService,
170 WRITE_AND_ALLOW_ZERO_BUFFER);
171 }
172
173 /* This function implements the buffer description writes for the traditional
174 * APIs */
175 CpaStatus
LacBuffDesc_BufferListDescWrite(const CpaBufferList * pUserBufferList,Cpa64U * pBufListAlignedPhyAddr,CpaBoolean isPhysicalAddress,sal_service_t * pService)176 LacBuffDesc_BufferListDescWrite(const CpaBufferList *pUserBufferList,
177 Cpa64U *pBufListAlignedPhyAddr,
178 CpaBoolean isPhysicalAddress,
179 sal_service_t *pService)
180 {
181 return LacBuffDesc_CommonBufferListDescWrite(pUserBufferList,
182 pBufListAlignedPhyAddr,
183 isPhysicalAddress,
184 NULL,
185 pService,
186 WRITE_NORMAL);
187 }
188
189 /* This function does the same processing as LacBuffDesc_BufferListDescWrite
190 * but calculate as well the total length in bytes of the buffer list. */
191 CpaStatus
LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList * pUserBufferList,Cpa64U * pBufListAlignedPhyAddr,CpaBoolean isPhysicalAddress,Cpa64U * totalDataLenInBytes,sal_service_t * pService)192 LacBuffDesc_BufferListDescWriteAndGetSize(const CpaBufferList *pUserBufferList,
193 Cpa64U *pBufListAlignedPhyAddr,
194 CpaBoolean isPhysicalAddress,
195 Cpa64U *totalDataLenInBytes,
196 sal_service_t *pService)
197 {
198 Cpa32U numBuffers = 0;
199 icp_qat_addr_width_t bufListDescPhyAddr = 0;
200 icp_qat_addr_width_t bufListAlignedPhyAddr = 0;
201 CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
202 icp_buffer_list_desc_t *pBufferListDesc = NULL;
203 icp_flat_buffer_desc_t *pCurrFlatBufDesc = NULL;
204 *totalDataLenInBytes = 0;
205
206 numBuffers = pUserBufferList->numBuffers;
207 pCurrClientFlatBuffer = pUserBufferList->pBuffers;
208
209 /*
210 * Get the physical address of this descriptor - need to offset by the
211 * alignment restrictions on the buffer descriptors
212 */
213 bufListDescPhyAddr = (icp_qat_addr_width_t)LAC_OS_VIRT_TO_PHYS_EXTERNAL(
214 (*pService), pUserBufferList->pPrivateMetaData);
215
216 if (INVALID_PHYSICAL_ADDRESS == bufListDescPhyAddr) {
217 QAT_UTILS_LOG(
218 "Unable to get the physical address of the metadata.\n");
219 return CPA_STATUS_FAIL;
220 }
221
222 bufListAlignedPhyAddr =
223 LAC_ALIGN_POW2_ROUNDUP(bufListDescPhyAddr,
224 ICP_DESCRIPTOR_ALIGNMENT_BYTES);
225
226 pBufferListDesc = (icp_buffer_list_desc_t *)(LAC_ARCH_UINT)(
227 (LAC_ARCH_UINT)pUserBufferList->pPrivateMetaData +
228 ((LAC_ARCH_UINT)bufListAlignedPhyAddr -
229 (LAC_ARCH_UINT)bufListDescPhyAddr));
230
231 /* Go past the Buffer List descriptor to the list of buffer descriptors
232 */
233 pCurrFlatBufDesc =
234 (icp_flat_buffer_desc_t *)((pBufferListDesc->phyBuffers));
235
236 pBufferListDesc->numBuffers = numBuffers;
237
238 while (0 != numBuffers) {
239 pCurrFlatBufDesc->dataLenInBytes =
240 pCurrClientFlatBuffer->dataLenInBytes;
241
242 /* Calculate the total data length in bytes */
243 *totalDataLenInBytes += pCurrClientFlatBuffer->dataLenInBytes;
244
245 if (isPhysicalAddress == CPA_TRUE) {
246 pCurrFlatBufDesc->phyBuffer =
247 LAC_MEM_CAST_PTR_TO_UINT64(
248 (LAC_ARCH_UINT)(pCurrClientFlatBuffer->pData));
249 } else {
250 pCurrFlatBufDesc->phyBuffer =
251 LAC_MEM_CAST_PTR_TO_UINT64(
252 LAC_OS_VIRT_TO_PHYS_EXTERNAL(
253 (*pService), pCurrClientFlatBuffer->pData));
254
255 if (pCurrFlatBufDesc->phyBuffer == 0) {
256 QAT_UTILS_LOG(
257 "Unable to get the physical address of the client buffer.\n");
258 return CPA_STATUS_FAIL;
259 }
260 }
261
262 pCurrFlatBufDesc++;
263 pCurrClientFlatBuffer++;
264
265 numBuffers--;
266 }
267
268 *pBufListAlignedPhyAddr = bufListAlignedPhyAddr;
269 return CPA_STATUS_SUCCESS;
270 }
271
272 CpaStatus
LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer * pUserFlatBuffer,Cpa64U * pPktSize,lac_aligment_shift_t alignmentShiftExpected)273 LacBuffDesc_FlatBufferVerify(const CpaFlatBuffer *pUserFlatBuffer,
274 Cpa64U *pPktSize,
275 lac_aligment_shift_t alignmentShiftExpected)
276 {
277 LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
278 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
279
280 if (0 == pUserFlatBuffer->dataLenInBytes) {
281 QAT_UTILS_LOG("FlatBuffer empty\n");
282 return CPA_STATUS_INVALID_PARAM;
283 }
284
285 /* Expected alignment */
286 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
287 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
288 alignmentShiftExpected)) {
289 QAT_UTILS_LOG(
290 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
291 1 << alignmentShiftExpected);
292 return CPA_STATUS_INVALID_PARAM;
293 }
294 }
295
296 /* Update the total size of the packet. This function being called in a
297 * loop
298 * for an entire buffer list we need to increment the value */
299 *pPktSize += pUserFlatBuffer->dataLenInBytes;
300
301 return CPA_STATUS_SUCCESS;
302 }
303
304 CpaStatus
LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer * pUserFlatBuffer,Cpa64U * pPktSize,lac_aligment_shift_t alignmentShiftExpected)305 LacBuffDesc_FlatBufferVerifyNull(const CpaFlatBuffer *pUserFlatBuffer,
306 Cpa64U *pPktSize,
307 lac_aligment_shift_t alignmentShiftExpected)
308 {
309 LAC_CHECK_NULL_PARAM(pUserFlatBuffer);
310
311 if (0 != pUserFlatBuffer->dataLenInBytes) {
312 LAC_CHECK_NULL_PARAM(pUserFlatBuffer->pData);
313 }
314
315 /* Expected alignment */
316 if (LAC_NO_ALIGNMENT_SHIFT != alignmentShiftExpected) {
317 if (!LAC_ADDRESS_ALIGNED(pUserFlatBuffer->pData,
318 alignmentShiftExpected)) {
319 QAT_UTILS_LOG(
320 "FlatBuffer not aligned correctly - expected alignment of %u bytes.\n",
321 1 << alignmentShiftExpected);
322 return CPA_STATUS_INVALID_PARAM;
323 }
324 }
325
326 /* Update the total size of the packet. This function being called in a
327 * loop
328 * for an entire buffer list we need to increment the value */
329 *pPktSize += pUserFlatBuffer->dataLenInBytes;
330
331 return CPA_STATUS_SUCCESS;
332 }
333
334 CpaStatus
LacBuffDesc_BufferListVerify(const CpaBufferList * pUserBufferList,Cpa64U * pPktSize,lac_aligment_shift_t alignmentShiftExpected)335 LacBuffDesc_BufferListVerify(const CpaBufferList *pUserBufferList,
336 Cpa64U *pPktSize,
337 lac_aligment_shift_t alignmentShiftExpected)
338 {
339 CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
340 Cpa32U numBuffers = 0;
341 CpaStatus status = CPA_STATUS_SUCCESS;
342
343 LAC_CHECK_NULL_PARAM(pUserBufferList);
344 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
345 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
346
347 numBuffers = pUserBufferList->numBuffers;
348
349 if (0 == pUserBufferList->numBuffers) {
350 QAT_UTILS_LOG("Number of buffers is 0.\n");
351 return CPA_STATUS_INVALID_PARAM;
352 }
353
354 pCurrClientFlatBuffer = pUserBufferList->pBuffers;
355
356 *pPktSize = 0;
357 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
358 status = LacBuffDesc_FlatBufferVerify(pCurrClientFlatBuffer,
359 pPktSize,
360 alignmentShiftExpected);
361
362 pCurrClientFlatBuffer++;
363 numBuffers--;
364 }
365 return status;
366 }
367
368 CpaStatus
LacBuffDesc_BufferListVerifyNull(const CpaBufferList * pUserBufferList,Cpa64U * pPktSize,lac_aligment_shift_t alignmentShiftExpected)369 LacBuffDesc_BufferListVerifyNull(const CpaBufferList *pUserBufferList,
370 Cpa64U *pPktSize,
371 lac_aligment_shift_t alignmentShiftExpected)
372 {
373 CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
374 Cpa32U numBuffers = 0;
375 CpaStatus status = CPA_STATUS_SUCCESS;
376
377 LAC_CHECK_NULL_PARAM(pUserBufferList);
378 LAC_CHECK_NULL_PARAM(pUserBufferList->pBuffers);
379 LAC_CHECK_NULL_PARAM(pUserBufferList->pPrivateMetaData);
380
381 numBuffers = pUserBufferList->numBuffers;
382
383 if (0 == pUserBufferList->numBuffers) {
384 QAT_UTILS_LOG("Number of buffers is 0.\n");
385 return CPA_STATUS_INVALID_PARAM;
386 }
387
388 pCurrClientFlatBuffer = pUserBufferList->pBuffers;
389
390 *pPktSize = 0;
391 while (0 != numBuffers && status == CPA_STATUS_SUCCESS) {
392 status =
393 LacBuffDesc_FlatBufferVerifyNull(pCurrClientFlatBuffer,
394 pPktSize,
395 alignmentShiftExpected);
396
397 pCurrClientFlatBuffer++;
398 numBuffers--;
399 }
400 return status;
401 }
402
403 /**
404 ******************************************************************************
405 * @ingroup LacBufferDesc
406 *****************************************************************************/
407 void
LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList * pUserBufferList,Cpa64U * pPktSize)408 LacBuffDesc_BufferListTotalSizeGet(const CpaBufferList *pUserBufferList,
409 Cpa64U *pPktSize)
410 {
411 CpaFlatBuffer *pCurrClientFlatBuffer = NULL;
412 Cpa32U numBuffers = 0;
413
414 pCurrClientFlatBuffer = pUserBufferList->pBuffers;
415 numBuffers = pUserBufferList->numBuffers;
416
417 *pPktSize = 0;
418 while (0 != numBuffers) {
419 *pPktSize += pCurrClientFlatBuffer->dataLenInBytes;
420 pCurrClientFlatBuffer++;
421 numBuffers--;
422 }
423 }
424
425 void
LacBuffDesc_BufferListZeroFromOffset(CpaBufferList * pBuffList,Cpa32U offset,Cpa32U lenToZero)426 LacBuffDesc_BufferListZeroFromOffset(CpaBufferList *pBuffList,
427 Cpa32U offset,
428 Cpa32U lenToZero)
429 {
430 Cpa32U zeroLen = 0, sizeLeftToZero = 0;
431 Cpa64U currentBufferSize = 0;
432 CpaFlatBuffer *pBuffer = NULL;
433 Cpa8U *pZero = NULL;
434 pBuffer = pBuffList->pBuffers;
435
436 /* Take a copy of total length to zero. */
437 sizeLeftToZero = lenToZero;
438
439 while (sizeLeftToZero > 0) {
440 currentBufferSize = pBuffer->dataLenInBytes;
441 /* check where to start zeroing */
442 if (offset >= currentBufferSize) {
443 /* Need to get to next buffer and reduce
444 * offset size by data len of buffer */
445 offset = offset - pBuffer->dataLenInBytes;
446 pBuffer++;
447 } else {
448 /* Start to Zero from this position */
449 pZero = (Cpa8U *)pBuffer->pData + offset;
450
451 /* Need to calculate the correct number of bytes to zero
452 * for this iteration and for this location.
453 */
454 if (sizeLeftToZero >= pBuffer->dataLenInBytes) {
455 /* The size to zero is spanning buffers, zeroLen
456 * in
457 * this case is from pZero (position) to end of
458 * buffer.
459 */
460 zeroLen = pBuffer->dataLenInBytes - offset;
461 } else {
462 /* zeroLen is set to sizeLeftToZero, then check
463 * if zeroLen and
464 * the offset is greater or equal to the size of
465 * the buffer, if
466 * yes, adjust the zeroLen to zero out the
467 * remainder of this
468 * buffer.
469 */
470 zeroLen = sizeLeftToZero;
471 if ((zeroLen + offset) >=
472 pBuffer->dataLenInBytes) {
473 zeroLen =
474 pBuffer->dataLenInBytes - offset;
475 }
476 } /* end inner else */
477 memset((void *)pZero, 0, zeroLen);
478 sizeLeftToZero = sizeLeftToZero - zeroLen;
479 /* offset is no longer required as any data left to zero
480 * is now
481 * at the start of the next buffer. set offset to zero
482 * and move on
483 * the buffer pointer to the next buffer.
484 */
485 offset = 0;
486 pBuffer++;
487
488 } /* end outer else */
489
490 } /* end while */
491 }
492