1 /* SPDX-License-Identifier: GPL-2.0-only
2  * Copyright (C) 2020 Marvell.
3  */
4 
5 #ifndef __OTX2_CPT_REQMGR_H
6 #define __OTX2_CPT_REQMGR_H
7 
8 #include "otx2_cpt_common.h"
9 
10 /* Completion code size and initial value */
11 #define OTX2_CPT_COMPLETION_CODE_SIZE 8
12 #define OTX2_CPT_COMPLETION_CODE_INIT OTX2_CPT_COMP_E_NOTDONE
13 /*
14  * Maximum total number of SG buffers is 100, we divide it equally
15  * between input and output
16  */
17 #define OTX2_CPT_MAX_SG_IN_CNT  50
18 #define OTX2_CPT_MAX_SG_OUT_CNT 50
19 
20 /* DMA mode direct or SG */
21 #define OTX2_CPT_DMA_MODE_DIRECT 0
22 #define OTX2_CPT_DMA_MODE_SG     1
23 
24 /* Context source CPTR or DPTR */
25 #define OTX2_CPT_FROM_CPTR 0
26 #define OTX2_CPT_FROM_DPTR 1
27 
28 #define OTX2_CPT_MAX_REQ_SIZE 65535
29 
30 #define SG_COMPS_MAX    4
31 #define SGV2_COMPS_MAX  3
32 
33 #define SG_COMP_3    3
34 #define SG_COMP_2    2
35 #define SG_COMP_1    1
36 
37 union otx2_cpt_opcode {
38 	u16 flags;
39 	struct {
40 		u8 major;
41 		u8 minor;
42 	} s;
43 };
44 
45 struct otx2_cptvf_request {
46 	u32 param1;
47 	u32 param2;
48 	u16 dlen;
49 	union otx2_cpt_opcode opcode;
50 	dma_addr_t cptr_dma;
51 	void *cptr;
52 };
53 
54 /*
55  * CPT_INST_S software command definitions
56  * Words EI (0-3)
57  */
58 union otx2_cpt_iq_cmd_word0 {
59 	u64 u;
60 	struct {
61 		__be16 opcode;
62 		__be16 param1;
63 		__be16 param2;
64 		__be16 dlen;
65 	} s;
66 };
67 
68 union otx2_cpt_iq_cmd_word3 {
69 	u64 u;
70 	struct {
71 		u64 cptr:61;
72 		u64 grp:3;
73 	} s;
74 };
75 
76 struct otx2_cpt_iq_command {
77 	union otx2_cpt_iq_cmd_word0 cmd;
78 	u64 dptr;
79 	u64 rptr;
80 	union otx2_cpt_iq_cmd_word3 cptr;
81 };
82 
83 struct otx2_cpt_pending_entry {
84 	void *completion_addr;	/* Completion address */
85 	void *info;
86 	/* Kernel async request callback */
87 	void (*callback)(int status, void *arg1, void *arg2);
88 	struct crypto_async_request *areq; /* Async request callback arg */
89 	u8 resume_sender;	/* Notify sender to resume sending requests */
90 	u8 busy;		/* Entry status (free/busy) */
91 };
92 
93 struct otx2_cpt_pending_queue {
94 	struct otx2_cpt_pending_entry *head; /* Head of the queue */
95 	u32 front;		/* Process work from here */
96 	u32 rear;		/* Append new work here */
97 	u32 pending_count;	/* Pending requests count */
98 	u32 qlen;		/* Queue length */
99 	spinlock_t lock;	/* Queue lock */
100 };
101 
102 struct otx2_cpt_buf_ptr {
103 	u8 *vptr;
104 	dma_addr_t dma_addr;
105 	u16 size;
106 };
107 
108 union otx2_cpt_ctrl_info {
109 	u32 flags;
110 	struct {
111 #if defined(__BIG_ENDIAN_BITFIELD)
112 		u32 reserved_6_31:26;
113 		u32 grp:3;	/* Group bits */
114 		u32 dma_mode:2;	/* DMA mode */
115 		u32 se_req:1;	/* To SE core */
116 #else
117 		u32 se_req:1;	/* To SE core */
118 		u32 dma_mode:2;	/* DMA mode */
119 		u32 grp:3;	/* Group bits */
120 		u32 reserved_6_31:26;
121 #endif
122 	} s;
123 };
124 
125 struct otx2_cpt_req_info {
126 	/* Kernel async request callback */
127 	void (*callback)(int status, void *arg1, void *arg2);
128 	struct crypto_async_request *areq; /* Async request callback arg */
129 	struct otx2_cptvf_request req;/* Request information (core specific) */
130 	union otx2_cpt_ctrl_info ctrl;/* User control information */
131 	struct otx2_cpt_buf_ptr in[OTX2_CPT_MAX_SG_IN_CNT];
132 	struct otx2_cpt_buf_ptr out[OTX2_CPT_MAX_SG_OUT_CNT];
133 	u8 *iv_out;     /* IV to send back */
134 	u16 rlen;	/* Output length */
135 	u8 in_cnt;	/* Number of input buffers */
136 	u8 out_cnt;	/* Number of output buffers */
137 	u8 req_type;	/* Type of request */
138 	u8 is_enc;	/* Is a request an encryption request */
139 	u8 is_trunc_hmac;/* Is truncated hmac used */
140 };
141 
142 struct otx2_cpt_inst_info {
143 	struct otx2_cpt_pending_entry *pentry;
144 	struct otx2_cpt_req_info *req;
145 	struct pci_dev *pdev;
146 	void *completion_addr;
147 	u8 *out_buffer;
148 	u8 *in_buffer;
149 	dma_addr_t dptr_baddr;
150 	dma_addr_t rptr_baddr;
151 	dma_addr_t comp_baddr;
152 	unsigned long time_in;
153 	u32 dlen;
154 	u32 dma_len;
155 	u64 gthr_sz;
156 	u64 sctr_sz;
157 	u8 extra_time;
158 };
159 
160 struct otx2_cpt_sglist_component {
161 	__be16 len0;
162 	__be16 len1;
163 	__be16 len2;
164 	__be16 len3;
165 	__be64 ptr0;
166 	__be64 ptr1;
167 	__be64 ptr2;
168 	__be64 ptr3;
169 };
170 
171 struct cn10kb_cpt_sglist_component {
172 	u16 len0;
173 	u16 len1;
174 	u16 len2;
175 	u16 valid_segs;
176 	u64 ptr0;
177 	u64 ptr1;
178 	u64 ptr2;
179 };
180 
otx2_cpt_info_destroy(struct pci_dev * pdev,struct otx2_cpt_inst_info * info)181 static inline void otx2_cpt_info_destroy(struct pci_dev *pdev,
182 					 struct otx2_cpt_inst_info *info)
183 {
184 	struct otx2_cpt_req_info *req;
185 	int i;
186 
187 	if (info->dptr_baddr)
188 		dma_unmap_single(&pdev->dev, info->dptr_baddr,
189 				 info->dma_len, DMA_BIDIRECTIONAL);
190 
191 	if (info->req) {
192 		req = info->req;
193 		for (i = 0; i < req->out_cnt; i++) {
194 			if (req->out[i].dma_addr)
195 				dma_unmap_single(&pdev->dev,
196 						 req->out[i].dma_addr,
197 						 req->out[i].size,
198 						 DMA_BIDIRECTIONAL);
199 		}
200 
201 		for (i = 0; i < req->in_cnt; i++) {
202 			if (req->in[i].dma_addr)
203 				dma_unmap_single(&pdev->dev,
204 						 req->in[i].dma_addr,
205 						 req->in[i].size,
206 						 DMA_BIDIRECTIONAL);
207 		}
208 	}
209 	kfree(info);
210 }
211 
setup_sgio_components(struct pci_dev * pdev,struct otx2_cpt_buf_ptr * list,int buf_count,u8 * buffer)212 static inline int setup_sgio_components(struct pci_dev *pdev,
213 					struct otx2_cpt_buf_ptr *list,
214 					int buf_count, u8 *buffer)
215 {
216 	struct otx2_cpt_sglist_component *sg_ptr;
217 	int components;
218 	int i, j;
219 
220 	if (unlikely(!list)) {
221 		dev_err(&pdev->dev, "Input list pointer is NULL\n");
222 		return -EINVAL;
223 	}
224 
225 	for (i = 0; i < buf_count; i++) {
226 		if (unlikely(!list[i].vptr))
227 			continue;
228 		list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,
229 						  list[i].size,
230 						  DMA_BIDIRECTIONAL);
231 		if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {
232 			dev_err(&pdev->dev, "Dma mapping failed\n");
233 			goto sg_cleanup;
234 		}
235 	}
236 	components = buf_count / SG_COMPS_MAX;
237 	sg_ptr = (struct otx2_cpt_sglist_component *)buffer;
238 	for (i = 0; i < components; i++) {
239 		sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size);
240 		sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size);
241 		sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size);
242 		sg_ptr->len3 = cpu_to_be16(list[i * SG_COMPS_MAX + 3].size);
243 		sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr);
244 		sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr);
245 		sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr);
246 		sg_ptr->ptr3 = cpu_to_be64(list[i * SG_COMPS_MAX + 3].dma_addr);
247 		sg_ptr++;
248 	}
249 	components = buf_count % SG_COMPS_MAX;
250 
251 	switch (components) {
252 	case SG_COMP_3:
253 		sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size);
254 		sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr);
255 		fallthrough;
256 	case SG_COMP_2:
257 		sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size);
258 		sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr);
259 		fallthrough;
260 	case SG_COMP_1:
261 		sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size);
262 		sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr);
263 		break;
264 	default:
265 		break;
266 	}
267 	return 0;
268 
269 sg_cleanup:
270 	for (j = 0; j < i; j++) {
271 		if (list[j].dma_addr) {
272 			dma_unmap_single(&pdev->dev, list[j].dma_addr,
273 					 list[j].size, DMA_BIDIRECTIONAL);
274 		}
275 
276 		list[j].dma_addr = 0;
277 	}
278 	return -EIO;
279 }
280 
sgv2io_components_setup(struct pci_dev * pdev,struct otx2_cpt_buf_ptr * list,int buf_count,u8 * buffer)281 static inline int sgv2io_components_setup(struct pci_dev *pdev,
282 					  struct otx2_cpt_buf_ptr *list,
283 					  int buf_count, u8 *buffer)
284 {
285 	struct cn10kb_cpt_sglist_component *sg_ptr;
286 	int components;
287 	int i, j;
288 
289 	if (unlikely(!list)) {
290 		dev_err(&pdev->dev, "Input list pointer is NULL\n");
291 		return -EFAULT;
292 	}
293 
294 	for (i = 0; i < buf_count; i++) {
295 		if (unlikely(!list[i].vptr))
296 			continue;
297 		list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,
298 						  list[i].size,
299 						  DMA_BIDIRECTIONAL);
300 		if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {
301 			dev_err(&pdev->dev, "Dma mapping failed\n");
302 			goto sg_cleanup;
303 		}
304 	}
305 	components = buf_count / SGV2_COMPS_MAX;
306 	sg_ptr = (struct cn10kb_cpt_sglist_component *)buffer;
307 	for (i = 0; i < components; i++) {
308 		sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size;
309 		sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size;
310 		sg_ptr->len2 = list[i * SGV2_COMPS_MAX + 2].size;
311 		sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr;
312 		sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr;
313 		sg_ptr->ptr2 = list[i * SGV2_COMPS_MAX + 2].dma_addr;
314 		sg_ptr->valid_segs = SGV2_COMPS_MAX;
315 		sg_ptr++;
316 	}
317 	components = buf_count % SGV2_COMPS_MAX;
318 
319 	sg_ptr->valid_segs = components;
320 	switch (components) {
321 	case SG_COMP_2:
322 		sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size;
323 		sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr;
324 		fallthrough;
325 	case SG_COMP_1:
326 		sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size;
327 		sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr;
328 		break;
329 	default:
330 		break;
331 	}
332 	return 0;
333 
334 sg_cleanup:
335 	for (j = 0; j < i; j++) {
336 		if (list[j].dma_addr) {
337 			dma_unmap_single(&pdev->dev, list[j].dma_addr,
338 					 list[j].size, DMA_BIDIRECTIONAL);
339 		}
340 
341 		list[j].dma_addr = 0;
342 	}
343 	return -EIO;
344 }
345 
346 static inline struct otx2_cpt_inst_info *
cn10k_sgv2_info_create(struct pci_dev * pdev,struct otx2_cpt_req_info * req,gfp_t gfp)347 cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
348 		       gfp_t gfp)
349 {
350 	u32 dlen = 0, g_len, sg_len, info_len;
351 	int align = OTX2_CPT_DMA_MINALIGN;
352 	struct otx2_cpt_inst_info *info;
353 	u16 g_sz_bytes, s_sz_bytes;
354 	u32 total_mem_len;
355 	int i;
356 
357 	g_sz_bytes = ((req->in_cnt + 2) / 3) *
358 		      sizeof(struct cn10kb_cpt_sglist_component);
359 	s_sz_bytes = ((req->out_cnt + 2) / 3) *
360 		      sizeof(struct cn10kb_cpt_sglist_component);
361 
362 	g_len = ALIGN(g_sz_bytes, align);
363 	sg_len = ALIGN(g_len + s_sz_bytes, align);
364 	info_len = ALIGN(sizeof(*info), align);
365 	total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s);
366 
367 	info = kzalloc(total_mem_len, gfp);
368 	if (unlikely(!info))
369 		return NULL;
370 
371 	for (i = 0; i < req->in_cnt; i++)
372 		dlen += req->in[i].size;
373 
374 	info->dlen = dlen;
375 	info->in_buffer = (u8 *)info + info_len;
376 	info->gthr_sz = req->in_cnt;
377 	info->sctr_sz = req->out_cnt;
378 
379 	/* Setup gather (input) components */
380 	if (sgv2io_components_setup(pdev, req->in, req->in_cnt,
381 				    info->in_buffer)) {
382 		dev_err(&pdev->dev, "Failed to setup gather list\n");
383 		goto destroy_info;
384 	}
385 
386 	if (sgv2io_components_setup(pdev, req->out, req->out_cnt,
387 				    &info->in_buffer[g_len])) {
388 		dev_err(&pdev->dev, "Failed to setup scatter list\n");
389 		goto destroy_info;
390 	}
391 
392 	info->dma_len = total_mem_len - info_len;
393 	info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,
394 					  info->dma_len, DMA_BIDIRECTIONAL);
395 	if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
396 		dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
397 		goto destroy_info;
398 	}
399 	info->rptr_baddr = info->dptr_baddr + g_len;
400 	/*
401 	 * Get buffer for union otx2_cpt_res_s response
402 	 * structure and its physical address
403 	 */
404 	info->completion_addr = info->in_buffer + sg_len;
405 	info->comp_baddr = info->dptr_baddr + sg_len;
406 
407 	return info;
408 
409 destroy_info:
410 	otx2_cpt_info_destroy(pdev, info);
411 	return NULL;
412 }
413 
414 /* SG list header size in bytes */
415 #define SG_LIST_HDR_SIZE	8
416 static inline struct otx2_cpt_inst_info *
otx2_sg_info_create(struct pci_dev * pdev,struct otx2_cpt_req_info * req,gfp_t gfp)417 otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
418 		    gfp_t gfp)
419 {
420 	int align = OTX2_CPT_DMA_MINALIGN;
421 	struct otx2_cpt_inst_info *info;
422 	u32 dlen, align_dlen, info_len;
423 	u16 g_sz_bytes, s_sz_bytes;
424 	u32 total_mem_len;
425 
426 	if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT ||
427 		     req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) {
428 		dev_err(&pdev->dev, "Error too many sg components\n");
429 		return NULL;
430 	}
431 
432 	g_sz_bytes = ((req->in_cnt + 3) / 4) *
433 		      sizeof(struct otx2_cpt_sglist_component);
434 	s_sz_bytes = ((req->out_cnt + 3) / 4) *
435 		      sizeof(struct otx2_cpt_sglist_component);
436 
437 	dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
438 	align_dlen = ALIGN(dlen, align);
439 	info_len = ALIGN(sizeof(*info), align);
440 	total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s);
441 
442 	info = kzalloc(total_mem_len, gfp);
443 	if (unlikely(!info))
444 		return NULL;
445 
446 	info->dlen = dlen;
447 	info->in_buffer = (u8 *)info + info_len;
448 
449 	((u16 *)info->in_buffer)[0] = req->out_cnt;
450 	((u16 *)info->in_buffer)[1] = req->in_cnt;
451 	((u16 *)info->in_buffer)[2] = 0;
452 	((u16 *)info->in_buffer)[3] = 0;
453 	cpu_to_be64s((u64 *)info->in_buffer);
454 
455 	/* Setup gather (input) components */
456 	if (setup_sgio_components(pdev, req->in, req->in_cnt,
457 				  &info->in_buffer[8])) {
458 		dev_err(&pdev->dev, "Failed to setup gather list\n");
459 		goto destroy_info;
460 	}
461 
462 	if (setup_sgio_components(pdev, req->out, req->out_cnt,
463 				  &info->in_buffer[8 + g_sz_bytes])) {
464 		dev_err(&pdev->dev, "Failed to setup scatter list\n");
465 		goto destroy_info;
466 	}
467 
468 	info->dma_len = total_mem_len - info_len;
469 	info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,
470 					  info->dma_len, DMA_BIDIRECTIONAL);
471 	if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
472 		dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
473 		goto destroy_info;
474 	}
475 	/*
476 	 * Get buffer for union otx2_cpt_res_s response
477 	 * structure and its physical address
478 	 */
479 	info->completion_addr = info->in_buffer + align_dlen;
480 	info->comp_baddr = info->dptr_baddr + align_dlen;
481 
482 	return info;
483 
484 destroy_info:
485 	otx2_cpt_info_destroy(pdev, info);
486 	return NULL;
487 }
488 
489 struct otx2_cptlf_wqe;
490 int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
491 			int cpu_num);
492 void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe);
493 int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev);
494 
495 #endif /* __OTX2_CPT_REQMGR_H */
496