1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <crypto/internal/acompress.h>
4 #include <linux/bitfield.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/scatterlist.h>
7 #include "zip.h"
8 
9 /* hisi_zip_sqe dw3 */
10 #define HZIP_BD_STATUS_M			GENMASK(7, 0)
11 /* hisi_zip_sqe dw7 */
12 #define HZIP_IN_SGE_DATA_OFFSET_M		GENMASK(23, 0)
13 #define HZIP_SQE_TYPE_M				GENMASK(31, 28)
14 /* hisi_zip_sqe dw8 */
15 #define HZIP_OUT_SGE_DATA_OFFSET_M		GENMASK(23, 0)
16 /* hisi_zip_sqe dw9 */
17 #define HZIP_REQ_TYPE_M				GENMASK(7, 0)
18 #define HZIP_ALG_TYPE_ZLIB			0x02
19 #define HZIP_ALG_TYPE_GZIP			0x03
20 #define HZIP_BUF_TYPE_M				GENMASK(11, 8)
21 #define HZIP_PBUFFER				0x0
22 #define HZIP_SGL				0x1
23 
24 #define HZIP_ZLIB_HEAD_SIZE			2
25 #define HZIP_GZIP_HEAD_SIZE			10
26 
27 #define GZIP_HEAD_FHCRC_BIT			BIT(1)
28 #define GZIP_HEAD_FEXTRA_BIT			BIT(2)
29 #define GZIP_HEAD_FNAME_BIT			BIT(3)
30 #define GZIP_HEAD_FCOMMENT_BIT			BIT(4)
31 
32 #define GZIP_HEAD_FLG_SHIFT			3
33 #define GZIP_HEAD_FEXTRA_SHIFT			10
34 #define GZIP_HEAD_FEXTRA_XLEN			2UL
35 #define GZIP_HEAD_FHCRC_SIZE			2
36 
37 #define HZIP_GZIP_HEAD_BUF			256
38 #define HZIP_ALG_PRIORITY			300
39 #define HZIP_SGL_SGE_NR				10
40 
41 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
42 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {
43 	0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03
44 };
45 
46 enum hisi_zip_alg_type {
47 	HZIP_ALG_TYPE_COMP = 0,
48 	HZIP_ALG_TYPE_DECOMP = 1,
49 };
50 
51 enum {
52 	HZIP_QPC_COMP,
53 	HZIP_QPC_DECOMP,
54 	HZIP_CTX_Q_NUM
55 };
56 
57 #define COMP_NAME_TO_TYPE(alg_name)					\
58 	(!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB :	\
59 	 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0)		\
60 
61 #define TO_HEAD_SIZE(req_type)						\
62 	(((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) :	\
63 	 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0)	\
64 
65 #define TO_HEAD(req_type)						\
66 	(((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head :		\
67 	 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL)		\
68 
69 struct hisi_zip_req {
70 	struct acomp_req *req;
71 	u32 sskip;
72 	u32 dskip;
73 	struct hisi_acc_hw_sgl *hw_src;
74 	struct hisi_acc_hw_sgl *hw_dst;
75 	dma_addr_t dma_src;
76 	dma_addr_t dma_dst;
77 	u16 req_id;
78 };
79 
80 struct hisi_zip_req_q {
81 	struct hisi_zip_req *q;
82 	unsigned long *req_bitmap;
83 	rwlock_t req_lock;
84 	u16 size;
85 };
86 
87 struct hisi_zip_qp_ctx {
88 	struct hisi_qp *qp;
89 	struct hisi_zip_req_q req_q;
90 	struct hisi_acc_sgl_pool *sgl_pool;
91 	struct hisi_zip *zip_dev;
92 	struct hisi_zip_ctx *ctx;
93 };
94 
95 struct hisi_zip_sqe_ops {
96 	u8 sqe_type;
97 	void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
98 	void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
99 	void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type);
100 	void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type);
101 	void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
102 	void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type);
103 	u32 (*get_tag)(struct hisi_zip_sqe *sqe);
104 	u32 (*get_status)(struct hisi_zip_sqe *sqe);
105 	u32 (*get_dstlen)(struct hisi_zip_sqe *sqe);
106 };
107 
108 struct hisi_zip_ctx {
109 	struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
110 	const struct hisi_zip_sqe_ops *ops;
111 };
112 
113 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp)
114 {
115 	int ret;
116 	u16 n;
117 
118 	if (!val)
119 		return -EINVAL;
120 
121 	ret = kstrtou16(val, 10, &n);
122 	if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX)
123 		return -EINVAL;
124 
125 	return param_set_int(val, kp);
126 }
127 
128 static const struct kernel_param_ops sgl_sge_nr_ops = {
129 	.set = sgl_sge_nr_set,
130 	.get = param_get_int,
131 };
132 
133 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR;
134 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444);
135 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)");
136 
137 static u16 get_extra_field_size(const u8 *start)
138 {
139 	return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
140 }
141 
142 static u32 get_name_field_size(const u8 *start)
143 {
144 	return strlen(start) + 1;
145 }
146 
147 static u32 get_comment_field_size(const u8 *start)
148 {
149 	return strlen(start) + 1;
150 }
151 
152 static u32 __get_gzip_head_size(const u8 *src)
153 {
154 	u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
155 	u32 size = GZIP_HEAD_FEXTRA_SHIFT;
156 
157 	if (head_flg & GZIP_HEAD_FEXTRA_BIT)
158 		size += get_extra_field_size(src + size);
159 	if (head_flg & GZIP_HEAD_FNAME_BIT)
160 		size += get_name_field_size(src + size);
161 	if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
162 		size += get_comment_field_size(src + size);
163 	if (head_flg & GZIP_HEAD_FHCRC_BIT)
164 		size += GZIP_HEAD_FHCRC_SIZE;
165 
166 	return size;
167 }
168 
169 static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl)
170 {
171 	char buf[HZIP_GZIP_HEAD_BUF];
172 
173 	sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
174 
175 	return __get_gzip_head_size(buf);
176 }
177 
178 static int add_comp_head(struct scatterlist *dst, u8 req_type)
179 {
180 	int head_size = TO_HEAD_SIZE(req_type);
181 	const u8 *head = TO_HEAD(req_type);
182 	int ret;
183 
184 	ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
185 	if (ret != head_size) {
186 		pr_err("the head size of buffer is wrong (%d)!\n", ret);
187 		return -ENOMEM;
188 	}
189 
190 	return head_size;
191 }
192 
193 static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type)
194 {
195 	if (!acomp_req->src || !acomp_req->slen)
196 		return -EINVAL;
197 
198 	if (req_type == HZIP_ALG_TYPE_GZIP &&
199 	    acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)
200 		return -EINVAL;
201 
202 	switch (req_type) {
203 	case HZIP_ALG_TYPE_ZLIB:
204 		return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
205 	case HZIP_ALG_TYPE_GZIP:
206 		return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP);
207 	default:
208 		pr_err("request type does not support!\n");
209 		return -EINVAL;
210 	}
211 }
212 
213 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
214 						struct hisi_zip_qp_ctx *qp_ctx,
215 						size_t head_size, bool is_comp)
216 {
217 	struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
218 	struct hisi_zip_req *q = req_q->q;
219 	struct hisi_zip_req *req_cache;
220 	int req_id;
221 
222 	write_lock(&req_q->req_lock);
223 
224 	req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
225 	if (req_id >= req_q->size) {
226 		write_unlock(&req_q->req_lock);
227 		dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
228 		return ERR_PTR(-EAGAIN);
229 	}
230 	set_bit(req_id, req_q->req_bitmap);
231 
232 	req_cache = q + req_id;
233 	req_cache->req_id = req_id;
234 	req_cache->req = req;
235 
236 	if (is_comp) {
237 		req_cache->sskip = 0;
238 		req_cache->dskip = head_size;
239 	} else {
240 		req_cache->sskip = head_size;
241 		req_cache->dskip = 0;
242 	}
243 
244 	write_unlock(&req_q->req_lock);
245 
246 	return req_cache;
247 }
248 
249 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
250 				struct hisi_zip_req *req)
251 {
252 	struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
253 
254 	write_lock(&req_q->req_lock);
255 	clear_bit(req->req_id, req_q->req_bitmap);
256 	memset(req, 0, sizeof(struct hisi_zip_req));
257 	write_unlock(&req_q->req_lock);
258 }
259 
260 static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
261 {
262 	sqe->source_addr_l = lower_32_bits(req->dma_src);
263 	sqe->source_addr_h = upper_32_bits(req->dma_src);
264 	sqe->dest_addr_l = lower_32_bits(req->dma_dst);
265 	sqe->dest_addr_h = upper_32_bits(req->dma_dst);
266 }
267 
268 static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
269 {
270 	struct acomp_req *a_req = req->req;
271 
272 	sqe->input_data_length = a_req->slen - req->sskip;
273 	sqe->dest_avail_out = a_req->dlen - req->dskip;
274 	sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip);
275 	sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip);
276 }
277 
278 static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
279 {
280 	u32 val;
281 
282 	val = sqe->dw9 & ~HZIP_BUF_TYPE_M;
283 	val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
284 	sqe->dw9 = val;
285 }
286 
287 static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type)
288 {
289 	u32 val;
290 
291 	val = sqe->dw9 & ~HZIP_REQ_TYPE_M;
292 	val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
293 	sqe->dw9 = val;
294 }
295 
296 static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
297 {
298 	sqe->dw13 = req->req_id;
299 }
300 
301 static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
302 {
303 	sqe->dw26 = req->req_id;
304 }
305 
306 static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type)
307 {
308 	u32 val;
309 
310 	val = sqe->dw7 & ~HZIP_SQE_TYPE_M;
311 	val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type);
312 	sqe->dw7 = val;
313 }
314 
315 static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe,
316 			      u8 req_type, struct hisi_zip_req *req)
317 {
318 	const struct hisi_zip_sqe_ops *ops = ctx->ops;
319 
320 	memset(sqe, 0, sizeof(struct hisi_zip_sqe));
321 
322 	ops->fill_addr(sqe, req);
323 	ops->fill_buf_size(sqe, req);
324 	ops->fill_buf_type(sqe, HZIP_SGL);
325 	ops->fill_req_type(sqe, req_type);
326 	ops->fill_tag(sqe, req);
327 	ops->fill_sqe_type(sqe, ops->sqe_type);
328 }
329 
330 static int hisi_zip_do_work(struct hisi_zip_req *req,
331 			    struct hisi_zip_qp_ctx *qp_ctx)
332 {
333 	struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
334 	struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
335 	struct acomp_req *a_req = req->req;
336 	struct hisi_qp *qp = qp_ctx->qp;
337 	struct device *dev = &qp->qm->pdev->dev;
338 	struct hisi_zip_sqe zip_sqe;
339 	int ret;
340 
341 	if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)
342 		return -EINVAL;
343 
344 	req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
345 						    req->req_id << 1, &req->dma_src);
346 	if (IS_ERR(req->hw_src)) {
347 		dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n",
348 			PTR_ERR(req->hw_src));
349 		return PTR_ERR(req->hw_src);
350 	}
351 
352 	req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
353 						    (req->req_id << 1) + 1,
354 						    &req->dma_dst);
355 	if (IS_ERR(req->hw_dst)) {
356 		ret = PTR_ERR(req->hw_dst);
357 		dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n",
358 			ret);
359 		goto err_unmap_input;
360 	}
361 
362 	hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req);
363 
364 	/* send command to start a task */
365 	atomic64_inc(&dfx->send_cnt);
366 	ret = hisi_qp_send(qp, &zip_sqe);
367 	if (ret < 0) {
368 		atomic64_inc(&dfx->send_busy_cnt);
369 		ret = -EAGAIN;
370 		dev_dbg_ratelimited(dev, "failed to send request!\n");
371 		goto err_unmap_output;
372 	}
373 
374 	return -EINPROGRESS;
375 
376 err_unmap_output:
377 	hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst);
378 err_unmap_input:
379 	hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src);
380 	return ret;
381 }
382 
383 static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe)
384 {
385 	return sqe->dw13;
386 }
387 
388 static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe)
389 {
390 	return sqe->dw26;
391 }
392 
393 static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe)
394 {
395 	return sqe->dw3 & HZIP_BD_STATUS_M;
396 }
397 
398 static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe)
399 {
400 	return sqe->produced;
401 }
402 
403 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
404 {
405 	struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
406 	const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops;
407 	struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
408 	struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
409 	struct device *dev = &qp->qm->pdev->dev;
410 	struct hisi_zip_sqe *sqe = data;
411 	u32 tag = ops->get_tag(sqe);
412 	struct hisi_zip_req *req = req_q->q + tag;
413 	struct acomp_req *acomp_req = req->req;
414 	u32 status, dlen, head_size;
415 	int err = 0;
416 
417 	atomic64_inc(&dfx->recv_cnt);
418 	status = ops->get_status(sqe);
419 	if (status != 0 && status != HZIP_NC_ERR) {
420 		dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
421 			(qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
422 			sqe->produced);
423 		atomic64_inc(&dfx->err_bd_cnt);
424 		err = -EIO;
425 	}
426 
427 	dlen = ops->get_dstlen(sqe);
428 
429 	hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src);
430 	hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst);
431 
432 	head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
433 	acomp_req->dlen = dlen + head_size;
434 
435 	if (acomp_req->base.complete)
436 		acomp_request_complete(acomp_req, err);
437 
438 	hisi_zip_remove_req(qp_ctx, req);
439 }
440 
441 static int hisi_zip_acompress(struct acomp_req *acomp_req)
442 {
443 	struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
444 	struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
445 	struct device *dev = &qp_ctx->qp->qm->pdev->dev;
446 	struct hisi_zip_req *req;
447 	int head_size;
448 	int ret;
449 
450 	/* let's output compression head now */
451 	head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
452 	if (head_size < 0) {
453 		dev_err_ratelimited(dev, "failed to add comp head (%d)!\n",
454 				    head_size);
455 		return head_size;
456 	}
457 
458 	req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
459 	if (IS_ERR(req))
460 		return PTR_ERR(req);
461 
462 	ret = hisi_zip_do_work(req, qp_ctx);
463 	if (ret != -EINPROGRESS) {
464 		dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
465 		hisi_zip_remove_req(qp_ctx, req);
466 	}
467 
468 	return ret;
469 }
470 
471 static int hisi_zip_adecompress(struct acomp_req *acomp_req)
472 {
473 	struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
474 	struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
475 	struct device *dev = &qp_ctx->qp->qm->pdev->dev;
476 	struct hisi_zip_req *req;
477 	int head_size, ret;
478 
479 	head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type);
480 	if (head_size < 0) {
481 		dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n",
482 				    head_size);
483 		return head_size;
484 	}
485 
486 	req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
487 	if (IS_ERR(req))
488 		return PTR_ERR(req);
489 
490 	ret = hisi_zip_do_work(req, qp_ctx);
491 	if (ret != -EINPROGRESS) {
492 		dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
493 				     ret);
494 		hisi_zip_remove_req(qp_ctx, req);
495 	}
496 
497 	return ret;
498 }
499 
500 static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx,
501 			     int alg_type, int req_type)
502 {
503 	struct device *dev = &qp->qm->pdev->dev;
504 	int ret;
505 
506 	qp->req_type = req_type;
507 	qp->alg_type = alg_type;
508 	qp->qp_ctx = ctx;
509 
510 	ret = hisi_qm_start_qp(qp, 0);
511 	if (ret < 0) {
512 		dev_err(dev, "failed to start qp (%d)!\n", ret);
513 		return ret;
514 	}
515 
516 	ctx->qp = qp;
517 
518 	return 0;
519 }
520 
521 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
522 {
523 	hisi_qm_stop_qp(ctx->qp);
524 	hisi_qm_free_qps(&ctx->qp, 1);
525 }
526 
527 static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = {
528 	.sqe_type		= 0,
529 	.fill_addr		= hisi_zip_fill_addr,
530 	.fill_buf_size		= hisi_zip_fill_buf_size,
531 	.fill_buf_type		= hisi_zip_fill_buf_type,
532 	.fill_req_type		= hisi_zip_fill_req_type,
533 	.fill_tag		= hisi_zip_fill_tag_v1,
534 	.fill_sqe_type		= hisi_zip_fill_sqe_type,
535 	.get_tag		= hisi_zip_get_tag_v1,
536 	.get_status		= hisi_zip_get_status,
537 	.get_dstlen		= hisi_zip_get_dstlen,
538 };
539 
540 static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = {
541 	.sqe_type		= 0x3,
542 	.fill_addr		= hisi_zip_fill_addr,
543 	.fill_buf_size		= hisi_zip_fill_buf_size,
544 	.fill_buf_type		= hisi_zip_fill_buf_type,
545 	.fill_req_type		= hisi_zip_fill_req_type,
546 	.fill_tag		= hisi_zip_fill_tag_v2,
547 	.fill_sqe_type		= hisi_zip_fill_sqe_type,
548 	.get_tag		= hisi_zip_get_tag_v2,
549 	.get_status		= hisi_zip_get_status,
550 	.get_dstlen		= hisi_zip_get_dstlen,
551 };
552 
553 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node)
554 {
555 	struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL };
556 	struct hisi_zip_qp_ctx *qp_ctx;
557 	struct hisi_zip *hisi_zip;
558 	int ret, i, j;
559 
560 	ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node);
561 	if (ret) {
562 		pr_err("failed to create zip qps (%d)!\n", ret);
563 		return -ENODEV;
564 	}
565 
566 	hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm);
567 
568 	for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
569 		/* alg_type = 0 for compress, 1 for decompress in hw sqe */
570 		qp_ctx = &hisi_zip_ctx->qp_ctx[i];
571 		qp_ctx->ctx = hisi_zip_ctx;
572 		ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type);
573 		if (ret) {
574 			for (j = i - 1; j >= 0; j--)
575 				hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp);
576 
577 			hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM);
578 			return ret;
579 		}
580 
581 		qp_ctx->zip_dev = hisi_zip;
582 	}
583 
584 	if (hisi_zip->qm.ver < QM_HW_V3)
585 		hisi_zip_ctx->ops = &hisi_zip_ops_v1;
586 	else
587 		hisi_zip_ctx->ops = &hisi_zip_ops_v2;
588 
589 	return 0;
590 }
591 
592 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
593 {
594 	int i;
595 
596 	for (i = 1; i >= 0; i--)
597 		hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
598 }
599 
600 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
601 {
602 	struct hisi_zip_req_q *req_q;
603 	int i, ret;
604 
605 	for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
606 		req_q = &ctx->qp_ctx[i].req_q;
607 		req_q->size = QM_Q_DEPTH;
608 
609 		req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size),
610 					    sizeof(long), GFP_KERNEL);
611 		if (!req_q->req_bitmap) {
612 			ret = -ENOMEM;
613 			if (i == 0)
614 				return ret;
615 
616 			goto err_free_loop0;
617 		}
618 		rwlock_init(&req_q->req_lock);
619 
620 		req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req),
621 				   GFP_KERNEL);
622 		if (!req_q->q) {
623 			ret = -ENOMEM;
624 			if (i == 0)
625 				goto err_free_bitmap;
626 			else
627 				goto err_free_loop1;
628 		}
629 	}
630 
631 	return 0;
632 
633 err_free_loop1:
634 	kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap);
635 err_free_loop0:
636 	kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q);
637 err_free_bitmap:
638 	kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
639 	return ret;
640 }
641 
642 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
643 {
644 	int i;
645 
646 	for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
647 		kfree(ctx->qp_ctx[i].req_q.q);
648 		kfree(ctx->qp_ctx[i].req_q.req_bitmap);
649 	}
650 }
651 
652 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
653 {
654 	struct hisi_zip_qp_ctx *tmp;
655 	struct device *dev;
656 	int i;
657 
658 	for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
659 		tmp = &ctx->qp_ctx[i];
660 		dev = &tmp->qp->qm->pdev->dev;
661 		tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1,
662 							 sgl_sge_nr);
663 		if (IS_ERR(tmp->sgl_pool)) {
664 			if (i == 1)
665 				goto err_free_sgl_pool0;
666 			return -ENOMEM;
667 		}
668 	}
669 
670 	return 0;
671 
672 err_free_sgl_pool0:
673 	hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev,
674 			       ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool);
675 	return -ENOMEM;
676 }
677 
678 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
679 {
680 	int i;
681 
682 	for (i = 0; i < HZIP_CTX_Q_NUM; i++)
683 		hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev,
684 				       ctx->qp_ctx[i].sgl_pool);
685 }
686 
687 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
688 				  void (*fn)(struct hisi_qp *, void *))
689 {
690 	int i;
691 
692 	for (i = 0; i < HZIP_CTX_Q_NUM; i++)
693 		ctx->qp_ctx[i].qp->req_cb = fn;
694 }
695 
696 static int hisi_zip_acomp_init(struct crypto_acomp *tfm)
697 {
698 	const char *alg_name = crypto_tfm_alg_name(&tfm->base);
699 	struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
700 	struct device *dev;
701 	int ret;
702 
703 	ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node);
704 	if (ret) {
705 		pr_err("failed to init ctx (%d)!\n", ret);
706 		return ret;
707 	}
708 
709 	dev = &ctx->qp_ctx[0].qp->qm->pdev->dev;
710 
711 	ret = hisi_zip_create_req_q(ctx);
712 	if (ret) {
713 		dev_err(dev, "failed to create request queue (%d)!\n", ret);
714 		goto err_ctx_exit;
715 	}
716 
717 	ret = hisi_zip_create_sgl_pool(ctx);
718 	if (ret) {
719 		dev_err(dev, "failed to create sgl pool (%d)!\n", ret);
720 		goto err_release_req_q;
721 	}
722 
723 	hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
724 
725 	return 0;
726 
727 err_release_req_q:
728 	hisi_zip_release_req_q(ctx);
729 err_ctx_exit:
730 	hisi_zip_ctx_exit(ctx);
731 	return ret;
732 }
733 
734 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
735 {
736 	struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
737 
738 	hisi_zip_set_acomp_cb(ctx, NULL);
739 	hisi_zip_release_sgl_pool(ctx);
740 	hisi_zip_release_req_q(ctx);
741 	hisi_zip_ctx_exit(ctx);
742 }
743 
744 static struct acomp_alg hisi_zip_acomp_zlib = {
745 	.init			= hisi_zip_acomp_init,
746 	.exit			= hisi_zip_acomp_exit,
747 	.compress		= hisi_zip_acompress,
748 	.decompress		= hisi_zip_adecompress,
749 	.base			= {
750 		.cra_name		= "zlib-deflate",
751 		.cra_driver_name	= "hisi-zlib-acomp",
752 		.cra_module		= THIS_MODULE,
753 		.cra_priority           = HZIP_ALG_PRIORITY,
754 		.cra_ctxsize		= sizeof(struct hisi_zip_ctx),
755 	}
756 };
757 
758 static struct acomp_alg hisi_zip_acomp_gzip = {
759 	.init			= hisi_zip_acomp_init,
760 	.exit			= hisi_zip_acomp_exit,
761 	.compress		= hisi_zip_acompress,
762 	.decompress		= hisi_zip_adecompress,
763 	.base			= {
764 		.cra_name		= "gzip",
765 		.cra_driver_name	= "hisi-gzip-acomp",
766 		.cra_module		= THIS_MODULE,
767 		.cra_priority           = HZIP_ALG_PRIORITY,
768 		.cra_ctxsize		= sizeof(struct hisi_zip_ctx),
769 	}
770 };
771 
772 int hisi_zip_register_to_crypto(struct hisi_qm *qm)
773 {
774 	int ret;
775 
776 	ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
777 	if (ret) {
778 		pr_err("failed to register to zlib (%d)!\n", ret);
779 		return ret;
780 	}
781 
782 	ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
783 	if (ret) {
784 		pr_err("failed to register to gzip (%d)!\n", ret);
785 		crypto_unregister_acomp(&hisi_zip_acomp_zlib);
786 	}
787 
788 	return ret;
789 }
790 
791 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm)
792 {
793 	crypto_unregister_acomp(&hisi_zip_acomp_gzip);
794 	crypto_unregister_acomp(&hisi_zip_acomp_zlib);
795 }
796