1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <crypto/internal/acompress.h> 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/scatterlist.h> 8 #include "zip.h" 9 10 /* hisi_zip_sqe dw3 */ 11 #define HZIP_BD_STATUS_M GENMASK(7, 0) 12 /* hisi_zip_sqe dw7 */ 13 #define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0) 14 #define HZIP_SQE_TYPE_M GENMASK(31, 28) 15 /* hisi_zip_sqe dw8 */ 16 #define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0) 17 /* hisi_zip_sqe dw9 */ 18 #define HZIP_REQ_TYPE_M GENMASK(7, 0) 19 #define HZIP_ALG_TYPE_ZLIB 0x02 20 #define HZIP_ALG_TYPE_GZIP 0x03 21 #define HZIP_BUF_TYPE_M GENMASK(11, 8) 22 #define HZIP_PBUFFER 0x0 23 #define HZIP_SGL 0x1 24 25 #define HZIP_ZLIB_HEAD_SIZE 2 26 #define HZIP_GZIP_HEAD_SIZE 10 27 28 #define GZIP_HEAD_FHCRC_BIT BIT(1) 29 #define GZIP_HEAD_FEXTRA_BIT BIT(2) 30 #define GZIP_HEAD_FNAME_BIT BIT(3) 31 #define GZIP_HEAD_FCOMMENT_BIT BIT(4) 32 33 #define GZIP_HEAD_FLG_SHIFT 3 34 #define GZIP_HEAD_FEXTRA_SHIFT 10 35 #define GZIP_HEAD_FEXTRA_XLEN 2UL 36 #define GZIP_HEAD_FHCRC_SIZE 2 37 38 #define HZIP_GZIP_HEAD_BUF 256 39 #define HZIP_ALG_PRIORITY 300 40 #define HZIP_SGL_SGE_NR 10 41 42 #define HZIP_ALG_ZLIB GENMASK(1, 0) 43 #define HZIP_ALG_GZIP GENMASK(3, 2) 44 45 static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; 46 static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 47 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 48 }; 49 50 enum hisi_zip_alg_type { 51 HZIP_ALG_TYPE_COMP = 0, 52 HZIP_ALG_TYPE_DECOMP = 1, 53 }; 54 55 enum { 56 HZIP_QPC_COMP, 57 HZIP_QPC_DECOMP, 58 HZIP_CTX_Q_NUM 59 }; 60 61 #define COMP_NAME_TO_TYPE(alg_name) \ 62 (!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \ 63 !strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \ 64 65 #define TO_HEAD_SIZE(req_type) \ 66 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \ 67 ((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \ 68 69 #define TO_HEAD(req_type) \ 70 (((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \ 71 ((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : NULL) \ 72 73 struct hisi_zip_req { 74 struct acomp_req *req; 75 u32 sskip; 76 u32 dskip; 77 struct hisi_acc_hw_sgl *hw_src; 78 struct hisi_acc_hw_sgl *hw_dst; 79 dma_addr_t dma_src; 80 dma_addr_t dma_dst; 81 u16 req_id; 82 }; 83 84 struct hisi_zip_req_q { 85 struct hisi_zip_req *q; 86 unsigned long *req_bitmap; 87 rwlock_t req_lock; 88 u16 size; 89 }; 90 91 struct hisi_zip_qp_ctx { 92 struct hisi_qp *qp; 93 struct hisi_zip_req_q req_q; 94 struct hisi_acc_sgl_pool *sgl_pool; 95 struct hisi_zip *zip_dev; 96 struct hisi_zip_ctx *ctx; 97 }; 98 99 struct hisi_zip_sqe_ops { 100 u8 sqe_type; 101 void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 102 void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 103 void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); 104 void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); 105 void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 106 void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); 107 u32 (*get_tag)(struct hisi_zip_sqe *sqe); 108 u32 (*get_status)(struct hisi_zip_sqe *sqe); 109 u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); 110 }; 111 112 struct hisi_zip_ctx { 113 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 114 const struct hisi_zip_sqe_ops *ops; 115 }; 116 117 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) 118 { 119 int ret; 120 u16 n; 121 122 if (!val) 123 return -EINVAL; 124 125 ret = kstrtou16(val, 10, &n); 126 if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) 127 return -EINVAL; 128 129 return param_set_ushort(val, kp); 130 } 131 132 static const struct kernel_param_ops sgl_sge_nr_ops = { 133 .set = sgl_sge_nr_set, 134 .get = param_get_ushort, 135 }; 136 137 static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; 138 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 139 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 140 141 static u32 get_extra_field_size(const u8 *start) 142 { 143 return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; 144 } 145 146 static u32 get_name_field_size(const u8 *start) 147 { 148 return strlen(start) + 1; 149 } 150 151 static u32 get_comment_field_size(const u8 *start) 152 { 153 return strlen(start) + 1; 154 } 155 156 static u32 __get_gzip_head_size(const u8 *src) 157 { 158 u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT); 159 u32 size = GZIP_HEAD_FEXTRA_SHIFT; 160 161 if (head_flg & GZIP_HEAD_FEXTRA_BIT) 162 size += get_extra_field_size(src + size); 163 if (head_flg & GZIP_HEAD_FNAME_BIT) 164 size += get_name_field_size(src + size); 165 if (head_flg & GZIP_HEAD_FCOMMENT_BIT) 166 size += get_comment_field_size(src + size); 167 if (head_flg & GZIP_HEAD_FHCRC_BIT) 168 size += GZIP_HEAD_FHCRC_SIZE; 169 170 return size; 171 } 172 173 static u32 __maybe_unused get_gzip_head_size(struct scatterlist *sgl) 174 { 175 char buf[HZIP_GZIP_HEAD_BUF]; 176 177 sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf)); 178 179 return __get_gzip_head_size(buf); 180 } 181 182 static int add_comp_head(struct scatterlist *dst, u8 req_type) 183 { 184 int head_size = TO_HEAD_SIZE(req_type); 185 const u8 *head = TO_HEAD(req_type); 186 int ret; 187 188 ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); 189 if (unlikely(ret != head_size)) { 190 pr_err("the head size of buffer is wrong (%d)!\n", ret); 191 return -ENOMEM; 192 } 193 194 return head_size; 195 } 196 197 static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) 198 { 199 if (unlikely(!acomp_req->src || !acomp_req->slen)) 200 return -EINVAL; 201 202 if (unlikely(req_type == HZIP_ALG_TYPE_GZIP && 203 acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) 204 return -EINVAL; 205 206 switch (req_type) { 207 case HZIP_ALG_TYPE_ZLIB: 208 return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB); 209 case HZIP_ALG_TYPE_GZIP: 210 return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP); 211 default: 212 pr_err("request type does not support!\n"); 213 return -EINVAL; 214 } 215 } 216 217 static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, 218 struct hisi_zip_qp_ctx *qp_ctx, 219 size_t head_size, bool is_comp) 220 { 221 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 222 struct hisi_zip_req *q = req_q->q; 223 struct hisi_zip_req *req_cache; 224 int req_id; 225 226 write_lock(&req_q->req_lock); 227 228 req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size); 229 if (req_id >= req_q->size) { 230 write_unlock(&req_q->req_lock); 231 dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n"); 232 return ERR_PTR(-EAGAIN); 233 } 234 set_bit(req_id, req_q->req_bitmap); 235 236 write_unlock(&req_q->req_lock); 237 238 req_cache = q + req_id; 239 req_cache->req_id = req_id; 240 req_cache->req = req; 241 242 if (is_comp) { 243 req_cache->sskip = 0; 244 req_cache->dskip = head_size; 245 } else { 246 req_cache->sskip = head_size; 247 req_cache->dskip = 0; 248 } 249 250 return req_cache; 251 } 252 253 static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, 254 struct hisi_zip_req *req) 255 { 256 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 257 258 write_lock(&req_q->req_lock); 259 clear_bit(req->req_id, req_q->req_bitmap); 260 write_unlock(&req_q->req_lock); 261 } 262 263 static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 264 { 265 sqe->source_addr_l = lower_32_bits(req->dma_src); 266 sqe->source_addr_h = upper_32_bits(req->dma_src); 267 sqe->dest_addr_l = lower_32_bits(req->dma_dst); 268 sqe->dest_addr_h = upper_32_bits(req->dma_dst); 269 } 270 271 static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 272 { 273 struct acomp_req *a_req = req->req; 274 275 sqe->input_data_length = a_req->slen - req->sskip; 276 sqe->dest_avail_out = a_req->dlen - req->dskip; 277 sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip); 278 sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip); 279 } 280 281 static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type) 282 { 283 u32 val; 284 285 val = sqe->dw9 & ~HZIP_BUF_TYPE_M; 286 val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type); 287 sqe->dw9 = val; 288 } 289 290 static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type) 291 { 292 u32 val; 293 294 val = sqe->dw9 & ~HZIP_REQ_TYPE_M; 295 val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type); 296 sqe->dw9 = val; 297 } 298 299 static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 300 { 301 sqe->dw13 = req->req_id; 302 } 303 304 static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 305 { 306 sqe->dw26 = req->req_id; 307 } 308 309 static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) 310 { 311 u32 val; 312 313 val = sqe->dw7 & ~HZIP_SQE_TYPE_M; 314 val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type); 315 sqe->dw7 = val; 316 } 317 318 static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe, 319 u8 req_type, struct hisi_zip_req *req) 320 { 321 const struct hisi_zip_sqe_ops *ops = ctx->ops; 322 323 memset(sqe, 0, sizeof(struct hisi_zip_sqe)); 324 325 ops->fill_addr(sqe, req); 326 ops->fill_buf_size(sqe, req); 327 ops->fill_buf_type(sqe, HZIP_SGL); 328 ops->fill_req_type(sqe, req_type); 329 ops->fill_tag(sqe, req); 330 ops->fill_sqe_type(sqe, ops->sqe_type); 331 } 332 333 static int hisi_zip_do_work(struct hisi_zip_req *req, 334 struct hisi_zip_qp_ctx *qp_ctx) 335 { 336 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 337 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 338 struct acomp_req *a_req = req->req; 339 struct hisi_qp *qp = qp_ctx->qp; 340 struct device *dev = &qp->qm->pdev->dev; 341 struct hisi_zip_sqe zip_sqe; 342 int ret; 343 344 if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) 345 return -EINVAL; 346 347 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 348 req->req_id << 1, &req->dma_src); 349 if (IS_ERR(req->hw_src)) { 350 dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", 351 PTR_ERR(req->hw_src)); 352 return PTR_ERR(req->hw_src); 353 } 354 355 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 356 (req->req_id << 1) + 1, 357 &req->dma_dst); 358 if (IS_ERR(req->hw_dst)) { 359 ret = PTR_ERR(req->hw_dst); 360 dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", 361 ret); 362 goto err_unmap_input; 363 } 364 365 hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); 366 367 /* send command to start a task */ 368 atomic64_inc(&dfx->send_cnt); 369 ret = hisi_qp_send(qp, &zip_sqe); 370 if (unlikely(ret < 0)) { 371 atomic64_inc(&dfx->send_busy_cnt); 372 ret = -EAGAIN; 373 dev_dbg_ratelimited(dev, "failed to send request!\n"); 374 goto err_unmap_output; 375 } 376 377 return -EINPROGRESS; 378 379 err_unmap_output: 380 hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 381 err_unmap_input: 382 hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 383 return ret; 384 } 385 386 static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe) 387 { 388 return sqe->dw13; 389 } 390 391 static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe) 392 { 393 return sqe->dw26; 394 } 395 396 static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) 397 { 398 return sqe->dw3 & HZIP_BD_STATUS_M; 399 } 400 401 static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe) 402 { 403 return sqe->produced; 404 } 405 406 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 407 { 408 struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 409 const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; 410 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 411 struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 412 struct device *dev = &qp->qm->pdev->dev; 413 struct hisi_zip_sqe *sqe = data; 414 u32 tag = ops->get_tag(sqe); 415 struct hisi_zip_req *req = req_q->q + tag; 416 struct acomp_req *acomp_req = req->req; 417 u32 status, dlen, head_size; 418 int err = 0; 419 420 atomic64_inc(&dfx->recv_cnt); 421 status = ops->get_status(sqe); 422 if (unlikely(status != 0 && status != HZIP_NC_ERR)) { 423 dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", 424 (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, 425 sqe->produced); 426 atomic64_inc(&dfx->err_bd_cnt); 427 err = -EIO; 428 } 429 430 dlen = ops->get_dstlen(sqe); 431 432 hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 433 hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 434 435 head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0; 436 acomp_req->dlen = dlen + head_size; 437 438 if (acomp_req->base.complete) 439 acomp_request_complete(acomp_req, err); 440 441 hisi_zip_remove_req(qp_ctx, req); 442 } 443 444 static int hisi_zip_acompress(struct acomp_req *acomp_req) 445 { 446 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 447 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; 448 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 449 struct hisi_zip_req *req; 450 int head_size; 451 int ret; 452 453 /* let's output compression head now */ 454 head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); 455 if (unlikely(head_size < 0)) { 456 dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", 457 head_size); 458 return head_size; 459 } 460 461 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true); 462 if (IS_ERR(req)) 463 return PTR_ERR(req); 464 465 ret = hisi_zip_do_work(req, qp_ctx); 466 if (unlikely(ret != -EINPROGRESS)) { 467 dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); 468 hisi_zip_remove_req(qp_ctx, req); 469 } 470 471 return ret; 472 } 473 474 static int hisi_zip_adecompress(struct acomp_req *acomp_req) 475 { 476 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 477 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; 478 struct device *dev = &qp_ctx->qp->qm->pdev->dev; 479 struct hisi_zip_req *req; 480 int head_size, ret; 481 482 head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); 483 if (unlikely(head_size < 0)) { 484 dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", 485 head_size); 486 return head_size; 487 } 488 489 req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false); 490 if (IS_ERR(req)) 491 return PTR_ERR(req); 492 493 ret = hisi_zip_do_work(req, qp_ctx); 494 if (unlikely(ret != -EINPROGRESS)) { 495 dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", 496 ret); 497 hisi_zip_remove_req(qp_ctx, req); 498 } 499 500 return ret; 501 } 502 503 static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, 504 int alg_type, int req_type) 505 { 506 struct device *dev = &qp->qm->pdev->dev; 507 int ret; 508 509 qp->req_type = req_type; 510 qp->alg_type = alg_type; 511 qp->qp_ctx = qp_ctx; 512 513 ret = hisi_qm_start_qp(qp, 0); 514 if (ret < 0) { 515 dev_err(dev, "failed to start qp (%d)!\n", ret); 516 return ret; 517 } 518 519 qp_ctx->qp = qp; 520 521 return 0; 522 } 523 524 static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) 525 { 526 hisi_qm_stop_qp(qp_ctx->qp); 527 hisi_qm_free_qps(&qp_ctx->qp, 1); 528 } 529 530 static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { 531 .sqe_type = 0, 532 .fill_addr = hisi_zip_fill_addr, 533 .fill_buf_size = hisi_zip_fill_buf_size, 534 .fill_buf_type = hisi_zip_fill_buf_type, 535 .fill_req_type = hisi_zip_fill_req_type, 536 .fill_tag = hisi_zip_fill_tag_v1, 537 .fill_sqe_type = hisi_zip_fill_sqe_type, 538 .get_tag = hisi_zip_get_tag_v1, 539 .get_status = hisi_zip_get_status, 540 .get_dstlen = hisi_zip_get_dstlen, 541 }; 542 543 static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = { 544 .sqe_type = 0x3, 545 .fill_addr = hisi_zip_fill_addr, 546 .fill_buf_size = hisi_zip_fill_buf_size, 547 .fill_buf_type = hisi_zip_fill_buf_type, 548 .fill_req_type = hisi_zip_fill_req_type, 549 .fill_tag = hisi_zip_fill_tag_v2, 550 .fill_sqe_type = hisi_zip_fill_sqe_type, 551 .get_tag = hisi_zip_get_tag_v2, 552 .get_status = hisi_zip_get_status, 553 .get_dstlen = hisi_zip_get_dstlen, 554 }; 555 556 static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node) 557 { 558 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 559 struct hisi_zip_qp_ctx *qp_ctx; 560 struct hisi_zip *hisi_zip; 561 int ret, i, j; 562 563 ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); 564 if (ret) { 565 pr_err("failed to create zip qps (%d)!\n", ret); 566 return -ENODEV; 567 } 568 569 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 570 571 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 572 /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 573 qp_ctx = &hisi_zip_ctx->qp_ctx[i]; 574 qp_ctx->ctx = hisi_zip_ctx; 575 ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); 576 if (ret) { 577 for (j = i - 1; j >= 0; j--) 578 hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 579 580 hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 581 return ret; 582 } 583 584 qp_ctx->zip_dev = hisi_zip; 585 } 586 587 if (hisi_zip->qm.ver < QM_HW_V3) 588 hisi_zip_ctx->ops = &hisi_zip_ops_v1; 589 else 590 hisi_zip_ctx->ops = &hisi_zip_ops_v2; 591 592 return 0; 593 } 594 595 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 596 { 597 int i; 598 599 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 600 hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 601 } 602 603 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) 604 { 605 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 606 struct hisi_zip_req_q *req_q; 607 int i, ret; 608 609 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 610 req_q = &ctx->qp_ctx[i].req_q; 611 req_q->size = q_depth; 612 613 req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); 614 if (!req_q->req_bitmap) { 615 ret = -ENOMEM; 616 if (i == 0) 617 return ret; 618 619 goto err_free_comp_q; 620 } 621 rwlock_init(&req_q->req_lock); 622 623 req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req), 624 GFP_KERNEL); 625 if (!req_q->q) { 626 ret = -ENOMEM; 627 if (i == 0) 628 goto err_free_comp_bitmap; 629 else 630 goto err_free_decomp_bitmap; 631 } 632 } 633 634 return 0; 635 636 err_free_decomp_bitmap: 637 bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); 638 err_free_comp_q: 639 kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); 640 err_free_comp_bitmap: 641 bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); 642 return ret; 643 } 644 645 static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) 646 { 647 int i; 648 649 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 650 kfree(ctx->qp_ctx[i].req_q.q); 651 bitmap_free(ctx->qp_ctx[i].req_q.req_bitmap); 652 } 653 } 654 655 static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) 656 { 657 u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; 658 struct hisi_zip_qp_ctx *tmp; 659 struct device *dev; 660 int i; 661 662 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 663 tmp = &ctx->qp_ctx[i]; 664 dev = &tmp->qp->qm->pdev->dev; 665 tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, 666 sgl_sge_nr); 667 if (IS_ERR(tmp->sgl_pool)) { 668 if (i == 1) 669 goto err_free_sgl_pool0; 670 return -ENOMEM; 671 } 672 } 673 674 return 0; 675 676 err_free_sgl_pool0: 677 hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev, 678 ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool); 679 return -ENOMEM; 680 } 681 682 static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx) 683 { 684 int i; 685 686 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 687 hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev, 688 ctx->qp_ctx[i].sgl_pool); 689 } 690 691 static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx, 692 void (*fn)(struct hisi_qp *, void *)) 693 { 694 int i; 695 696 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 697 ctx->qp_ctx[i].qp->req_cb = fn; 698 } 699 700 static int hisi_zip_acomp_init(struct crypto_acomp *tfm) 701 { 702 const char *alg_name = crypto_tfm_alg_name(&tfm->base); 703 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 704 struct device *dev; 705 int ret; 706 707 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); 708 if (ret) { 709 pr_err("failed to init ctx (%d)!\n", ret); 710 return ret; 711 } 712 713 dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; 714 715 ret = hisi_zip_create_req_q(ctx); 716 if (ret) { 717 dev_err(dev, "failed to create request queue (%d)!\n", ret); 718 goto err_ctx_exit; 719 } 720 721 ret = hisi_zip_create_sgl_pool(ctx); 722 if (ret) { 723 dev_err(dev, "failed to create sgl pool (%d)!\n", ret); 724 goto err_release_req_q; 725 } 726 727 hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb); 728 729 return 0; 730 731 err_release_req_q: 732 hisi_zip_release_req_q(ctx); 733 err_ctx_exit: 734 hisi_zip_ctx_exit(ctx); 735 return ret; 736 } 737 738 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 739 { 740 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 741 742 hisi_zip_set_acomp_cb(ctx, NULL); 743 hisi_zip_release_sgl_pool(ctx); 744 hisi_zip_release_req_q(ctx); 745 hisi_zip_ctx_exit(ctx); 746 } 747 748 static struct acomp_alg hisi_zip_acomp_zlib = { 749 .init = hisi_zip_acomp_init, 750 .exit = hisi_zip_acomp_exit, 751 .compress = hisi_zip_acompress, 752 .decompress = hisi_zip_adecompress, 753 .base = { 754 .cra_name = "zlib-deflate", 755 .cra_driver_name = "hisi-zlib-acomp", 756 .cra_module = THIS_MODULE, 757 .cra_priority = HZIP_ALG_PRIORITY, 758 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 759 } 760 }; 761 762 static int hisi_zip_register_zlib(struct hisi_qm *qm) 763 { 764 int ret; 765 766 if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) 767 return 0; 768 769 ret = crypto_register_acomp(&hisi_zip_acomp_zlib); 770 if (ret) 771 dev_err(&qm->pdev->dev, "failed to register to zlib (%d)!\n", ret); 772 773 return ret; 774 } 775 776 static void hisi_zip_unregister_zlib(struct hisi_qm *qm) 777 { 778 if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) 779 return; 780 781 crypto_unregister_acomp(&hisi_zip_acomp_zlib); 782 } 783 784 static struct acomp_alg hisi_zip_acomp_gzip = { 785 .init = hisi_zip_acomp_init, 786 .exit = hisi_zip_acomp_exit, 787 .compress = hisi_zip_acompress, 788 .decompress = hisi_zip_adecompress, 789 .base = { 790 .cra_name = "gzip", 791 .cra_driver_name = "hisi-gzip-acomp", 792 .cra_module = THIS_MODULE, 793 .cra_priority = HZIP_ALG_PRIORITY, 794 .cra_ctxsize = sizeof(struct hisi_zip_ctx), 795 } 796 }; 797 798 static int hisi_zip_register_gzip(struct hisi_qm *qm) 799 { 800 int ret; 801 802 if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) 803 return 0; 804 805 ret = crypto_register_acomp(&hisi_zip_acomp_gzip); 806 if (ret) 807 dev_err(&qm->pdev->dev, "failed to register to gzip (%d)!\n", ret); 808 809 return ret; 810 } 811 812 static void hisi_zip_unregister_gzip(struct hisi_qm *qm) 813 { 814 if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) 815 return; 816 817 crypto_unregister_acomp(&hisi_zip_acomp_gzip); 818 } 819 820 int hisi_zip_register_to_crypto(struct hisi_qm *qm) 821 { 822 int ret = 0; 823 824 ret = hisi_zip_register_zlib(qm); 825 if (ret) 826 return ret; 827 828 ret = hisi_zip_register_gzip(qm); 829 if (ret) 830 hisi_zip_unregister_zlib(qm); 831 832 return ret; 833 } 834 835 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) 836 { 837 hisi_zip_unregister_zlib(qm); 838 hisi_zip_unregister_gzip(qm); 839 } 840