1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2018 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include <linux/fsl/mc.h> 19 #include <soc/fsl/dpaa2-io.h> 20 #include <soc/fsl/dpaa2-fd.h> 21 22 #define CAAM_CRA_PRIORITY 2000 23 24 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 25 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 26 SHA512_DIGEST_SIZE * 2) 27 28 /* 29 * This is a a cache of buffers, from which the users of CAAM QI driver 30 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 31 * NOTE: A more elegant solution would be to have some headroom in the frames 32 * being processed. This can be added by the dpaa2-eth driver. This would 33 * pose a problem for userspace application processing which cannot 34 * know of this limitation. So for now, this will work. 35 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 36 */ 37 static struct kmem_cache *qi_cache; 38 39 struct caam_alg_entry { 40 struct device *dev; 41 int class1_alg_type; 42 int class2_alg_type; 43 bool rfc3686; 44 bool geniv; 45 bool nodkp; 46 }; 47 48 struct caam_aead_alg { 49 struct aead_alg aead; 50 struct caam_alg_entry caam; 51 bool registered; 52 }; 53 54 struct caam_skcipher_alg { 55 struct skcipher_alg skcipher; 56 struct caam_alg_entry caam; 57 bool registered; 58 }; 59 60 /** 61 * caam_ctx - per-session context 62 * @flc: Flow Contexts array 63 * @key: [authentication key], encryption key 64 * @flc_dma: I/O virtual addresses of the Flow Contexts 65 * @key_dma: I/O virtual address of the key 66 * @dir: DMA direction for mapping key and Flow Contexts 67 * @dev: dpseci device 68 * @adata: authentication algorithm details 69 * @cdata: encryption algorithm details 70 * @authsize: authentication tag (a.k.a. ICV / MAC) size 71 */ 72 struct caam_ctx { 73 struct caam_flc flc[NUM_OP]; 74 u8 key[CAAM_MAX_KEY_SIZE]; 75 dma_addr_t flc_dma[NUM_OP]; 76 dma_addr_t key_dma; 77 enum dma_data_direction dir; 78 struct device *dev; 79 struct alginfo adata; 80 struct alginfo cdata; 81 unsigned int authsize; 82 }; 83 84 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 85 dma_addr_t iova_addr) 86 { 87 phys_addr_t phys_addr; 88 89 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 90 iova_addr; 91 92 return phys_to_virt(phys_addr); 93 } 94 95 /* 96 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 97 * 98 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 99 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 100 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 101 * hosting 16 SG entries. 102 * 103 * @flags - flags that would be used for the equivalent kmalloc(..) call 104 * 105 * Returns a pointer to a retrieved buffer on success or NULL on failure. 106 */ 107 static inline void *qi_cache_zalloc(gfp_t flags) 108 { 109 return kmem_cache_zalloc(qi_cache, flags); 110 } 111 112 /* 113 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 114 * 115 * @obj - buffer previously allocated by qi_cache_zalloc 116 * 117 * No checking is being done, the call is a passthrough call to 118 * kmem_cache_free(...) 119 */ 120 static inline void qi_cache_free(void *obj) 121 { 122 kmem_cache_free(qi_cache, obj); 123 } 124 125 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 126 { 127 switch (crypto_tfm_alg_type(areq->tfm)) { 128 case CRYPTO_ALG_TYPE_SKCIPHER: 129 return skcipher_request_ctx(skcipher_request_cast(areq)); 130 case CRYPTO_ALG_TYPE_AEAD: 131 return aead_request_ctx(container_of(areq, struct aead_request, 132 base)); 133 case CRYPTO_ALG_TYPE_AHASH: 134 return ahash_request_ctx(ahash_request_cast(areq)); 135 default: 136 return ERR_PTR(-EINVAL); 137 } 138 } 139 140 static void caam_unmap(struct device *dev, struct scatterlist *src, 141 struct scatterlist *dst, int src_nents, 142 int dst_nents, dma_addr_t iv_dma, int ivsize, 143 dma_addr_t qm_sg_dma, int qm_sg_bytes) 144 { 145 if (dst != src) { 146 if (src_nents) 147 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 148 if (dst_nents) 149 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 150 } else { 151 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 152 } 153 154 if (iv_dma) 155 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); 156 157 if (qm_sg_bytes) 158 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 159 } 160 161 static int aead_set_sh_desc(struct crypto_aead *aead) 162 { 163 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 164 typeof(*alg), aead); 165 struct caam_ctx *ctx = crypto_aead_ctx(aead); 166 unsigned int ivsize = crypto_aead_ivsize(aead); 167 struct device *dev = ctx->dev; 168 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 169 struct caam_flc *flc; 170 u32 *desc; 171 u32 ctx1_iv_off = 0; 172 u32 *nonce = NULL; 173 unsigned int data_len[2]; 174 u32 inl_mask; 175 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 176 OP_ALG_AAI_CTR_MOD128); 177 const bool is_rfc3686 = alg->caam.rfc3686; 178 179 if (!ctx->cdata.keylen || !ctx->authsize) 180 return 0; 181 182 /* 183 * AES-CTR needs to load IV in CONTEXT1 reg 184 * at an offset of 128bits (16bytes) 185 * CONTEXT1[255:128] = IV 186 */ 187 if (ctr_mode) 188 ctx1_iv_off = 16; 189 190 /* 191 * RFC3686 specific: 192 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 193 */ 194 if (is_rfc3686) { 195 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 196 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 197 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 198 } 199 200 data_len[0] = ctx->adata.keylen_pad; 201 data_len[1] = ctx->cdata.keylen; 202 203 /* aead_encrypt shared descriptor */ 204 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 205 DESC_QI_AEAD_ENC_LEN) + 206 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 207 DESC_JOB_IO_LEN, data_len, &inl_mask, 208 ARRAY_SIZE(data_len)) < 0) 209 return -EINVAL; 210 211 if (inl_mask & 1) 212 ctx->adata.key_virt = ctx->key; 213 else 214 ctx->adata.key_dma = ctx->key_dma; 215 216 if (inl_mask & 2) 217 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 218 else 219 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 220 221 ctx->adata.key_inline = !!(inl_mask & 1); 222 ctx->cdata.key_inline = !!(inl_mask & 2); 223 224 flc = &ctx->flc[ENCRYPT]; 225 desc = flc->sh_desc; 226 227 if (alg->caam.geniv) 228 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 229 ivsize, ctx->authsize, is_rfc3686, 230 nonce, ctx1_iv_off, true, 231 priv->sec_attr.era); 232 else 233 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 234 ivsize, ctx->authsize, is_rfc3686, nonce, 235 ctx1_iv_off, true, priv->sec_attr.era); 236 237 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 238 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 239 sizeof(flc->flc) + desc_bytes(desc), 240 ctx->dir); 241 242 /* aead_decrypt shared descriptor */ 243 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 244 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 245 DESC_JOB_IO_LEN, data_len, &inl_mask, 246 ARRAY_SIZE(data_len)) < 0) 247 return -EINVAL; 248 249 if (inl_mask & 1) 250 ctx->adata.key_virt = ctx->key; 251 else 252 ctx->adata.key_dma = ctx->key_dma; 253 254 if (inl_mask & 2) 255 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 256 else 257 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 258 259 ctx->adata.key_inline = !!(inl_mask & 1); 260 ctx->cdata.key_inline = !!(inl_mask & 2); 261 262 flc = &ctx->flc[DECRYPT]; 263 desc = flc->sh_desc; 264 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 265 ivsize, ctx->authsize, alg->caam.geniv, 266 is_rfc3686, nonce, ctx1_iv_off, true, 267 priv->sec_attr.era); 268 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 269 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 270 sizeof(flc->flc) + desc_bytes(desc), 271 ctx->dir); 272 273 return 0; 274 } 275 276 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 277 { 278 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 279 280 ctx->authsize = authsize; 281 aead_set_sh_desc(authenc); 282 283 return 0; 284 } 285 286 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 287 unsigned int keylen) 288 { 289 struct caam_ctx *ctx = crypto_aead_ctx(aead); 290 struct device *dev = ctx->dev; 291 struct crypto_authenc_keys keys; 292 293 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 294 goto badkey; 295 296 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 297 keys.authkeylen + keys.enckeylen, keys.enckeylen, 298 keys.authkeylen); 299 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 300 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 301 302 ctx->adata.keylen = keys.authkeylen; 303 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 304 OP_ALG_ALGSEL_MASK); 305 306 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 307 goto badkey; 308 309 memcpy(ctx->key, keys.authkey, keys.authkeylen); 310 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 311 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 312 keys.enckeylen, ctx->dir); 313 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 314 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 315 ctx->adata.keylen_pad + keys.enckeylen, 1); 316 317 ctx->cdata.keylen = keys.enckeylen; 318 319 memzero_explicit(&keys, sizeof(keys)); 320 return aead_set_sh_desc(aead); 321 badkey: 322 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 323 memzero_explicit(&keys, sizeof(keys)); 324 return -EINVAL; 325 } 326 327 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 328 unsigned int keylen) 329 { 330 struct crypto_authenc_keys keys; 331 u32 flags; 332 int err; 333 334 err = crypto_authenc_extractkeys(&keys, key, keylen); 335 if (unlikely(err)) 336 goto badkey; 337 338 err = -EINVAL; 339 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 340 goto badkey; 341 342 flags = crypto_aead_get_flags(aead); 343 err = __des3_verify_key(&flags, keys.enckey); 344 if (unlikely(err)) { 345 crypto_aead_set_flags(aead, flags); 346 goto out; 347 } 348 349 err = aead_setkey(aead, key, keylen); 350 351 out: 352 memzero_explicit(&keys, sizeof(keys)); 353 return err; 354 355 badkey: 356 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 357 goto out; 358 } 359 360 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 361 bool encrypt) 362 { 363 struct crypto_aead *aead = crypto_aead_reqtfm(req); 364 struct caam_request *req_ctx = aead_request_ctx(req); 365 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 366 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 367 struct caam_ctx *ctx = crypto_aead_ctx(aead); 368 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 369 typeof(*alg), aead); 370 struct device *dev = ctx->dev; 371 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 372 GFP_KERNEL : GFP_ATOMIC; 373 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 374 struct aead_edesc *edesc; 375 dma_addr_t qm_sg_dma, iv_dma = 0; 376 int ivsize = 0; 377 unsigned int authsize = ctx->authsize; 378 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 379 int in_len, out_len; 380 struct dpaa2_sg_entry *sg_table; 381 382 /* allocate space for base edesc, link tables and IV */ 383 edesc = qi_cache_zalloc(GFP_DMA | flags); 384 if (unlikely(!edesc)) { 385 dev_err(dev, "could not allocate extended descriptor\n"); 386 return ERR_PTR(-ENOMEM); 387 } 388 389 if (unlikely(req->dst != req->src)) { 390 src_nents = sg_nents_for_len(req->src, req->assoclen + 391 req->cryptlen); 392 if (unlikely(src_nents < 0)) { 393 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 394 req->assoclen + req->cryptlen); 395 qi_cache_free(edesc); 396 return ERR_PTR(src_nents); 397 } 398 399 dst_nents = sg_nents_for_len(req->dst, req->assoclen + 400 req->cryptlen + 401 (encrypt ? authsize : 402 (-authsize))); 403 if (unlikely(dst_nents < 0)) { 404 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 405 req->assoclen + req->cryptlen + 406 (encrypt ? authsize : (-authsize))); 407 qi_cache_free(edesc); 408 return ERR_PTR(dst_nents); 409 } 410 411 if (src_nents) { 412 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 413 DMA_TO_DEVICE); 414 if (unlikely(!mapped_src_nents)) { 415 dev_err(dev, "unable to map source\n"); 416 qi_cache_free(edesc); 417 return ERR_PTR(-ENOMEM); 418 } 419 } else { 420 mapped_src_nents = 0; 421 } 422 423 if (dst_nents) { 424 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 425 DMA_FROM_DEVICE); 426 if (unlikely(!mapped_dst_nents)) { 427 dev_err(dev, "unable to map destination\n"); 428 dma_unmap_sg(dev, req->src, src_nents, 429 DMA_TO_DEVICE); 430 qi_cache_free(edesc); 431 return ERR_PTR(-ENOMEM); 432 } 433 } else { 434 mapped_dst_nents = 0; 435 } 436 } else { 437 src_nents = sg_nents_for_len(req->src, req->assoclen + 438 req->cryptlen + 439 (encrypt ? authsize : 0)); 440 if (unlikely(src_nents < 0)) { 441 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 442 req->assoclen + req->cryptlen + 443 (encrypt ? authsize : 0)); 444 qi_cache_free(edesc); 445 return ERR_PTR(src_nents); 446 } 447 448 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 449 DMA_BIDIRECTIONAL); 450 if (unlikely(!mapped_src_nents)) { 451 dev_err(dev, "unable to map source\n"); 452 qi_cache_free(edesc); 453 return ERR_PTR(-ENOMEM); 454 } 455 } 456 457 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 458 ivsize = crypto_aead_ivsize(aead); 459 460 /* 461 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 462 * Input is not contiguous. 463 */ 464 qm_sg_nents = 1 + !!ivsize + mapped_src_nents + 465 (mapped_dst_nents > 1 ? mapped_dst_nents : 0); 466 sg_table = &edesc->sgt[0]; 467 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 468 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 469 CAAM_QI_MEMCACHE_SIZE)) { 470 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 471 qm_sg_nents, ivsize); 472 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 473 0, 0, 0); 474 qi_cache_free(edesc); 475 return ERR_PTR(-ENOMEM); 476 } 477 478 if (ivsize) { 479 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 480 481 /* Make sure IV is located in a DMAable area */ 482 memcpy(iv, req->iv, ivsize); 483 484 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 485 if (dma_mapping_error(dev, iv_dma)) { 486 dev_err(dev, "unable to map IV\n"); 487 caam_unmap(dev, req->src, req->dst, src_nents, 488 dst_nents, 0, 0, 0, 0); 489 qi_cache_free(edesc); 490 return ERR_PTR(-ENOMEM); 491 } 492 } 493 494 edesc->src_nents = src_nents; 495 edesc->dst_nents = dst_nents; 496 edesc->iv_dma = iv_dma; 497 498 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 499 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 500 /* 501 * The associated data comes already with the IV but we need 502 * to skip it when we authenticate or encrypt... 503 */ 504 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 505 else 506 edesc->assoclen = cpu_to_caam32(req->assoclen); 507 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 508 DMA_TO_DEVICE); 509 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 510 dev_err(dev, "unable to map assoclen\n"); 511 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 512 iv_dma, ivsize, 0, 0); 513 qi_cache_free(edesc); 514 return ERR_PTR(-ENOMEM); 515 } 516 517 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 518 qm_sg_index++; 519 if (ivsize) { 520 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 521 qm_sg_index++; 522 } 523 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); 524 qm_sg_index += mapped_src_nents; 525 526 if (mapped_dst_nents > 1) 527 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 528 qm_sg_index, 0); 529 530 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 531 if (dma_mapping_error(dev, qm_sg_dma)) { 532 dev_err(dev, "unable to map S/G table\n"); 533 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 534 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 535 iv_dma, ivsize, 0, 0); 536 qi_cache_free(edesc); 537 return ERR_PTR(-ENOMEM); 538 } 539 540 edesc->qm_sg_dma = qm_sg_dma; 541 edesc->qm_sg_bytes = qm_sg_bytes; 542 543 out_len = req->assoclen + req->cryptlen + 544 (encrypt ? ctx->authsize : (-ctx->authsize)); 545 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 546 547 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 548 dpaa2_fl_set_final(in_fle, true); 549 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 550 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 551 dpaa2_fl_set_len(in_fle, in_len); 552 553 if (req->dst == req->src) { 554 if (mapped_src_nents == 1) { 555 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 556 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 557 } else { 558 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 559 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 560 (1 + !!ivsize) * sizeof(*sg_table)); 561 } 562 } else if (mapped_dst_nents == 1) { 563 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 564 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 565 } else { 566 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 567 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 568 sizeof(*sg_table)); 569 } 570 571 dpaa2_fl_set_len(out_fle, out_len); 572 573 return edesc; 574 } 575 576 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 577 { 578 struct caam_ctx *ctx = crypto_aead_ctx(aead); 579 unsigned int ivsize = crypto_aead_ivsize(aead); 580 struct device *dev = ctx->dev; 581 struct caam_flc *flc; 582 u32 *desc; 583 584 if (!ctx->cdata.keylen || !ctx->authsize) 585 return 0; 586 587 flc = &ctx->flc[ENCRYPT]; 588 desc = flc->sh_desc; 589 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 590 ctx->authsize, true, true); 591 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 592 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 593 sizeof(flc->flc) + desc_bytes(desc), 594 ctx->dir); 595 596 flc = &ctx->flc[DECRYPT]; 597 desc = flc->sh_desc; 598 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 599 ctx->authsize, false, true); 600 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 601 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 602 sizeof(flc->flc) + desc_bytes(desc), 603 ctx->dir); 604 605 return 0; 606 } 607 608 static int chachapoly_setauthsize(struct crypto_aead *aead, 609 unsigned int authsize) 610 { 611 struct caam_ctx *ctx = crypto_aead_ctx(aead); 612 613 if (authsize != POLY1305_DIGEST_SIZE) 614 return -EINVAL; 615 616 ctx->authsize = authsize; 617 return chachapoly_set_sh_desc(aead); 618 } 619 620 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 621 unsigned int keylen) 622 { 623 struct caam_ctx *ctx = crypto_aead_ctx(aead); 624 unsigned int ivsize = crypto_aead_ivsize(aead); 625 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 626 627 if (keylen != CHACHA_KEY_SIZE + saltlen) { 628 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); 629 return -EINVAL; 630 } 631 632 ctx->cdata.key_virt = key; 633 ctx->cdata.keylen = keylen - saltlen; 634 635 return chachapoly_set_sh_desc(aead); 636 } 637 638 static int gcm_set_sh_desc(struct crypto_aead *aead) 639 { 640 struct caam_ctx *ctx = crypto_aead_ctx(aead); 641 struct device *dev = ctx->dev; 642 unsigned int ivsize = crypto_aead_ivsize(aead); 643 struct caam_flc *flc; 644 u32 *desc; 645 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 646 ctx->cdata.keylen; 647 648 if (!ctx->cdata.keylen || !ctx->authsize) 649 return 0; 650 651 /* 652 * AES GCM encrypt shared descriptor 653 * Job Descriptor and Shared Descriptor 654 * must fit into the 64-word Descriptor h/w Buffer 655 */ 656 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 657 ctx->cdata.key_inline = true; 658 ctx->cdata.key_virt = ctx->key; 659 } else { 660 ctx->cdata.key_inline = false; 661 ctx->cdata.key_dma = ctx->key_dma; 662 } 663 664 flc = &ctx->flc[ENCRYPT]; 665 desc = flc->sh_desc; 666 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 667 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 668 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 669 sizeof(flc->flc) + desc_bytes(desc), 670 ctx->dir); 671 672 /* 673 * Job Descriptor and Shared Descriptors 674 * must all fit into the 64-word Descriptor h/w Buffer 675 */ 676 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 677 ctx->cdata.key_inline = true; 678 ctx->cdata.key_virt = ctx->key; 679 } else { 680 ctx->cdata.key_inline = false; 681 ctx->cdata.key_dma = ctx->key_dma; 682 } 683 684 flc = &ctx->flc[DECRYPT]; 685 desc = flc->sh_desc; 686 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 687 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 688 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 689 sizeof(flc->flc) + desc_bytes(desc), 690 ctx->dir); 691 692 return 0; 693 } 694 695 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 696 { 697 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 698 699 ctx->authsize = authsize; 700 gcm_set_sh_desc(authenc); 701 702 return 0; 703 } 704 705 static int gcm_setkey(struct crypto_aead *aead, 706 const u8 *key, unsigned int keylen) 707 { 708 struct caam_ctx *ctx = crypto_aead_ctx(aead); 709 struct device *dev = ctx->dev; 710 711 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 712 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 713 714 memcpy(ctx->key, key, keylen); 715 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 716 ctx->cdata.keylen = keylen; 717 718 return gcm_set_sh_desc(aead); 719 } 720 721 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 722 { 723 struct caam_ctx *ctx = crypto_aead_ctx(aead); 724 struct device *dev = ctx->dev; 725 unsigned int ivsize = crypto_aead_ivsize(aead); 726 struct caam_flc *flc; 727 u32 *desc; 728 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 729 ctx->cdata.keylen; 730 731 if (!ctx->cdata.keylen || !ctx->authsize) 732 return 0; 733 734 ctx->cdata.key_virt = ctx->key; 735 736 /* 737 * RFC4106 encrypt shared descriptor 738 * Job Descriptor and Shared Descriptor 739 * must fit into the 64-word Descriptor h/w Buffer 740 */ 741 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 742 ctx->cdata.key_inline = true; 743 } else { 744 ctx->cdata.key_inline = false; 745 ctx->cdata.key_dma = ctx->key_dma; 746 } 747 748 flc = &ctx->flc[ENCRYPT]; 749 desc = flc->sh_desc; 750 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 751 true); 752 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 753 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 754 sizeof(flc->flc) + desc_bytes(desc), 755 ctx->dir); 756 757 /* 758 * Job Descriptor and Shared Descriptors 759 * must all fit into the 64-word Descriptor h/w Buffer 760 */ 761 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 762 ctx->cdata.key_inline = true; 763 } else { 764 ctx->cdata.key_inline = false; 765 ctx->cdata.key_dma = ctx->key_dma; 766 } 767 768 flc = &ctx->flc[DECRYPT]; 769 desc = flc->sh_desc; 770 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 771 true); 772 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 773 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 774 sizeof(flc->flc) + desc_bytes(desc), 775 ctx->dir); 776 777 return 0; 778 } 779 780 static int rfc4106_setauthsize(struct crypto_aead *authenc, 781 unsigned int authsize) 782 { 783 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 784 785 ctx->authsize = authsize; 786 rfc4106_set_sh_desc(authenc); 787 788 return 0; 789 } 790 791 static int rfc4106_setkey(struct crypto_aead *aead, 792 const u8 *key, unsigned int keylen) 793 { 794 struct caam_ctx *ctx = crypto_aead_ctx(aead); 795 struct device *dev = ctx->dev; 796 797 if (keylen < 4) 798 return -EINVAL; 799 800 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 801 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 802 803 memcpy(ctx->key, key, keylen); 804 /* 805 * The last four bytes of the key material are used as the salt value 806 * in the nonce. Update the AES key length. 807 */ 808 ctx->cdata.keylen = keylen - 4; 809 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 810 ctx->dir); 811 812 return rfc4106_set_sh_desc(aead); 813 } 814 815 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 816 { 817 struct caam_ctx *ctx = crypto_aead_ctx(aead); 818 struct device *dev = ctx->dev; 819 unsigned int ivsize = crypto_aead_ivsize(aead); 820 struct caam_flc *flc; 821 u32 *desc; 822 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 823 ctx->cdata.keylen; 824 825 if (!ctx->cdata.keylen || !ctx->authsize) 826 return 0; 827 828 ctx->cdata.key_virt = ctx->key; 829 830 /* 831 * RFC4543 encrypt shared descriptor 832 * Job Descriptor and Shared Descriptor 833 * must fit into the 64-word Descriptor h/w Buffer 834 */ 835 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 836 ctx->cdata.key_inline = true; 837 } else { 838 ctx->cdata.key_inline = false; 839 ctx->cdata.key_dma = ctx->key_dma; 840 } 841 842 flc = &ctx->flc[ENCRYPT]; 843 desc = flc->sh_desc; 844 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 845 true); 846 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 847 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 848 sizeof(flc->flc) + desc_bytes(desc), 849 ctx->dir); 850 851 /* 852 * Job Descriptor and Shared Descriptors 853 * must all fit into the 64-word Descriptor h/w Buffer 854 */ 855 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 856 ctx->cdata.key_inline = true; 857 } else { 858 ctx->cdata.key_inline = false; 859 ctx->cdata.key_dma = ctx->key_dma; 860 } 861 862 flc = &ctx->flc[DECRYPT]; 863 desc = flc->sh_desc; 864 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 865 true); 866 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 867 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 868 sizeof(flc->flc) + desc_bytes(desc), 869 ctx->dir); 870 871 return 0; 872 } 873 874 static int rfc4543_setauthsize(struct crypto_aead *authenc, 875 unsigned int authsize) 876 { 877 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 878 879 ctx->authsize = authsize; 880 rfc4543_set_sh_desc(authenc); 881 882 return 0; 883 } 884 885 static int rfc4543_setkey(struct crypto_aead *aead, 886 const u8 *key, unsigned int keylen) 887 { 888 struct caam_ctx *ctx = crypto_aead_ctx(aead); 889 struct device *dev = ctx->dev; 890 891 if (keylen < 4) 892 return -EINVAL; 893 894 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 895 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 896 897 memcpy(ctx->key, key, keylen); 898 /* 899 * The last four bytes of the key material are used as the salt value 900 * in the nonce. Update the AES key length. 901 */ 902 ctx->cdata.keylen = keylen - 4; 903 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 904 ctx->dir); 905 906 return rfc4543_set_sh_desc(aead); 907 } 908 909 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 910 unsigned int keylen) 911 { 912 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 913 struct caam_skcipher_alg *alg = 914 container_of(crypto_skcipher_alg(skcipher), 915 struct caam_skcipher_alg, skcipher); 916 struct device *dev = ctx->dev; 917 struct caam_flc *flc; 918 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 919 u32 *desc; 920 u32 ctx1_iv_off = 0; 921 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 922 OP_ALG_AAI_CTR_MOD128) && 923 ((ctx->cdata.algtype & OP_ALG_ALGSEL_MASK) != 924 OP_ALG_ALGSEL_CHACHA20); 925 const bool is_rfc3686 = alg->caam.rfc3686; 926 927 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 928 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 929 930 /* 931 * AES-CTR needs to load IV in CONTEXT1 reg 932 * at an offset of 128bits (16bytes) 933 * CONTEXT1[255:128] = IV 934 */ 935 if (ctr_mode) 936 ctx1_iv_off = 16; 937 938 /* 939 * RFC3686 specific: 940 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 941 * | *key = {KEY, NONCE} 942 */ 943 if (is_rfc3686) { 944 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 945 keylen -= CTR_RFC3686_NONCE_SIZE; 946 } 947 948 ctx->cdata.keylen = keylen; 949 ctx->cdata.key_virt = key; 950 ctx->cdata.key_inline = true; 951 952 /* skcipher_encrypt shared descriptor */ 953 flc = &ctx->flc[ENCRYPT]; 954 desc = flc->sh_desc; 955 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 956 ctx1_iv_off); 957 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 958 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 959 sizeof(flc->flc) + desc_bytes(desc), 960 ctx->dir); 961 962 /* skcipher_decrypt shared descriptor */ 963 flc = &ctx->flc[DECRYPT]; 964 desc = flc->sh_desc; 965 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 966 ctx1_iv_off); 967 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 968 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 969 sizeof(flc->flc) + desc_bytes(desc), 970 ctx->dir); 971 972 return 0; 973 } 974 975 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 976 const u8 *key, unsigned int keylen) 977 { 978 return unlikely(des3_verify_key(skcipher, key)) ?: 979 skcipher_setkey(skcipher, key, keylen); 980 } 981 982 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 983 unsigned int keylen) 984 { 985 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 986 struct device *dev = ctx->dev; 987 struct caam_flc *flc; 988 u32 *desc; 989 990 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 991 dev_err(dev, "key size mismatch\n"); 992 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 993 return -EINVAL; 994 } 995 996 ctx->cdata.keylen = keylen; 997 ctx->cdata.key_virt = key; 998 ctx->cdata.key_inline = true; 999 1000 /* xts_skcipher_encrypt shared descriptor */ 1001 flc = &ctx->flc[ENCRYPT]; 1002 desc = flc->sh_desc; 1003 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1004 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1005 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1006 sizeof(flc->flc) + desc_bytes(desc), 1007 ctx->dir); 1008 1009 /* xts_skcipher_decrypt shared descriptor */ 1010 flc = &ctx->flc[DECRYPT]; 1011 desc = flc->sh_desc; 1012 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1013 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1014 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1015 sizeof(flc->flc) + desc_bytes(desc), 1016 ctx->dir); 1017 1018 return 0; 1019 } 1020 1021 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1022 { 1023 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1024 struct caam_request *req_ctx = skcipher_request_ctx(req); 1025 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1026 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1027 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1028 struct device *dev = ctx->dev; 1029 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1030 GFP_KERNEL : GFP_ATOMIC; 1031 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1032 struct skcipher_edesc *edesc; 1033 dma_addr_t iv_dma; 1034 u8 *iv; 1035 int ivsize = crypto_skcipher_ivsize(skcipher); 1036 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1037 struct dpaa2_sg_entry *sg_table; 1038 1039 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1040 if (unlikely(src_nents < 0)) { 1041 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1042 req->cryptlen); 1043 return ERR_PTR(src_nents); 1044 } 1045 1046 if (unlikely(req->dst != req->src)) { 1047 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1048 if (unlikely(dst_nents < 0)) { 1049 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1050 req->cryptlen); 1051 return ERR_PTR(dst_nents); 1052 } 1053 1054 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1055 DMA_TO_DEVICE); 1056 if (unlikely(!mapped_src_nents)) { 1057 dev_err(dev, "unable to map source\n"); 1058 return ERR_PTR(-ENOMEM); 1059 } 1060 1061 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1062 DMA_FROM_DEVICE); 1063 if (unlikely(!mapped_dst_nents)) { 1064 dev_err(dev, "unable to map destination\n"); 1065 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1066 return ERR_PTR(-ENOMEM); 1067 } 1068 } else { 1069 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1070 DMA_BIDIRECTIONAL); 1071 if (unlikely(!mapped_src_nents)) { 1072 dev_err(dev, "unable to map source\n"); 1073 return ERR_PTR(-ENOMEM); 1074 } 1075 } 1076 1077 qm_sg_ents = 1 + mapped_src_nents; 1078 dst_sg_idx = qm_sg_ents; 1079 1080 qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; 1081 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1082 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1083 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1084 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1085 qm_sg_ents, ivsize); 1086 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1087 0, 0, 0); 1088 return ERR_PTR(-ENOMEM); 1089 } 1090 1091 /* allocate space for base edesc, link tables and IV */ 1092 edesc = qi_cache_zalloc(GFP_DMA | flags); 1093 if (unlikely(!edesc)) { 1094 dev_err(dev, "could not allocate extended descriptor\n"); 1095 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1096 0, 0, 0); 1097 return ERR_PTR(-ENOMEM); 1098 } 1099 1100 /* Make sure IV is located in a DMAable area */ 1101 sg_table = &edesc->sgt[0]; 1102 iv = (u8 *)(sg_table + qm_sg_ents); 1103 memcpy(iv, req->iv, ivsize); 1104 1105 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 1106 if (dma_mapping_error(dev, iv_dma)) { 1107 dev_err(dev, "unable to map IV\n"); 1108 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1109 0, 0, 0); 1110 qi_cache_free(edesc); 1111 return ERR_PTR(-ENOMEM); 1112 } 1113 1114 edesc->src_nents = src_nents; 1115 edesc->dst_nents = dst_nents; 1116 edesc->iv_dma = iv_dma; 1117 edesc->qm_sg_bytes = qm_sg_bytes; 1118 1119 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1120 sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); 1121 1122 if (mapped_dst_nents > 1) 1123 sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + 1124 dst_sg_idx, 0); 1125 1126 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1127 DMA_TO_DEVICE); 1128 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1129 dev_err(dev, "unable to map S/G table\n"); 1130 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1131 iv_dma, ivsize, 0, 0); 1132 qi_cache_free(edesc); 1133 return ERR_PTR(-ENOMEM); 1134 } 1135 1136 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1137 dpaa2_fl_set_final(in_fle, true); 1138 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1139 dpaa2_fl_set_len(out_fle, req->cryptlen); 1140 1141 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1142 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1143 1144 if (req->src == req->dst) { 1145 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1146 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1147 sizeof(*sg_table)); 1148 } else if (mapped_dst_nents > 1) { 1149 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1150 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1151 sizeof(*sg_table)); 1152 } else { 1153 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 1154 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 1155 } 1156 1157 return edesc; 1158 } 1159 1160 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1161 struct aead_request *req) 1162 { 1163 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1164 int ivsize = crypto_aead_ivsize(aead); 1165 1166 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1167 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 1168 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1169 } 1170 1171 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1172 struct skcipher_request *req) 1173 { 1174 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1175 int ivsize = crypto_skcipher_ivsize(skcipher); 1176 1177 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1178 edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); 1179 } 1180 1181 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1182 { 1183 struct crypto_async_request *areq = cbk_ctx; 1184 struct aead_request *req = container_of(areq, struct aead_request, 1185 base); 1186 struct caam_request *req_ctx = to_caam_req(areq); 1187 struct aead_edesc *edesc = req_ctx->edesc; 1188 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1189 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1190 int ecode = 0; 1191 1192 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1193 1194 if (unlikely(status)) { 1195 caam_qi2_strstatus(ctx->dev, status); 1196 ecode = -EIO; 1197 } 1198 1199 aead_unmap(ctx->dev, edesc, req); 1200 qi_cache_free(edesc); 1201 aead_request_complete(req, ecode); 1202 } 1203 1204 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1205 { 1206 struct crypto_async_request *areq = cbk_ctx; 1207 struct aead_request *req = container_of(areq, struct aead_request, 1208 base); 1209 struct caam_request *req_ctx = to_caam_req(areq); 1210 struct aead_edesc *edesc = req_ctx->edesc; 1211 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1212 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1213 int ecode = 0; 1214 1215 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1216 1217 if (unlikely(status)) { 1218 caam_qi2_strstatus(ctx->dev, status); 1219 /* 1220 * verify hw auth check passed else return -EBADMSG 1221 */ 1222 if ((status & JRSTA_CCBERR_ERRID_MASK) == 1223 JRSTA_CCBERR_ERRID_ICVCHK) 1224 ecode = -EBADMSG; 1225 else 1226 ecode = -EIO; 1227 } 1228 1229 aead_unmap(ctx->dev, edesc, req); 1230 qi_cache_free(edesc); 1231 aead_request_complete(req, ecode); 1232 } 1233 1234 static int aead_encrypt(struct aead_request *req) 1235 { 1236 struct aead_edesc *edesc; 1237 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1238 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1239 struct caam_request *caam_req = aead_request_ctx(req); 1240 int ret; 1241 1242 /* allocate extended descriptor */ 1243 edesc = aead_edesc_alloc(req, true); 1244 if (IS_ERR(edesc)) 1245 return PTR_ERR(edesc); 1246 1247 caam_req->flc = &ctx->flc[ENCRYPT]; 1248 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1249 caam_req->cbk = aead_encrypt_done; 1250 caam_req->ctx = &req->base; 1251 caam_req->edesc = edesc; 1252 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1253 if (ret != -EINPROGRESS && 1254 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1255 aead_unmap(ctx->dev, edesc, req); 1256 qi_cache_free(edesc); 1257 } 1258 1259 return ret; 1260 } 1261 1262 static int aead_decrypt(struct aead_request *req) 1263 { 1264 struct aead_edesc *edesc; 1265 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1266 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1267 struct caam_request *caam_req = aead_request_ctx(req); 1268 int ret; 1269 1270 /* allocate extended descriptor */ 1271 edesc = aead_edesc_alloc(req, false); 1272 if (IS_ERR(edesc)) 1273 return PTR_ERR(edesc); 1274 1275 caam_req->flc = &ctx->flc[DECRYPT]; 1276 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1277 caam_req->cbk = aead_decrypt_done; 1278 caam_req->ctx = &req->base; 1279 caam_req->edesc = edesc; 1280 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1281 if (ret != -EINPROGRESS && 1282 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1283 aead_unmap(ctx->dev, edesc, req); 1284 qi_cache_free(edesc); 1285 } 1286 1287 return ret; 1288 } 1289 1290 static int ipsec_gcm_encrypt(struct aead_request *req) 1291 { 1292 if (req->assoclen < 8) 1293 return -EINVAL; 1294 1295 return aead_encrypt(req); 1296 } 1297 1298 static int ipsec_gcm_decrypt(struct aead_request *req) 1299 { 1300 if (req->assoclen < 8) 1301 return -EINVAL; 1302 1303 return aead_decrypt(req); 1304 } 1305 1306 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1307 { 1308 struct crypto_async_request *areq = cbk_ctx; 1309 struct skcipher_request *req = skcipher_request_cast(areq); 1310 struct caam_request *req_ctx = to_caam_req(areq); 1311 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1312 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1313 struct skcipher_edesc *edesc = req_ctx->edesc; 1314 int ecode = 0; 1315 int ivsize = crypto_skcipher_ivsize(skcipher); 1316 1317 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1318 1319 if (unlikely(status)) { 1320 caam_qi2_strstatus(ctx->dev, status); 1321 ecode = -EIO; 1322 } 1323 1324 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1325 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1326 edesc->src_nents > 1 ? 100 : ivsize, 1); 1327 caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", 1328 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1329 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1330 1331 skcipher_unmap(ctx->dev, edesc, req); 1332 1333 /* 1334 * The crypto API expects us to set the IV (req->iv) to the last 1335 * ciphertext block. This is used e.g. by the CTS mode. 1336 */ 1337 scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, 1338 ivsize, 0); 1339 1340 qi_cache_free(edesc); 1341 skcipher_request_complete(req, ecode); 1342 } 1343 1344 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1345 { 1346 struct crypto_async_request *areq = cbk_ctx; 1347 struct skcipher_request *req = skcipher_request_cast(areq); 1348 struct caam_request *req_ctx = to_caam_req(areq); 1349 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1350 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1351 struct skcipher_edesc *edesc = req_ctx->edesc; 1352 int ecode = 0; 1353 int ivsize = crypto_skcipher_ivsize(skcipher); 1354 1355 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1356 1357 if (unlikely(status)) { 1358 caam_qi2_strstatus(ctx->dev, status); 1359 ecode = -EIO; 1360 } 1361 1362 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1363 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1364 edesc->src_nents > 1 ? 100 : ivsize, 1); 1365 caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", 1366 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1367 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1368 1369 skcipher_unmap(ctx->dev, edesc, req); 1370 qi_cache_free(edesc); 1371 skcipher_request_complete(req, ecode); 1372 } 1373 1374 static int skcipher_encrypt(struct skcipher_request *req) 1375 { 1376 struct skcipher_edesc *edesc; 1377 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1378 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1379 struct caam_request *caam_req = skcipher_request_ctx(req); 1380 int ret; 1381 1382 /* allocate extended descriptor */ 1383 edesc = skcipher_edesc_alloc(req); 1384 if (IS_ERR(edesc)) 1385 return PTR_ERR(edesc); 1386 1387 caam_req->flc = &ctx->flc[ENCRYPT]; 1388 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1389 caam_req->cbk = skcipher_encrypt_done; 1390 caam_req->ctx = &req->base; 1391 caam_req->edesc = edesc; 1392 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1393 if (ret != -EINPROGRESS && 1394 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1395 skcipher_unmap(ctx->dev, edesc, req); 1396 qi_cache_free(edesc); 1397 } 1398 1399 return ret; 1400 } 1401 1402 static int skcipher_decrypt(struct skcipher_request *req) 1403 { 1404 struct skcipher_edesc *edesc; 1405 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1406 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1407 struct caam_request *caam_req = skcipher_request_ctx(req); 1408 int ivsize = crypto_skcipher_ivsize(skcipher); 1409 int ret; 1410 1411 /* allocate extended descriptor */ 1412 edesc = skcipher_edesc_alloc(req); 1413 if (IS_ERR(edesc)) 1414 return PTR_ERR(edesc); 1415 1416 /* 1417 * The crypto API expects us to set the IV (req->iv) to the last 1418 * ciphertext block. 1419 */ 1420 scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, 1421 ivsize, 0); 1422 1423 caam_req->flc = &ctx->flc[DECRYPT]; 1424 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1425 caam_req->cbk = skcipher_decrypt_done; 1426 caam_req->ctx = &req->base; 1427 caam_req->edesc = edesc; 1428 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1429 if (ret != -EINPROGRESS && 1430 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1431 skcipher_unmap(ctx->dev, edesc, req); 1432 qi_cache_free(edesc); 1433 } 1434 1435 return ret; 1436 } 1437 1438 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1439 bool uses_dkp) 1440 { 1441 dma_addr_t dma_addr; 1442 int i; 1443 1444 /* copy descriptor header template value */ 1445 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1446 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1447 1448 ctx->dev = caam->dev; 1449 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1450 1451 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1452 offsetof(struct caam_ctx, flc_dma), 1453 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1454 if (dma_mapping_error(ctx->dev, dma_addr)) { 1455 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1456 return -ENOMEM; 1457 } 1458 1459 for (i = 0; i < NUM_OP; i++) 1460 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1461 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1462 1463 return 0; 1464 } 1465 1466 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1467 { 1468 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1469 struct caam_skcipher_alg *caam_alg = 1470 container_of(alg, typeof(*caam_alg), skcipher); 1471 1472 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1473 return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); 1474 } 1475 1476 static int caam_cra_init_aead(struct crypto_aead *tfm) 1477 { 1478 struct aead_alg *alg = crypto_aead_alg(tfm); 1479 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1480 aead); 1481 1482 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1483 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1484 !caam_alg->caam.nodkp); 1485 } 1486 1487 static void caam_exit_common(struct caam_ctx *ctx) 1488 { 1489 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1490 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1491 DMA_ATTR_SKIP_CPU_SYNC); 1492 } 1493 1494 static void caam_cra_exit(struct crypto_skcipher *tfm) 1495 { 1496 caam_exit_common(crypto_skcipher_ctx(tfm)); 1497 } 1498 1499 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1500 { 1501 caam_exit_common(crypto_aead_ctx(tfm)); 1502 } 1503 1504 static struct caam_skcipher_alg driver_algs[] = { 1505 { 1506 .skcipher = { 1507 .base = { 1508 .cra_name = "cbc(aes)", 1509 .cra_driver_name = "cbc-aes-caam-qi2", 1510 .cra_blocksize = AES_BLOCK_SIZE, 1511 }, 1512 .setkey = skcipher_setkey, 1513 .encrypt = skcipher_encrypt, 1514 .decrypt = skcipher_decrypt, 1515 .min_keysize = AES_MIN_KEY_SIZE, 1516 .max_keysize = AES_MAX_KEY_SIZE, 1517 .ivsize = AES_BLOCK_SIZE, 1518 }, 1519 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1520 }, 1521 { 1522 .skcipher = { 1523 .base = { 1524 .cra_name = "cbc(des3_ede)", 1525 .cra_driver_name = "cbc-3des-caam-qi2", 1526 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1527 }, 1528 .setkey = des3_skcipher_setkey, 1529 .encrypt = skcipher_encrypt, 1530 .decrypt = skcipher_decrypt, 1531 .min_keysize = DES3_EDE_KEY_SIZE, 1532 .max_keysize = DES3_EDE_KEY_SIZE, 1533 .ivsize = DES3_EDE_BLOCK_SIZE, 1534 }, 1535 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1536 }, 1537 { 1538 .skcipher = { 1539 .base = { 1540 .cra_name = "cbc(des)", 1541 .cra_driver_name = "cbc-des-caam-qi2", 1542 .cra_blocksize = DES_BLOCK_SIZE, 1543 }, 1544 .setkey = skcipher_setkey, 1545 .encrypt = skcipher_encrypt, 1546 .decrypt = skcipher_decrypt, 1547 .min_keysize = DES_KEY_SIZE, 1548 .max_keysize = DES_KEY_SIZE, 1549 .ivsize = DES_BLOCK_SIZE, 1550 }, 1551 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1552 }, 1553 { 1554 .skcipher = { 1555 .base = { 1556 .cra_name = "ctr(aes)", 1557 .cra_driver_name = "ctr-aes-caam-qi2", 1558 .cra_blocksize = 1, 1559 }, 1560 .setkey = skcipher_setkey, 1561 .encrypt = skcipher_encrypt, 1562 .decrypt = skcipher_decrypt, 1563 .min_keysize = AES_MIN_KEY_SIZE, 1564 .max_keysize = AES_MAX_KEY_SIZE, 1565 .ivsize = AES_BLOCK_SIZE, 1566 .chunksize = AES_BLOCK_SIZE, 1567 }, 1568 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1569 OP_ALG_AAI_CTR_MOD128, 1570 }, 1571 { 1572 .skcipher = { 1573 .base = { 1574 .cra_name = "rfc3686(ctr(aes))", 1575 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1576 .cra_blocksize = 1, 1577 }, 1578 .setkey = skcipher_setkey, 1579 .encrypt = skcipher_encrypt, 1580 .decrypt = skcipher_decrypt, 1581 .min_keysize = AES_MIN_KEY_SIZE + 1582 CTR_RFC3686_NONCE_SIZE, 1583 .max_keysize = AES_MAX_KEY_SIZE + 1584 CTR_RFC3686_NONCE_SIZE, 1585 .ivsize = CTR_RFC3686_IV_SIZE, 1586 .chunksize = AES_BLOCK_SIZE, 1587 }, 1588 .caam = { 1589 .class1_alg_type = OP_ALG_ALGSEL_AES | 1590 OP_ALG_AAI_CTR_MOD128, 1591 .rfc3686 = true, 1592 }, 1593 }, 1594 { 1595 .skcipher = { 1596 .base = { 1597 .cra_name = "xts(aes)", 1598 .cra_driver_name = "xts-aes-caam-qi2", 1599 .cra_blocksize = AES_BLOCK_SIZE, 1600 }, 1601 .setkey = xts_skcipher_setkey, 1602 .encrypt = skcipher_encrypt, 1603 .decrypt = skcipher_decrypt, 1604 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1605 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1606 .ivsize = AES_BLOCK_SIZE, 1607 }, 1608 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1609 }, 1610 { 1611 .skcipher = { 1612 .base = { 1613 .cra_name = "chacha20", 1614 .cra_driver_name = "chacha20-caam-qi2", 1615 .cra_blocksize = 1, 1616 }, 1617 .setkey = skcipher_setkey, 1618 .encrypt = skcipher_encrypt, 1619 .decrypt = skcipher_decrypt, 1620 .min_keysize = CHACHA_KEY_SIZE, 1621 .max_keysize = CHACHA_KEY_SIZE, 1622 .ivsize = CHACHA_IV_SIZE, 1623 }, 1624 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1625 }, 1626 }; 1627 1628 static struct caam_aead_alg driver_aeads[] = { 1629 { 1630 .aead = { 1631 .base = { 1632 .cra_name = "rfc4106(gcm(aes))", 1633 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1634 .cra_blocksize = 1, 1635 }, 1636 .setkey = rfc4106_setkey, 1637 .setauthsize = rfc4106_setauthsize, 1638 .encrypt = ipsec_gcm_encrypt, 1639 .decrypt = ipsec_gcm_decrypt, 1640 .ivsize = 8, 1641 .maxauthsize = AES_BLOCK_SIZE, 1642 }, 1643 .caam = { 1644 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1645 .nodkp = true, 1646 }, 1647 }, 1648 { 1649 .aead = { 1650 .base = { 1651 .cra_name = "rfc4543(gcm(aes))", 1652 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1653 .cra_blocksize = 1, 1654 }, 1655 .setkey = rfc4543_setkey, 1656 .setauthsize = rfc4543_setauthsize, 1657 .encrypt = ipsec_gcm_encrypt, 1658 .decrypt = ipsec_gcm_decrypt, 1659 .ivsize = 8, 1660 .maxauthsize = AES_BLOCK_SIZE, 1661 }, 1662 .caam = { 1663 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1664 .nodkp = true, 1665 }, 1666 }, 1667 /* Galois Counter Mode */ 1668 { 1669 .aead = { 1670 .base = { 1671 .cra_name = "gcm(aes)", 1672 .cra_driver_name = "gcm-aes-caam-qi2", 1673 .cra_blocksize = 1, 1674 }, 1675 .setkey = gcm_setkey, 1676 .setauthsize = gcm_setauthsize, 1677 .encrypt = aead_encrypt, 1678 .decrypt = aead_decrypt, 1679 .ivsize = 12, 1680 .maxauthsize = AES_BLOCK_SIZE, 1681 }, 1682 .caam = { 1683 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1684 .nodkp = true, 1685 } 1686 }, 1687 /* single-pass ipsec_esp descriptor */ 1688 { 1689 .aead = { 1690 .base = { 1691 .cra_name = "authenc(hmac(md5),cbc(aes))", 1692 .cra_driver_name = "authenc-hmac-md5-" 1693 "cbc-aes-caam-qi2", 1694 .cra_blocksize = AES_BLOCK_SIZE, 1695 }, 1696 .setkey = aead_setkey, 1697 .setauthsize = aead_setauthsize, 1698 .encrypt = aead_encrypt, 1699 .decrypt = aead_decrypt, 1700 .ivsize = AES_BLOCK_SIZE, 1701 .maxauthsize = MD5_DIGEST_SIZE, 1702 }, 1703 .caam = { 1704 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1705 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1706 OP_ALG_AAI_HMAC_PRECOMP, 1707 } 1708 }, 1709 { 1710 .aead = { 1711 .base = { 1712 .cra_name = "echainiv(authenc(hmac(md5)," 1713 "cbc(aes)))", 1714 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1715 "cbc-aes-caam-qi2", 1716 .cra_blocksize = AES_BLOCK_SIZE, 1717 }, 1718 .setkey = aead_setkey, 1719 .setauthsize = aead_setauthsize, 1720 .encrypt = aead_encrypt, 1721 .decrypt = aead_decrypt, 1722 .ivsize = AES_BLOCK_SIZE, 1723 .maxauthsize = MD5_DIGEST_SIZE, 1724 }, 1725 .caam = { 1726 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1727 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1728 OP_ALG_AAI_HMAC_PRECOMP, 1729 .geniv = true, 1730 } 1731 }, 1732 { 1733 .aead = { 1734 .base = { 1735 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1736 .cra_driver_name = "authenc-hmac-sha1-" 1737 "cbc-aes-caam-qi2", 1738 .cra_blocksize = AES_BLOCK_SIZE, 1739 }, 1740 .setkey = aead_setkey, 1741 .setauthsize = aead_setauthsize, 1742 .encrypt = aead_encrypt, 1743 .decrypt = aead_decrypt, 1744 .ivsize = AES_BLOCK_SIZE, 1745 .maxauthsize = SHA1_DIGEST_SIZE, 1746 }, 1747 .caam = { 1748 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1749 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1750 OP_ALG_AAI_HMAC_PRECOMP, 1751 } 1752 }, 1753 { 1754 .aead = { 1755 .base = { 1756 .cra_name = "echainiv(authenc(hmac(sha1)," 1757 "cbc(aes)))", 1758 .cra_driver_name = "echainiv-authenc-" 1759 "hmac-sha1-cbc-aes-caam-qi2", 1760 .cra_blocksize = AES_BLOCK_SIZE, 1761 }, 1762 .setkey = aead_setkey, 1763 .setauthsize = aead_setauthsize, 1764 .encrypt = aead_encrypt, 1765 .decrypt = aead_decrypt, 1766 .ivsize = AES_BLOCK_SIZE, 1767 .maxauthsize = SHA1_DIGEST_SIZE, 1768 }, 1769 .caam = { 1770 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1771 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1772 OP_ALG_AAI_HMAC_PRECOMP, 1773 .geniv = true, 1774 }, 1775 }, 1776 { 1777 .aead = { 1778 .base = { 1779 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1780 .cra_driver_name = "authenc-hmac-sha224-" 1781 "cbc-aes-caam-qi2", 1782 .cra_blocksize = AES_BLOCK_SIZE, 1783 }, 1784 .setkey = aead_setkey, 1785 .setauthsize = aead_setauthsize, 1786 .encrypt = aead_encrypt, 1787 .decrypt = aead_decrypt, 1788 .ivsize = AES_BLOCK_SIZE, 1789 .maxauthsize = SHA224_DIGEST_SIZE, 1790 }, 1791 .caam = { 1792 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1793 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1794 OP_ALG_AAI_HMAC_PRECOMP, 1795 } 1796 }, 1797 { 1798 .aead = { 1799 .base = { 1800 .cra_name = "echainiv(authenc(hmac(sha224)," 1801 "cbc(aes)))", 1802 .cra_driver_name = "echainiv-authenc-" 1803 "hmac-sha224-cbc-aes-caam-qi2", 1804 .cra_blocksize = AES_BLOCK_SIZE, 1805 }, 1806 .setkey = aead_setkey, 1807 .setauthsize = aead_setauthsize, 1808 .encrypt = aead_encrypt, 1809 .decrypt = aead_decrypt, 1810 .ivsize = AES_BLOCK_SIZE, 1811 .maxauthsize = SHA224_DIGEST_SIZE, 1812 }, 1813 .caam = { 1814 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1815 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1816 OP_ALG_AAI_HMAC_PRECOMP, 1817 .geniv = true, 1818 } 1819 }, 1820 { 1821 .aead = { 1822 .base = { 1823 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1824 .cra_driver_name = "authenc-hmac-sha256-" 1825 "cbc-aes-caam-qi2", 1826 .cra_blocksize = AES_BLOCK_SIZE, 1827 }, 1828 .setkey = aead_setkey, 1829 .setauthsize = aead_setauthsize, 1830 .encrypt = aead_encrypt, 1831 .decrypt = aead_decrypt, 1832 .ivsize = AES_BLOCK_SIZE, 1833 .maxauthsize = SHA256_DIGEST_SIZE, 1834 }, 1835 .caam = { 1836 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1837 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1838 OP_ALG_AAI_HMAC_PRECOMP, 1839 } 1840 }, 1841 { 1842 .aead = { 1843 .base = { 1844 .cra_name = "echainiv(authenc(hmac(sha256)," 1845 "cbc(aes)))", 1846 .cra_driver_name = "echainiv-authenc-" 1847 "hmac-sha256-cbc-aes-" 1848 "caam-qi2", 1849 .cra_blocksize = AES_BLOCK_SIZE, 1850 }, 1851 .setkey = aead_setkey, 1852 .setauthsize = aead_setauthsize, 1853 .encrypt = aead_encrypt, 1854 .decrypt = aead_decrypt, 1855 .ivsize = AES_BLOCK_SIZE, 1856 .maxauthsize = SHA256_DIGEST_SIZE, 1857 }, 1858 .caam = { 1859 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1860 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1861 OP_ALG_AAI_HMAC_PRECOMP, 1862 .geniv = true, 1863 } 1864 }, 1865 { 1866 .aead = { 1867 .base = { 1868 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1869 .cra_driver_name = "authenc-hmac-sha384-" 1870 "cbc-aes-caam-qi2", 1871 .cra_blocksize = AES_BLOCK_SIZE, 1872 }, 1873 .setkey = aead_setkey, 1874 .setauthsize = aead_setauthsize, 1875 .encrypt = aead_encrypt, 1876 .decrypt = aead_decrypt, 1877 .ivsize = AES_BLOCK_SIZE, 1878 .maxauthsize = SHA384_DIGEST_SIZE, 1879 }, 1880 .caam = { 1881 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1882 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1883 OP_ALG_AAI_HMAC_PRECOMP, 1884 } 1885 }, 1886 { 1887 .aead = { 1888 .base = { 1889 .cra_name = "echainiv(authenc(hmac(sha384)," 1890 "cbc(aes)))", 1891 .cra_driver_name = "echainiv-authenc-" 1892 "hmac-sha384-cbc-aes-" 1893 "caam-qi2", 1894 .cra_blocksize = AES_BLOCK_SIZE, 1895 }, 1896 .setkey = aead_setkey, 1897 .setauthsize = aead_setauthsize, 1898 .encrypt = aead_encrypt, 1899 .decrypt = aead_decrypt, 1900 .ivsize = AES_BLOCK_SIZE, 1901 .maxauthsize = SHA384_DIGEST_SIZE, 1902 }, 1903 .caam = { 1904 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1905 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1906 OP_ALG_AAI_HMAC_PRECOMP, 1907 .geniv = true, 1908 } 1909 }, 1910 { 1911 .aead = { 1912 .base = { 1913 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1914 .cra_driver_name = "authenc-hmac-sha512-" 1915 "cbc-aes-caam-qi2", 1916 .cra_blocksize = AES_BLOCK_SIZE, 1917 }, 1918 .setkey = aead_setkey, 1919 .setauthsize = aead_setauthsize, 1920 .encrypt = aead_encrypt, 1921 .decrypt = aead_decrypt, 1922 .ivsize = AES_BLOCK_SIZE, 1923 .maxauthsize = SHA512_DIGEST_SIZE, 1924 }, 1925 .caam = { 1926 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1927 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1928 OP_ALG_AAI_HMAC_PRECOMP, 1929 } 1930 }, 1931 { 1932 .aead = { 1933 .base = { 1934 .cra_name = "echainiv(authenc(hmac(sha512)," 1935 "cbc(aes)))", 1936 .cra_driver_name = "echainiv-authenc-" 1937 "hmac-sha512-cbc-aes-" 1938 "caam-qi2", 1939 .cra_blocksize = AES_BLOCK_SIZE, 1940 }, 1941 .setkey = aead_setkey, 1942 .setauthsize = aead_setauthsize, 1943 .encrypt = aead_encrypt, 1944 .decrypt = aead_decrypt, 1945 .ivsize = AES_BLOCK_SIZE, 1946 .maxauthsize = SHA512_DIGEST_SIZE, 1947 }, 1948 .caam = { 1949 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1950 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1951 OP_ALG_AAI_HMAC_PRECOMP, 1952 .geniv = true, 1953 } 1954 }, 1955 { 1956 .aead = { 1957 .base = { 1958 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 1959 .cra_driver_name = "authenc-hmac-md5-" 1960 "cbc-des3_ede-caam-qi2", 1961 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1962 }, 1963 .setkey = des3_aead_setkey, 1964 .setauthsize = aead_setauthsize, 1965 .encrypt = aead_encrypt, 1966 .decrypt = aead_decrypt, 1967 .ivsize = DES3_EDE_BLOCK_SIZE, 1968 .maxauthsize = MD5_DIGEST_SIZE, 1969 }, 1970 .caam = { 1971 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1972 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1973 OP_ALG_AAI_HMAC_PRECOMP, 1974 } 1975 }, 1976 { 1977 .aead = { 1978 .base = { 1979 .cra_name = "echainiv(authenc(hmac(md5)," 1980 "cbc(des3_ede)))", 1981 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1982 "cbc-des3_ede-caam-qi2", 1983 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1984 }, 1985 .setkey = des3_aead_setkey, 1986 .setauthsize = aead_setauthsize, 1987 .encrypt = aead_encrypt, 1988 .decrypt = aead_decrypt, 1989 .ivsize = DES3_EDE_BLOCK_SIZE, 1990 .maxauthsize = MD5_DIGEST_SIZE, 1991 }, 1992 .caam = { 1993 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1994 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1995 OP_ALG_AAI_HMAC_PRECOMP, 1996 .geniv = true, 1997 } 1998 }, 1999 { 2000 .aead = { 2001 .base = { 2002 .cra_name = "authenc(hmac(sha1)," 2003 "cbc(des3_ede))", 2004 .cra_driver_name = "authenc-hmac-sha1-" 2005 "cbc-des3_ede-caam-qi2", 2006 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2007 }, 2008 .setkey = des3_aead_setkey, 2009 .setauthsize = aead_setauthsize, 2010 .encrypt = aead_encrypt, 2011 .decrypt = aead_decrypt, 2012 .ivsize = DES3_EDE_BLOCK_SIZE, 2013 .maxauthsize = SHA1_DIGEST_SIZE, 2014 }, 2015 .caam = { 2016 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2017 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2018 OP_ALG_AAI_HMAC_PRECOMP, 2019 }, 2020 }, 2021 { 2022 .aead = { 2023 .base = { 2024 .cra_name = "echainiv(authenc(hmac(sha1)," 2025 "cbc(des3_ede)))", 2026 .cra_driver_name = "echainiv-authenc-" 2027 "hmac-sha1-" 2028 "cbc-des3_ede-caam-qi2", 2029 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2030 }, 2031 .setkey = des3_aead_setkey, 2032 .setauthsize = aead_setauthsize, 2033 .encrypt = aead_encrypt, 2034 .decrypt = aead_decrypt, 2035 .ivsize = DES3_EDE_BLOCK_SIZE, 2036 .maxauthsize = SHA1_DIGEST_SIZE, 2037 }, 2038 .caam = { 2039 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2040 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2041 OP_ALG_AAI_HMAC_PRECOMP, 2042 .geniv = true, 2043 } 2044 }, 2045 { 2046 .aead = { 2047 .base = { 2048 .cra_name = "authenc(hmac(sha224)," 2049 "cbc(des3_ede))", 2050 .cra_driver_name = "authenc-hmac-sha224-" 2051 "cbc-des3_ede-caam-qi2", 2052 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2053 }, 2054 .setkey = des3_aead_setkey, 2055 .setauthsize = aead_setauthsize, 2056 .encrypt = aead_encrypt, 2057 .decrypt = aead_decrypt, 2058 .ivsize = DES3_EDE_BLOCK_SIZE, 2059 .maxauthsize = SHA224_DIGEST_SIZE, 2060 }, 2061 .caam = { 2062 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2063 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2064 OP_ALG_AAI_HMAC_PRECOMP, 2065 }, 2066 }, 2067 { 2068 .aead = { 2069 .base = { 2070 .cra_name = "echainiv(authenc(hmac(sha224)," 2071 "cbc(des3_ede)))", 2072 .cra_driver_name = "echainiv-authenc-" 2073 "hmac-sha224-" 2074 "cbc-des3_ede-caam-qi2", 2075 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2076 }, 2077 .setkey = des3_aead_setkey, 2078 .setauthsize = aead_setauthsize, 2079 .encrypt = aead_encrypt, 2080 .decrypt = aead_decrypt, 2081 .ivsize = DES3_EDE_BLOCK_SIZE, 2082 .maxauthsize = SHA224_DIGEST_SIZE, 2083 }, 2084 .caam = { 2085 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2086 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2087 OP_ALG_AAI_HMAC_PRECOMP, 2088 .geniv = true, 2089 } 2090 }, 2091 { 2092 .aead = { 2093 .base = { 2094 .cra_name = "authenc(hmac(sha256)," 2095 "cbc(des3_ede))", 2096 .cra_driver_name = "authenc-hmac-sha256-" 2097 "cbc-des3_ede-caam-qi2", 2098 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2099 }, 2100 .setkey = des3_aead_setkey, 2101 .setauthsize = aead_setauthsize, 2102 .encrypt = aead_encrypt, 2103 .decrypt = aead_decrypt, 2104 .ivsize = DES3_EDE_BLOCK_SIZE, 2105 .maxauthsize = SHA256_DIGEST_SIZE, 2106 }, 2107 .caam = { 2108 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2109 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2110 OP_ALG_AAI_HMAC_PRECOMP, 2111 }, 2112 }, 2113 { 2114 .aead = { 2115 .base = { 2116 .cra_name = "echainiv(authenc(hmac(sha256)," 2117 "cbc(des3_ede)))", 2118 .cra_driver_name = "echainiv-authenc-" 2119 "hmac-sha256-" 2120 "cbc-des3_ede-caam-qi2", 2121 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2122 }, 2123 .setkey = des3_aead_setkey, 2124 .setauthsize = aead_setauthsize, 2125 .encrypt = aead_encrypt, 2126 .decrypt = aead_decrypt, 2127 .ivsize = DES3_EDE_BLOCK_SIZE, 2128 .maxauthsize = SHA256_DIGEST_SIZE, 2129 }, 2130 .caam = { 2131 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2132 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2133 OP_ALG_AAI_HMAC_PRECOMP, 2134 .geniv = true, 2135 } 2136 }, 2137 { 2138 .aead = { 2139 .base = { 2140 .cra_name = "authenc(hmac(sha384)," 2141 "cbc(des3_ede))", 2142 .cra_driver_name = "authenc-hmac-sha384-" 2143 "cbc-des3_ede-caam-qi2", 2144 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2145 }, 2146 .setkey = des3_aead_setkey, 2147 .setauthsize = aead_setauthsize, 2148 .encrypt = aead_encrypt, 2149 .decrypt = aead_decrypt, 2150 .ivsize = DES3_EDE_BLOCK_SIZE, 2151 .maxauthsize = SHA384_DIGEST_SIZE, 2152 }, 2153 .caam = { 2154 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2155 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2156 OP_ALG_AAI_HMAC_PRECOMP, 2157 }, 2158 }, 2159 { 2160 .aead = { 2161 .base = { 2162 .cra_name = "echainiv(authenc(hmac(sha384)," 2163 "cbc(des3_ede)))", 2164 .cra_driver_name = "echainiv-authenc-" 2165 "hmac-sha384-" 2166 "cbc-des3_ede-caam-qi2", 2167 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2168 }, 2169 .setkey = des3_aead_setkey, 2170 .setauthsize = aead_setauthsize, 2171 .encrypt = aead_encrypt, 2172 .decrypt = aead_decrypt, 2173 .ivsize = DES3_EDE_BLOCK_SIZE, 2174 .maxauthsize = SHA384_DIGEST_SIZE, 2175 }, 2176 .caam = { 2177 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2178 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2179 OP_ALG_AAI_HMAC_PRECOMP, 2180 .geniv = true, 2181 } 2182 }, 2183 { 2184 .aead = { 2185 .base = { 2186 .cra_name = "authenc(hmac(sha512)," 2187 "cbc(des3_ede))", 2188 .cra_driver_name = "authenc-hmac-sha512-" 2189 "cbc-des3_ede-caam-qi2", 2190 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2191 }, 2192 .setkey = des3_aead_setkey, 2193 .setauthsize = aead_setauthsize, 2194 .encrypt = aead_encrypt, 2195 .decrypt = aead_decrypt, 2196 .ivsize = DES3_EDE_BLOCK_SIZE, 2197 .maxauthsize = SHA512_DIGEST_SIZE, 2198 }, 2199 .caam = { 2200 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2201 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2202 OP_ALG_AAI_HMAC_PRECOMP, 2203 }, 2204 }, 2205 { 2206 .aead = { 2207 .base = { 2208 .cra_name = "echainiv(authenc(hmac(sha512)," 2209 "cbc(des3_ede)))", 2210 .cra_driver_name = "echainiv-authenc-" 2211 "hmac-sha512-" 2212 "cbc-des3_ede-caam-qi2", 2213 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2214 }, 2215 .setkey = des3_aead_setkey, 2216 .setauthsize = aead_setauthsize, 2217 .encrypt = aead_encrypt, 2218 .decrypt = aead_decrypt, 2219 .ivsize = DES3_EDE_BLOCK_SIZE, 2220 .maxauthsize = SHA512_DIGEST_SIZE, 2221 }, 2222 .caam = { 2223 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2224 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2225 OP_ALG_AAI_HMAC_PRECOMP, 2226 .geniv = true, 2227 } 2228 }, 2229 { 2230 .aead = { 2231 .base = { 2232 .cra_name = "authenc(hmac(md5),cbc(des))", 2233 .cra_driver_name = "authenc-hmac-md5-" 2234 "cbc-des-caam-qi2", 2235 .cra_blocksize = DES_BLOCK_SIZE, 2236 }, 2237 .setkey = aead_setkey, 2238 .setauthsize = aead_setauthsize, 2239 .encrypt = aead_encrypt, 2240 .decrypt = aead_decrypt, 2241 .ivsize = DES_BLOCK_SIZE, 2242 .maxauthsize = MD5_DIGEST_SIZE, 2243 }, 2244 .caam = { 2245 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2246 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2247 OP_ALG_AAI_HMAC_PRECOMP, 2248 }, 2249 }, 2250 { 2251 .aead = { 2252 .base = { 2253 .cra_name = "echainiv(authenc(hmac(md5)," 2254 "cbc(des)))", 2255 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2256 "cbc-des-caam-qi2", 2257 .cra_blocksize = DES_BLOCK_SIZE, 2258 }, 2259 .setkey = aead_setkey, 2260 .setauthsize = aead_setauthsize, 2261 .encrypt = aead_encrypt, 2262 .decrypt = aead_decrypt, 2263 .ivsize = DES_BLOCK_SIZE, 2264 .maxauthsize = MD5_DIGEST_SIZE, 2265 }, 2266 .caam = { 2267 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2268 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2269 OP_ALG_AAI_HMAC_PRECOMP, 2270 .geniv = true, 2271 } 2272 }, 2273 { 2274 .aead = { 2275 .base = { 2276 .cra_name = "authenc(hmac(sha1),cbc(des))", 2277 .cra_driver_name = "authenc-hmac-sha1-" 2278 "cbc-des-caam-qi2", 2279 .cra_blocksize = DES_BLOCK_SIZE, 2280 }, 2281 .setkey = aead_setkey, 2282 .setauthsize = aead_setauthsize, 2283 .encrypt = aead_encrypt, 2284 .decrypt = aead_decrypt, 2285 .ivsize = DES_BLOCK_SIZE, 2286 .maxauthsize = SHA1_DIGEST_SIZE, 2287 }, 2288 .caam = { 2289 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2290 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2291 OP_ALG_AAI_HMAC_PRECOMP, 2292 }, 2293 }, 2294 { 2295 .aead = { 2296 .base = { 2297 .cra_name = "echainiv(authenc(hmac(sha1)," 2298 "cbc(des)))", 2299 .cra_driver_name = "echainiv-authenc-" 2300 "hmac-sha1-cbc-des-caam-qi2", 2301 .cra_blocksize = DES_BLOCK_SIZE, 2302 }, 2303 .setkey = aead_setkey, 2304 .setauthsize = aead_setauthsize, 2305 .encrypt = aead_encrypt, 2306 .decrypt = aead_decrypt, 2307 .ivsize = DES_BLOCK_SIZE, 2308 .maxauthsize = SHA1_DIGEST_SIZE, 2309 }, 2310 .caam = { 2311 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2312 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2313 OP_ALG_AAI_HMAC_PRECOMP, 2314 .geniv = true, 2315 } 2316 }, 2317 { 2318 .aead = { 2319 .base = { 2320 .cra_name = "authenc(hmac(sha224),cbc(des))", 2321 .cra_driver_name = "authenc-hmac-sha224-" 2322 "cbc-des-caam-qi2", 2323 .cra_blocksize = DES_BLOCK_SIZE, 2324 }, 2325 .setkey = aead_setkey, 2326 .setauthsize = aead_setauthsize, 2327 .encrypt = aead_encrypt, 2328 .decrypt = aead_decrypt, 2329 .ivsize = DES_BLOCK_SIZE, 2330 .maxauthsize = SHA224_DIGEST_SIZE, 2331 }, 2332 .caam = { 2333 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2334 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2335 OP_ALG_AAI_HMAC_PRECOMP, 2336 }, 2337 }, 2338 { 2339 .aead = { 2340 .base = { 2341 .cra_name = "echainiv(authenc(hmac(sha224)," 2342 "cbc(des)))", 2343 .cra_driver_name = "echainiv-authenc-" 2344 "hmac-sha224-cbc-des-" 2345 "caam-qi2", 2346 .cra_blocksize = DES_BLOCK_SIZE, 2347 }, 2348 .setkey = aead_setkey, 2349 .setauthsize = aead_setauthsize, 2350 .encrypt = aead_encrypt, 2351 .decrypt = aead_decrypt, 2352 .ivsize = DES_BLOCK_SIZE, 2353 .maxauthsize = SHA224_DIGEST_SIZE, 2354 }, 2355 .caam = { 2356 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2357 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2358 OP_ALG_AAI_HMAC_PRECOMP, 2359 .geniv = true, 2360 } 2361 }, 2362 { 2363 .aead = { 2364 .base = { 2365 .cra_name = "authenc(hmac(sha256),cbc(des))", 2366 .cra_driver_name = "authenc-hmac-sha256-" 2367 "cbc-des-caam-qi2", 2368 .cra_blocksize = DES_BLOCK_SIZE, 2369 }, 2370 .setkey = aead_setkey, 2371 .setauthsize = aead_setauthsize, 2372 .encrypt = aead_encrypt, 2373 .decrypt = aead_decrypt, 2374 .ivsize = DES_BLOCK_SIZE, 2375 .maxauthsize = SHA256_DIGEST_SIZE, 2376 }, 2377 .caam = { 2378 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2379 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2380 OP_ALG_AAI_HMAC_PRECOMP, 2381 }, 2382 }, 2383 { 2384 .aead = { 2385 .base = { 2386 .cra_name = "echainiv(authenc(hmac(sha256)," 2387 "cbc(des)))", 2388 .cra_driver_name = "echainiv-authenc-" 2389 "hmac-sha256-cbc-desi-" 2390 "caam-qi2", 2391 .cra_blocksize = DES_BLOCK_SIZE, 2392 }, 2393 .setkey = aead_setkey, 2394 .setauthsize = aead_setauthsize, 2395 .encrypt = aead_encrypt, 2396 .decrypt = aead_decrypt, 2397 .ivsize = DES_BLOCK_SIZE, 2398 .maxauthsize = SHA256_DIGEST_SIZE, 2399 }, 2400 .caam = { 2401 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2402 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2403 OP_ALG_AAI_HMAC_PRECOMP, 2404 .geniv = true, 2405 }, 2406 }, 2407 { 2408 .aead = { 2409 .base = { 2410 .cra_name = "authenc(hmac(sha384),cbc(des))", 2411 .cra_driver_name = "authenc-hmac-sha384-" 2412 "cbc-des-caam-qi2", 2413 .cra_blocksize = DES_BLOCK_SIZE, 2414 }, 2415 .setkey = aead_setkey, 2416 .setauthsize = aead_setauthsize, 2417 .encrypt = aead_encrypt, 2418 .decrypt = aead_decrypt, 2419 .ivsize = DES_BLOCK_SIZE, 2420 .maxauthsize = SHA384_DIGEST_SIZE, 2421 }, 2422 .caam = { 2423 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2424 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2425 OP_ALG_AAI_HMAC_PRECOMP, 2426 }, 2427 }, 2428 { 2429 .aead = { 2430 .base = { 2431 .cra_name = "echainiv(authenc(hmac(sha384)," 2432 "cbc(des)))", 2433 .cra_driver_name = "echainiv-authenc-" 2434 "hmac-sha384-cbc-des-" 2435 "caam-qi2", 2436 .cra_blocksize = DES_BLOCK_SIZE, 2437 }, 2438 .setkey = aead_setkey, 2439 .setauthsize = aead_setauthsize, 2440 .encrypt = aead_encrypt, 2441 .decrypt = aead_decrypt, 2442 .ivsize = DES_BLOCK_SIZE, 2443 .maxauthsize = SHA384_DIGEST_SIZE, 2444 }, 2445 .caam = { 2446 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2447 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2448 OP_ALG_AAI_HMAC_PRECOMP, 2449 .geniv = true, 2450 } 2451 }, 2452 { 2453 .aead = { 2454 .base = { 2455 .cra_name = "authenc(hmac(sha512),cbc(des))", 2456 .cra_driver_name = "authenc-hmac-sha512-" 2457 "cbc-des-caam-qi2", 2458 .cra_blocksize = DES_BLOCK_SIZE, 2459 }, 2460 .setkey = aead_setkey, 2461 .setauthsize = aead_setauthsize, 2462 .encrypt = aead_encrypt, 2463 .decrypt = aead_decrypt, 2464 .ivsize = DES_BLOCK_SIZE, 2465 .maxauthsize = SHA512_DIGEST_SIZE, 2466 }, 2467 .caam = { 2468 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2469 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2470 OP_ALG_AAI_HMAC_PRECOMP, 2471 } 2472 }, 2473 { 2474 .aead = { 2475 .base = { 2476 .cra_name = "echainiv(authenc(hmac(sha512)," 2477 "cbc(des)))", 2478 .cra_driver_name = "echainiv-authenc-" 2479 "hmac-sha512-cbc-des-" 2480 "caam-qi2", 2481 .cra_blocksize = DES_BLOCK_SIZE, 2482 }, 2483 .setkey = aead_setkey, 2484 .setauthsize = aead_setauthsize, 2485 .encrypt = aead_encrypt, 2486 .decrypt = aead_decrypt, 2487 .ivsize = DES_BLOCK_SIZE, 2488 .maxauthsize = SHA512_DIGEST_SIZE, 2489 }, 2490 .caam = { 2491 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2492 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2493 OP_ALG_AAI_HMAC_PRECOMP, 2494 .geniv = true, 2495 } 2496 }, 2497 { 2498 .aead = { 2499 .base = { 2500 .cra_name = "authenc(hmac(md5)," 2501 "rfc3686(ctr(aes)))", 2502 .cra_driver_name = "authenc-hmac-md5-" 2503 "rfc3686-ctr-aes-caam-qi2", 2504 .cra_blocksize = 1, 2505 }, 2506 .setkey = aead_setkey, 2507 .setauthsize = aead_setauthsize, 2508 .encrypt = aead_encrypt, 2509 .decrypt = aead_decrypt, 2510 .ivsize = CTR_RFC3686_IV_SIZE, 2511 .maxauthsize = MD5_DIGEST_SIZE, 2512 }, 2513 .caam = { 2514 .class1_alg_type = OP_ALG_ALGSEL_AES | 2515 OP_ALG_AAI_CTR_MOD128, 2516 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2517 OP_ALG_AAI_HMAC_PRECOMP, 2518 .rfc3686 = true, 2519 }, 2520 }, 2521 { 2522 .aead = { 2523 .base = { 2524 .cra_name = "seqiv(authenc(" 2525 "hmac(md5),rfc3686(ctr(aes))))", 2526 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2527 "rfc3686-ctr-aes-caam-qi2", 2528 .cra_blocksize = 1, 2529 }, 2530 .setkey = aead_setkey, 2531 .setauthsize = aead_setauthsize, 2532 .encrypt = aead_encrypt, 2533 .decrypt = aead_decrypt, 2534 .ivsize = CTR_RFC3686_IV_SIZE, 2535 .maxauthsize = MD5_DIGEST_SIZE, 2536 }, 2537 .caam = { 2538 .class1_alg_type = OP_ALG_ALGSEL_AES | 2539 OP_ALG_AAI_CTR_MOD128, 2540 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2541 OP_ALG_AAI_HMAC_PRECOMP, 2542 .rfc3686 = true, 2543 .geniv = true, 2544 }, 2545 }, 2546 { 2547 .aead = { 2548 .base = { 2549 .cra_name = "authenc(hmac(sha1)," 2550 "rfc3686(ctr(aes)))", 2551 .cra_driver_name = "authenc-hmac-sha1-" 2552 "rfc3686-ctr-aes-caam-qi2", 2553 .cra_blocksize = 1, 2554 }, 2555 .setkey = aead_setkey, 2556 .setauthsize = aead_setauthsize, 2557 .encrypt = aead_encrypt, 2558 .decrypt = aead_decrypt, 2559 .ivsize = CTR_RFC3686_IV_SIZE, 2560 .maxauthsize = SHA1_DIGEST_SIZE, 2561 }, 2562 .caam = { 2563 .class1_alg_type = OP_ALG_ALGSEL_AES | 2564 OP_ALG_AAI_CTR_MOD128, 2565 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2566 OP_ALG_AAI_HMAC_PRECOMP, 2567 .rfc3686 = true, 2568 }, 2569 }, 2570 { 2571 .aead = { 2572 .base = { 2573 .cra_name = "seqiv(authenc(" 2574 "hmac(sha1),rfc3686(ctr(aes))))", 2575 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2576 "rfc3686-ctr-aes-caam-qi2", 2577 .cra_blocksize = 1, 2578 }, 2579 .setkey = aead_setkey, 2580 .setauthsize = aead_setauthsize, 2581 .encrypt = aead_encrypt, 2582 .decrypt = aead_decrypt, 2583 .ivsize = CTR_RFC3686_IV_SIZE, 2584 .maxauthsize = SHA1_DIGEST_SIZE, 2585 }, 2586 .caam = { 2587 .class1_alg_type = OP_ALG_ALGSEL_AES | 2588 OP_ALG_AAI_CTR_MOD128, 2589 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2590 OP_ALG_AAI_HMAC_PRECOMP, 2591 .rfc3686 = true, 2592 .geniv = true, 2593 }, 2594 }, 2595 { 2596 .aead = { 2597 .base = { 2598 .cra_name = "authenc(hmac(sha224)," 2599 "rfc3686(ctr(aes)))", 2600 .cra_driver_name = "authenc-hmac-sha224-" 2601 "rfc3686-ctr-aes-caam-qi2", 2602 .cra_blocksize = 1, 2603 }, 2604 .setkey = aead_setkey, 2605 .setauthsize = aead_setauthsize, 2606 .encrypt = aead_encrypt, 2607 .decrypt = aead_decrypt, 2608 .ivsize = CTR_RFC3686_IV_SIZE, 2609 .maxauthsize = SHA224_DIGEST_SIZE, 2610 }, 2611 .caam = { 2612 .class1_alg_type = OP_ALG_ALGSEL_AES | 2613 OP_ALG_AAI_CTR_MOD128, 2614 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2615 OP_ALG_AAI_HMAC_PRECOMP, 2616 .rfc3686 = true, 2617 }, 2618 }, 2619 { 2620 .aead = { 2621 .base = { 2622 .cra_name = "seqiv(authenc(" 2623 "hmac(sha224),rfc3686(ctr(aes))))", 2624 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2625 "rfc3686-ctr-aes-caam-qi2", 2626 .cra_blocksize = 1, 2627 }, 2628 .setkey = aead_setkey, 2629 .setauthsize = aead_setauthsize, 2630 .encrypt = aead_encrypt, 2631 .decrypt = aead_decrypt, 2632 .ivsize = CTR_RFC3686_IV_SIZE, 2633 .maxauthsize = SHA224_DIGEST_SIZE, 2634 }, 2635 .caam = { 2636 .class1_alg_type = OP_ALG_ALGSEL_AES | 2637 OP_ALG_AAI_CTR_MOD128, 2638 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2639 OP_ALG_AAI_HMAC_PRECOMP, 2640 .rfc3686 = true, 2641 .geniv = true, 2642 }, 2643 }, 2644 { 2645 .aead = { 2646 .base = { 2647 .cra_name = "authenc(hmac(sha256)," 2648 "rfc3686(ctr(aes)))", 2649 .cra_driver_name = "authenc-hmac-sha256-" 2650 "rfc3686-ctr-aes-caam-qi2", 2651 .cra_blocksize = 1, 2652 }, 2653 .setkey = aead_setkey, 2654 .setauthsize = aead_setauthsize, 2655 .encrypt = aead_encrypt, 2656 .decrypt = aead_decrypt, 2657 .ivsize = CTR_RFC3686_IV_SIZE, 2658 .maxauthsize = SHA256_DIGEST_SIZE, 2659 }, 2660 .caam = { 2661 .class1_alg_type = OP_ALG_ALGSEL_AES | 2662 OP_ALG_AAI_CTR_MOD128, 2663 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2664 OP_ALG_AAI_HMAC_PRECOMP, 2665 .rfc3686 = true, 2666 }, 2667 }, 2668 { 2669 .aead = { 2670 .base = { 2671 .cra_name = "seqiv(authenc(hmac(sha256)," 2672 "rfc3686(ctr(aes))))", 2673 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2674 "rfc3686-ctr-aes-caam-qi2", 2675 .cra_blocksize = 1, 2676 }, 2677 .setkey = aead_setkey, 2678 .setauthsize = aead_setauthsize, 2679 .encrypt = aead_encrypt, 2680 .decrypt = aead_decrypt, 2681 .ivsize = CTR_RFC3686_IV_SIZE, 2682 .maxauthsize = SHA256_DIGEST_SIZE, 2683 }, 2684 .caam = { 2685 .class1_alg_type = OP_ALG_ALGSEL_AES | 2686 OP_ALG_AAI_CTR_MOD128, 2687 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2688 OP_ALG_AAI_HMAC_PRECOMP, 2689 .rfc3686 = true, 2690 .geniv = true, 2691 }, 2692 }, 2693 { 2694 .aead = { 2695 .base = { 2696 .cra_name = "authenc(hmac(sha384)," 2697 "rfc3686(ctr(aes)))", 2698 .cra_driver_name = "authenc-hmac-sha384-" 2699 "rfc3686-ctr-aes-caam-qi2", 2700 .cra_blocksize = 1, 2701 }, 2702 .setkey = aead_setkey, 2703 .setauthsize = aead_setauthsize, 2704 .encrypt = aead_encrypt, 2705 .decrypt = aead_decrypt, 2706 .ivsize = CTR_RFC3686_IV_SIZE, 2707 .maxauthsize = SHA384_DIGEST_SIZE, 2708 }, 2709 .caam = { 2710 .class1_alg_type = OP_ALG_ALGSEL_AES | 2711 OP_ALG_AAI_CTR_MOD128, 2712 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2713 OP_ALG_AAI_HMAC_PRECOMP, 2714 .rfc3686 = true, 2715 }, 2716 }, 2717 { 2718 .aead = { 2719 .base = { 2720 .cra_name = "seqiv(authenc(hmac(sha384)," 2721 "rfc3686(ctr(aes))))", 2722 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2723 "rfc3686-ctr-aes-caam-qi2", 2724 .cra_blocksize = 1, 2725 }, 2726 .setkey = aead_setkey, 2727 .setauthsize = aead_setauthsize, 2728 .encrypt = aead_encrypt, 2729 .decrypt = aead_decrypt, 2730 .ivsize = CTR_RFC3686_IV_SIZE, 2731 .maxauthsize = SHA384_DIGEST_SIZE, 2732 }, 2733 .caam = { 2734 .class1_alg_type = OP_ALG_ALGSEL_AES | 2735 OP_ALG_AAI_CTR_MOD128, 2736 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2737 OP_ALG_AAI_HMAC_PRECOMP, 2738 .rfc3686 = true, 2739 .geniv = true, 2740 }, 2741 }, 2742 { 2743 .aead = { 2744 .base = { 2745 .cra_name = "rfc7539(chacha20,poly1305)", 2746 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2747 "caam-qi2", 2748 .cra_blocksize = 1, 2749 }, 2750 .setkey = chachapoly_setkey, 2751 .setauthsize = chachapoly_setauthsize, 2752 .encrypt = aead_encrypt, 2753 .decrypt = aead_decrypt, 2754 .ivsize = CHACHAPOLY_IV_SIZE, 2755 .maxauthsize = POLY1305_DIGEST_SIZE, 2756 }, 2757 .caam = { 2758 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2759 OP_ALG_AAI_AEAD, 2760 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2761 OP_ALG_AAI_AEAD, 2762 .nodkp = true, 2763 }, 2764 }, 2765 { 2766 .aead = { 2767 .base = { 2768 .cra_name = "rfc7539esp(chacha20,poly1305)", 2769 .cra_driver_name = "rfc7539esp-chacha20-" 2770 "poly1305-caam-qi2", 2771 .cra_blocksize = 1, 2772 }, 2773 .setkey = chachapoly_setkey, 2774 .setauthsize = chachapoly_setauthsize, 2775 .encrypt = aead_encrypt, 2776 .decrypt = aead_decrypt, 2777 .ivsize = 8, 2778 .maxauthsize = POLY1305_DIGEST_SIZE, 2779 }, 2780 .caam = { 2781 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2782 OP_ALG_AAI_AEAD, 2783 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2784 OP_ALG_AAI_AEAD, 2785 .nodkp = true, 2786 }, 2787 }, 2788 { 2789 .aead = { 2790 .base = { 2791 .cra_name = "authenc(hmac(sha512)," 2792 "rfc3686(ctr(aes)))", 2793 .cra_driver_name = "authenc-hmac-sha512-" 2794 "rfc3686-ctr-aes-caam-qi2", 2795 .cra_blocksize = 1, 2796 }, 2797 .setkey = aead_setkey, 2798 .setauthsize = aead_setauthsize, 2799 .encrypt = aead_encrypt, 2800 .decrypt = aead_decrypt, 2801 .ivsize = CTR_RFC3686_IV_SIZE, 2802 .maxauthsize = SHA512_DIGEST_SIZE, 2803 }, 2804 .caam = { 2805 .class1_alg_type = OP_ALG_ALGSEL_AES | 2806 OP_ALG_AAI_CTR_MOD128, 2807 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2808 OP_ALG_AAI_HMAC_PRECOMP, 2809 .rfc3686 = true, 2810 }, 2811 }, 2812 { 2813 .aead = { 2814 .base = { 2815 .cra_name = "seqiv(authenc(hmac(sha512)," 2816 "rfc3686(ctr(aes))))", 2817 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2818 "rfc3686-ctr-aes-caam-qi2", 2819 .cra_blocksize = 1, 2820 }, 2821 .setkey = aead_setkey, 2822 .setauthsize = aead_setauthsize, 2823 .encrypt = aead_encrypt, 2824 .decrypt = aead_decrypt, 2825 .ivsize = CTR_RFC3686_IV_SIZE, 2826 .maxauthsize = SHA512_DIGEST_SIZE, 2827 }, 2828 .caam = { 2829 .class1_alg_type = OP_ALG_ALGSEL_AES | 2830 OP_ALG_AAI_CTR_MOD128, 2831 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2832 OP_ALG_AAI_HMAC_PRECOMP, 2833 .rfc3686 = true, 2834 .geniv = true, 2835 }, 2836 }, 2837 }; 2838 2839 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2840 { 2841 struct skcipher_alg *alg = &t_alg->skcipher; 2842 2843 alg->base.cra_module = THIS_MODULE; 2844 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2845 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2846 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2847 2848 alg->init = caam_cra_init_skcipher; 2849 alg->exit = caam_cra_exit; 2850 } 2851 2852 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2853 { 2854 struct aead_alg *alg = &t_alg->aead; 2855 2856 alg->base.cra_module = THIS_MODULE; 2857 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2858 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2859 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2860 2861 alg->init = caam_cra_init_aead; 2862 alg->exit = caam_cra_exit_aead; 2863 } 2864 2865 /* max hash key is max split key size */ 2866 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 2867 2868 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 2869 2870 /* caam context sizes for hashes: running digest + 8 */ 2871 #define HASH_MSG_LEN 8 2872 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 2873 2874 enum hash_optype { 2875 UPDATE = 0, 2876 UPDATE_FIRST, 2877 FINALIZE, 2878 DIGEST, 2879 HASH_NUM_OP 2880 }; 2881 2882 /** 2883 * caam_hash_ctx - ahash per-session context 2884 * @flc: Flow Contexts array 2885 * @flc_dma: I/O virtual addresses of the Flow Contexts 2886 * @dev: dpseci device 2887 * @ctx_len: size of Context Register 2888 * @adata: hashing algorithm details 2889 */ 2890 struct caam_hash_ctx { 2891 struct caam_flc flc[HASH_NUM_OP]; 2892 dma_addr_t flc_dma[HASH_NUM_OP]; 2893 struct device *dev; 2894 int ctx_len; 2895 struct alginfo adata; 2896 }; 2897 2898 /* ahash state */ 2899 struct caam_hash_state { 2900 struct caam_request caam_req; 2901 dma_addr_t buf_dma; 2902 dma_addr_t ctx_dma; 2903 int ctx_dma_len; 2904 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2905 int buflen_0; 2906 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2907 int buflen_1; 2908 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 2909 int (*update)(struct ahash_request *req); 2910 int (*final)(struct ahash_request *req); 2911 int (*finup)(struct ahash_request *req); 2912 int current_buf; 2913 }; 2914 2915 struct caam_export_state { 2916 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 2917 u8 caam_ctx[MAX_CTX_LEN]; 2918 int buflen; 2919 int (*update)(struct ahash_request *req); 2920 int (*final)(struct ahash_request *req); 2921 int (*finup)(struct ahash_request *req); 2922 }; 2923 2924 static inline void switch_buf(struct caam_hash_state *state) 2925 { 2926 state->current_buf ^= 1; 2927 } 2928 2929 static inline u8 *current_buf(struct caam_hash_state *state) 2930 { 2931 return state->current_buf ? state->buf_1 : state->buf_0; 2932 } 2933 2934 static inline u8 *alt_buf(struct caam_hash_state *state) 2935 { 2936 return state->current_buf ? state->buf_0 : state->buf_1; 2937 } 2938 2939 static inline int *current_buflen(struct caam_hash_state *state) 2940 { 2941 return state->current_buf ? &state->buflen_1 : &state->buflen_0; 2942 } 2943 2944 static inline int *alt_buflen(struct caam_hash_state *state) 2945 { 2946 return state->current_buf ? &state->buflen_0 : &state->buflen_1; 2947 } 2948 2949 /* Map current buffer in state (if length > 0) and put it in link table */ 2950 static inline int buf_map_to_qm_sg(struct device *dev, 2951 struct dpaa2_sg_entry *qm_sg, 2952 struct caam_hash_state *state) 2953 { 2954 int buflen = *current_buflen(state); 2955 2956 if (!buflen) 2957 return 0; 2958 2959 state->buf_dma = dma_map_single(dev, current_buf(state), buflen, 2960 DMA_TO_DEVICE); 2961 if (dma_mapping_error(dev, state->buf_dma)) { 2962 dev_err(dev, "unable to map buf\n"); 2963 state->buf_dma = 0; 2964 return -ENOMEM; 2965 } 2966 2967 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 2968 2969 return 0; 2970 } 2971 2972 /* Map state->caam_ctx, and add it to link table */ 2973 static inline int ctx_map_to_qm_sg(struct device *dev, 2974 struct caam_hash_state *state, int ctx_len, 2975 struct dpaa2_sg_entry *qm_sg, u32 flag) 2976 { 2977 state->ctx_dma_len = ctx_len; 2978 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 2979 if (dma_mapping_error(dev, state->ctx_dma)) { 2980 dev_err(dev, "unable to map ctx\n"); 2981 state->ctx_dma = 0; 2982 return -ENOMEM; 2983 } 2984 2985 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 2986 2987 return 0; 2988 } 2989 2990 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 2991 { 2992 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 2993 int digestsize = crypto_ahash_digestsize(ahash); 2994 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 2995 struct caam_flc *flc; 2996 u32 *desc; 2997 2998 /* ahash_update shared descriptor */ 2999 flc = &ctx->flc[UPDATE]; 3000 desc = flc->sh_desc; 3001 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3002 ctx->ctx_len, true, priv->sec_attr.era); 3003 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3004 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3005 desc_bytes(desc), DMA_BIDIRECTIONAL); 3006 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3007 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3008 1); 3009 3010 /* ahash_update_first shared descriptor */ 3011 flc = &ctx->flc[UPDATE_FIRST]; 3012 desc = flc->sh_desc; 3013 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3014 ctx->ctx_len, false, priv->sec_attr.era); 3015 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3016 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3017 desc_bytes(desc), DMA_BIDIRECTIONAL); 3018 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3019 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3020 1); 3021 3022 /* ahash_final shared descriptor */ 3023 flc = &ctx->flc[FINALIZE]; 3024 desc = flc->sh_desc; 3025 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3026 ctx->ctx_len, true, priv->sec_attr.era); 3027 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3028 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3029 desc_bytes(desc), DMA_BIDIRECTIONAL); 3030 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3031 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3032 1); 3033 3034 /* ahash_digest shared descriptor */ 3035 flc = &ctx->flc[DIGEST]; 3036 desc = flc->sh_desc; 3037 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3038 ctx->ctx_len, false, priv->sec_attr.era); 3039 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3040 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3041 desc_bytes(desc), DMA_BIDIRECTIONAL); 3042 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3043 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3044 1); 3045 3046 return 0; 3047 } 3048 3049 struct split_key_sh_result { 3050 struct completion completion; 3051 int err; 3052 struct device *dev; 3053 }; 3054 3055 static void split_key_sh_done(void *cbk_ctx, u32 err) 3056 { 3057 struct split_key_sh_result *res = cbk_ctx; 3058 3059 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3060 3061 if (err) 3062 caam_qi2_strstatus(res->dev, err); 3063 3064 res->err = err; 3065 complete(&res->completion); 3066 } 3067 3068 /* Digest hash size if it is too large */ 3069 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3070 u32 digestsize) 3071 { 3072 struct caam_request *req_ctx; 3073 u32 *desc; 3074 struct split_key_sh_result result; 3075 dma_addr_t key_dma; 3076 struct caam_flc *flc; 3077 dma_addr_t flc_dma; 3078 int ret = -ENOMEM; 3079 struct dpaa2_fl_entry *in_fle, *out_fle; 3080 3081 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 3082 if (!req_ctx) 3083 return -ENOMEM; 3084 3085 in_fle = &req_ctx->fd_flt[1]; 3086 out_fle = &req_ctx->fd_flt[0]; 3087 3088 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 3089 if (!flc) 3090 goto err_flc; 3091 3092 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3093 if (dma_mapping_error(ctx->dev, key_dma)) { 3094 dev_err(ctx->dev, "unable to map key memory\n"); 3095 goto err_key_dma; 3096 } 3097 3098 desc = flc->sh_desc; 3099 3100 init_sh_desc(desc, 0); 3101 3102 /* descriptor to perform unkeyed hash on key_in */ 3103 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3104 OP_ALG_AS_INITFINAL); 3105 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3106 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3107 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3108 LDST_SRCDST_BYTE_CONTEXT); 3109 3110 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3111 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3112 desc_bytes(desc), DMA_TO_DEVICE); 3113 if (dma_mapping_error(ctx->dev, flc_dma)) { 3114 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3115 goto err_flc_dma; 3116 } 3117 3118 dpaa2_fl_set_final(in_fle, true); 3119 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3120 dpaa2_fl_set_addr(in_fle, key_dma); 3121 dpaa2_fl_set_len(in_fle, *keylen); 3122 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3123 dpaa2_fl_set_addr(out_fle, key_dma); 3124 dpaa2_fl_set_len(out_fle, digestsize); 3125 3126 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3127 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3128 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3129 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3130 1); 3131 3132 result.err = 0; 3133 init_completion(&result.completion); 3134 result.dev = ctx->dev; 3135 3136 req_ctx->flc = flc; 3137 req_ctx->flc_dma = flc_dma; 3138 req_ctx->cbk = split_key_sh_done; 3139 req_ctx->ctx = &result; 3140 3141 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3142 if (ret == -EINPROGRESS) { 3143 /* in progress */ 3144 wait_for_completion(&result.completion); 3145 ret = result.err; 3146 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3147 DUMP_PREFIX_ADDRESS, 16, 4, key, 3148 digestsize, 1); 3149 } 3150 3151 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3152 DMA_TO_DEVICE); 3153 err_flc_dma: 3154 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3155 err_key_dma: 3156 kfree(flc); 3157 err_flc: 3158 kfree(req_ctx); 3159 3160 *keylen = digestsize; 3161 3162 return ret; 3163 } 3164 3165 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3166 unsigned int keylen) 3167 { 3168 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3169 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3170 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3171 int ret; 3172 u8 *hashed_key = NULL; 3173 3174 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3175 3176 if (keylen > blocksize) { 3177 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 3178 if (!hashed_key) 3179 return -ENOMEM; 3180 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3181 if (ret) 3182 goto bad_free_key; 3183 key = hashed_key; 3184 } 3185 3186 ctx->adata.keylen = keylen; 3187 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3188 OP_ALG_ALGSEL_MASK); 3189 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3190 goto bad_free_key; 3191 3192 ctx->adata.key_virt = key; 3193 ctx->adata.key_inline = true; 3194 3195 ret = ahash_set_sh_desc(ahash); 3196 kfree(hashed_key); 3197 return ret; 3198 bad_free_key: 3199 kfree(hashed_key); 3200 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); 3201 return -EINVAL; 3202 } 3203 3204 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3205 struct ahash_request *req) 3206 { 3207 struct caam_hash_state *state = ahash_request_ctx(req); 3208 3209 if (edesc->src_nents) 3210 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3211 3212 if (edesc->qm_sg_bytes) 3213 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3214 DMA_TO_DEVICE); 3215 3216 if (state->buf_dma) { 3217 dma_unmap_single(dev, state->buf_dma, *current_buflen(state), 3218 DMA_TO_DEVICE); 3219 state->buf_dma = 0; 3220 } 3221 } 3222 3223 static inline void ahash_unmap_ctx(struct device *dev, 3224 struct ahash_edesc *edesc, 3225 struct ahash_request *req, u32 flag) 3226 { 3227 struct caam_hash_state *state = ahash_request_ctx(req); 3228 3229 if (state->ctx_dma) { 3230 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3231 state->ctx_dma = 0; 3232 } 3233 ahash_unmap(dev, edesc, req); 3234 } 3235 3236 static void ahash_done(void *cbk_ctx, u32 status) 3237 { 3238 struct crypto_async_request *areq = cbk_ctx; 3239 struct ahash_request *req = ahash_request_cast(areq); 3240 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3241 struct caam_hash_state *state = ahash_request_ctx(req); 3242 struct ahash_edesc *edesc = state->caam_req.edesc; 3243 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3244 int digestsize = crypto_ahash_digestsize(ahash); 3245 int ecode = 0; 3246 3247 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3248 3249 if (unlikely(status)) { 3250 caam_qi2_strstatus(ctx->dev, status); 3251 ecode = -EIO; 3252 } 3253 3254 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3255 memcpy(req->result, state->caam_ctx, digestsize); 3256 qi_cache_free(edesc); 3257 3258 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3259 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3260 ctx->ctx_len, 1); 3261 3262 req->base.complete(&req->base, ecode); 3263 } 3264 3265 static void ahash_done_bi(void *cbk_ctx, u32 status) 3266 { 3267 struct crypto_async_request *areq = cbk_ctx; 3268 struct ahash_request *req = ahash_request_cast(areq); 3269 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3270 struct caam_hash_state *state = ahash_request_ctx(req); 3271 struct ahash_edesc *edesc = state->caam_req.edesc; 3272 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3273 int ecode = 0; 3274 3275 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3276 3277 if (unlikely(status)) { 3278 caam_qi2_strstatus(ctx->dev, status); 3279 ecode = -EIO; 3280 } 3281 3282 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3283 switch_buf(state); 3284 qi_cache_free(edesc); 3285 3286 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3287 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3288 ctx->ctx_len, 1); 3289 if (req->result) 3290 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3291 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3292 crypto_ahash_digestsize(ahash), 1); 3293 3294 req->base.complete(&req->base, ecode); 3295 } 3296 3297 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3298 { 3299 struct crypto_async_request *areq = cbk_ctx; 3300 struct ahash_request *req = ahash_request_cast(areq); 3301 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3302 struct caam_hash_state *state = ahash_request_ctx(req); 3303 struct ahash_edesc *edesc = state->caam_req.edesc; 3304 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3305 int digestsize = crypto_ahash_digestsize(ahash); 3306 int ecode = 0; 3307 3308 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3309 3310 if (unlikely(status)) { 3311 caam_qi2_strstatus(ctx->dev, status); 3312 ecode = -EIO; 3313 } 3314 3315 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3316 memcpy(req->result, state->caam_ctx, digestsize); 3317 qi_cache_free(edesc); 3318 3319 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3320 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3321 ctx->ctx_len, 1); 3322 3323 req->base.complete(&req->base, ecode); 3324 } 3325 3326 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3327 { 3328 struct crypto_async_request *areq = cbk_ctx; 3329 struct ahash_request *req = ahash_request_cast(areq); 3330 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3331 struct caam_hash_state *state = ahash_request_ctx(req); 3332 struct ahash_edesc *edesc = state->caam_req.edesc; 3333 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3334 int ecode = 0; 3335 3336 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3337 3338 if (unlikely(status)) { 3339 caam_qi2_strstatus(ctx->dev, status); 3340 ecode = -EIO; 3341 } 3342 3343 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3344 switch_buf(state); 3345 qi_cache_free(edesc); 3346 3347 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3348 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3349 ctx->ctx_len, 1); 3350 if (req->result) 3351 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3352 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3353 crypto_ahash_digestsize(ahash), 1); 3354 3355 req->base.complete(&req->base, ecode); 3356 } 3357 3358 static int ahash_update_ctx(struct ahash_request *req) 3359 { 3360 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3361 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3362 struct caam_hash_state *state = ahash_request_ctx(req); 3363 struct caam_request *req_ctx = &state->caam_req; 3364 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3365 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3366 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3367 GFP_KERNEL : GFP_ATOMIC; 3368 u8 *buf = current_buf(state); 3369 int *buflen = current_buflen(state); 3370 u8 *next_buf = alt_buf(state); 3371 int *next_buflen = alt_buflen(state), last_buflen; 3372 int in_len = *buflen + req->nbytes, to_hash; 3373 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3374 struct ahash_edesc *edesc; 3375 int ret = 0; 3376 3377 last_buflen = *next_buflen; 3378 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3379 to_hash = in_len - *next_buflen; 3380 3381 if (to_hash) { 3382 struct dpaa2_sg_entry *sg_table; 3383 3384 src_nents = sg_nents_for_len(req->src, 3385 req->nbytes - (*next_buflen)); 3386 if (src_nents < 0) { 3387 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3388 return src_nents; 3389 } 3390 3391 if (src_nents) { 3392 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3393 DMA_TO_DEVICE); 3394 if (!mapped_nents) { 3395 dev_err(ctx->dev, "unable to DMA map source\n"); 3396 return -ENOMEM; 3397 } 3398 } else { 3399 mapped_nents = 0; 3400 } 3401 3402 /* allocate space for base edesc and link tables */ 3403 edesc = qi_cache_zalloc(GFP_DMA | flags); 3404 if (!edesc) { 3405 dma_unmap_sg(ctx->dev, req->src, src_nents, 3406 DMA_TO_DEVICE); 3407 return -ENOMEM; 3408 } 3409 3410 edesc->src_nents = src_nents; 3411 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3412 qm_sg_bytes = (qm_sg_src_index + mapped_nents) * 3413 sizeof(*sg_table); 3414 sg_table = &edesc->sgt[0]; 3415 3416 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3417 DMA_BIDIRECTIONAL); 3418 if (ret) 3419 goto unmap_ctx; 3420 3421 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3422 if (ret) 3423 goto unmap_ctx; 3424 3425 if (mapped_nents) { 3426 sg_to_qm_sg_last(req->src, mapped_nents, 3427 sg_table + qm_sg_src_index, 0); 3428 if (*next_buflen) 3429 scatterwalk_map_and_copy(next_buf, req->src, 3430 to_hash - *buflen, 3431 *next_buflen, 0); 3432 } else { 3433 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3434 true); 3435 } 3436 3437 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3438 qm_sg_bytes, DMA_TO_DEVICE); 3439 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3440 dev_err(ctx->dev, "unable to map S/G table\n"); 3441 ret = -ENOMEM; 3442 goto unmap_ctx; 3443 } 3444 edesc->qm_sg_bytes = qm_sg_bytes; 3445 3446 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3447 dpaa2_fl_set_final(in_fle, true); 3448 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3449 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3450 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3451 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3452 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3453 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3454 3455 req_ctx->flc = &ctx->flc[UPDATE]; 3456 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3457 req_ctx->cbk = ahash_done_bi; 3458 req_ctx->ctx = &req->base; 3459 req_ctx->edesc = edesc; 3460 3461 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3462 if (ret != -EINPROGRESS && 3463 !(ret == -EBUSY && 3464 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3465 goto unmap_ctx; 3466 } else if (*next_buflen) { 3467 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3468 req->nbytes, 0); 3469 *buflen = *next_buflen; 3470 *next_buflen = last_buflen; 3471 } 3472 3473 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3474 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 3475 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 3476 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 3477 1); 3478 3479 return ret; 3480 unmap_ctx: 3481 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3482 qi_cache_free(edesc); 3483 return ret; 3484 } 3485 3486 static int ahash_final_ctx(struct ahash_request *req) 3487 { 3488 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3489 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3490 struct caam_hash_state *state = ahash_request_ctx(req); 3491 struct caam_request *req_ctx = &state->caam_req; 3492 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3493 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3494 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3495 GFP_KERNEL : GFP_ATOMIC; 3496 int buflen = *current_buflen(state); 3497 int qm_sg_bytes, qm_sg_src_index; 3498 int digestsize = crypto_ahash_digestsize(ahash); 3499 struct ahash_edesc *edesc; 3500 struct dpaa2_sg_entry *sg_table; 3501 int ret; 3502 3503 /* allocate space for base edesc and link tables */ 3504 edesc = qi_cache_zalloc(GFP_DMA | flags); 3505 if (!edesc) 3506 return -ENOMEM; 3507 3508 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3509 qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); 3510 sg_table = &edesc->sgt[0]; 3511 3512 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3513 DMA_BIDIRECTIONAL); 3514 if (ret) 3515 goto unmap_ctx; 3516 3517 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3518 if (ret) 3519 goto unmap_ctx; 3520 3521 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); 3522 3523 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3524 DMA_TO_DEVICE); 3525 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3526 dev_err(ctx->dev, "unable to map S/G table\n"); 3527 ret = -ENOMEM; 3528 goto unmap_ctx; 3529 } 3530 edesc->qm_sg_bytes = qm_sg_bytes; 3531 3532 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3533 dpaa2_fl_set_final(in_fle, true); 3534 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3535 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3536 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3537 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3538 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3539 dpaa2_fl_set_len(out_fle, digestsize); 3540 3541 req_ctx->flc = &ctx->flc[FINALIZE]; 3542 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3543 req_ctx->cbk = ahash_done_ctx_src; 3544 req_ctx->ctx = &req->base; 3545 req_ctx->edesc = edesc; 3546 3547 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3548 if (ret == -EINPROGRESS || 3549 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3550 return ret; 3551 3552 unmap_ctx: 3553 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3554 qi_cache_free(edesc); 3555 return ret; 3556 } 3557 3558 static int ahash_finup_ctx(struct ahash_request *req) 3559 { 3560 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3561 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3562 struct caam_hash_state *state = ahash_request_ctx(req); 3563 struct caam_request *req_ctx = &state->caam_req; 3564 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3565 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3566 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3567 GFP_KERNEL : GFP_ATOMIC; 3568 int buflen = *current_buflen(state); 3569 int qm_sg_bytes, qm_sg_src_index; 3570 int src_nents, mapped_nents; 3571 int digestsize = crypto_ahash_digestsize(ahash); 3572 struct ahash_edesc *edesc; 3573 struct dpaa2_sg_entry *sg_table; 3574 int ret; 3575 3576 src_nents = sg_nents_for_len(req->src, req->nbytes); 3577 if (src_nents < 0) { 3578 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3579 return src_nents; 3580 } 3581 3582 if (src_nents) { 3583 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3584 DMA_TO_DEVICE); 3585 if (!mapped_nents) { 3586 dev_err(ctx->dev, "unable to DMA map source\n"); 3587 return -ENOMEM; 3588 } 3589 } else { 3590 mapped_nents = 0; 3591 } 3592 3593 /* allocate space for base edesc and link tables */ 3594 edesc = qi_cache_zalloc(GFP_DMA | flags); 3595 if (!edesc) { 3596 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3597 return -ENOMEM; 3598 } 3599 3600 edesc->src_nents = src_nents; 3601 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3602 qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); 3603 sg_table = &edesc->sgt[0]; 3604 3605 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3606 DMA_BIDIRECTIONAL); 3607 if (ret) 3608 goto unmap_ctx; 3609 3610 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3611 if (ret) 3612 goto unmap_ctx; 3613 3614 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); 3615 3616 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3617 DMA_TO_DEVICE); 3618 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3619 dev_err(ctx->dev, "unable to map S/G table\n"); 3620 ret = -ENOMEM; 3621 goto unmap_ctx; 3622 } 3623 edesc->qm_sg_bytes = qm_sg_bytes; 3624 3625 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3626 dpaa2_fl_set_final(in_fle, true); 3627 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3628 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3629 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3630 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3631 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3632 dpaa2_fl_set_len(out_fle, digestsize); 3633 3634 req_ctx->flc = &ctx->flc[FINALIZE]; 3635 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3636 req_ctx->cbk = ahash_done_ctx_src; 3637 req_ctx->ctx = &req->base; 3638 req_ctx->edesc = edesc; 3639 3640 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3641 if (ret == -EINPROGRESS || 3642 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3643 return ret; 3644 3645 unmap_ctx: 3646 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3647 qi_cache_free(edesc); 3648 return ret; 3649 } 3650 3651 static int ahash_digest(struct ahash_request *req) 3652 { 3653 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3654 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3655 struct caam_hash_state *state = ahash_request_ctx(req); 3656 struct caam_request *req_ctx = &state->caam_req; 3657 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3658 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3659 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3660 GFP_KERNEL : GFP_ATOMIC; 3661 int digestsize = crypto_ahash_digestsize(ahash); 3662 int src_nents, mapped_nents; 3663 struct ahash_edesc *edesc; 3664 int ret = -ENOMEM; 3665 3666 state->buf_dma = 0; 3667 3668 src_nents = sg_nents_for_len(req->src, req->nbytes); 3669 if (src_nents < 0) { 3670 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3671 return src_nents; 3672 } 3673 3674 if (src_nents) { 3675 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3676 DMA_TO_DEVICE); 3677 if (!mapped_nents) { 3678 dev_err(ctx->dev, "unable to map source for DMA\n"); 3679 return ret; 3680 } 3681 } else { 3682 mapped_nents = 0; 3683 } 3684 3685 /* allocate space for base edesc and link tables */ 3686 edesc = qi_cache_zalloc(GFP_DMA | flags); 3687 if (!edesc) { 3688 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3689 return ret; 3690 } 3691 3692 edesc->src_nents = src_nents; 3693 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3694 3695 if (mapped_nents > 1) { 3696 int qm_sg_bytes; 3697 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3698 3699 qm_sg_bytes = mapped_nents * sizeof(*sg_table); 3700 sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); 3701 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3702 qm_sg_bytes, DMA_TO_DEVICE); 3703 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3704 dev_err(ctx->dev, "unable to map S/G table\n"); 3705 goto unmap; 3706 } 3707 edesc->qm_sg_bytes = qm_sg_bytes; 3708 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3709 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3710 } else { 3711 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3712 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3713 } 3714 3715 state->ctx_dma_len = digestsize; 3716 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3717 DMA_FROM_DEVICE); 3718 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3719 dev_err(ctx->dev, "unable to map ctx\n"); 3720 state->ctx_dma = 0; 3721 goto unmap; 3722 } 3723 3724 dpaa2_fl_set_final(in_fle, true); 3725 dpaa2_fl_set_len(in_fle, req->nbytes); 3726 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3727 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3728 dpaa2_fl_set_len(out_fle, digestsize); 3729 3730 req_ctx->flc = &ctx->flc[DIGEST]; 3731 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3732 req_ctx->cbk = ahash_done; 3733 req_ctx->ctx = &req->base; 3734 req_ctx->edesc = edesc; 3735 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3736 if (ret == -EINPROGRESS || 3737 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3738 return ret; 3739 3740 unmap: 3741 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3742 qi_cache_free(edesc); 3743 return ret; 3744 } 3745 3746 static int ahash_final_no_ctx(struct ahash_request *req) 3747 { 3748 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3749 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3750 struct caam_hash_state *state = ahash_request_ctx(req); 3751 struct caam_request *req_ctx = &state->caam_req; 3752 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3753 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3754 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3755 GFP_KERNEL : GFP_ATOMIC; 3756 u8 *buf = current_buf(state); 3757 int buflen = *current_buflen(state); 3758 int digestsize = crypto_ahash_digestsize(ahash); 3759 struct ahash_edesc *edesc; 3760 int ret = -ENOMEM; 3761 3762 /* allocate space for base edesc and link tables */ 3763 edesc = qi_cache_zalloc(GFP_DMA | flags); 3764 if (!edesc) 3765 return ret; 3766 3767 if (buflen) { 3768 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3769 DMA_TO_DEVICE); 3770 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3771 dev_err(ctx->dev, "unable to map src\n"); 3772 goto unmap; 3773 } 3774 } 3775 3776 state->ctx_dma_len = digestsize; 3777 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3778 DMA_FROM_DEVICE); 3779 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3780 dev_err(ctx->dev, "unable to map ctx\n"); 3781 state->ctx_dma = 0; 3782 goto unmap; 3783 } 3784 3785 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3786 dpaa2_fl_set_final(in_fle, true); 3787 /* 3788 * crypto engine requires the input entry to be present when 3789 * "frame list" FD is used. 3790 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3791 * in_fle zeroized (except for "Final" flag) is the best option. 3792 */ 3793 if (buflen) { 3794 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3795 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3796 dpaa2_fl_set_len(in_fle, buflen); 3797 } 3798 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3799 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3800 dpaa2_fl_set_len(out_fle, digestsize); 3801 3802 req_ctx->flc = &ctx->flc[DIGEST]; 3803 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3804 req_ctx->cbk = ahash_done; 3805 req_ctx->ctx = &req->base; 3806 req_ctx->edesc = edesc; 3807 3808 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3809 if (ret == -EINPROGRESS || 3810 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3811 return ret; 3812 3813 unmap: 3814 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3815 qi_cache_free(edesc); 3816 return ret; 3817 } 3818 3819 static int ahash_update_no_ctx(struct ahash_request *req) 3820 { 3821 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3822 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3823 struct caam_hash_state *state = ahash_request_ctx(req); 3824 struct caam_request *req_ctx = &state->caam_req; 3825 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3826 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3827 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3828 GFP_KERNEL : GFP_ATOMIC; 3829 u8 *buf = current_buf(state); 3830 int *buflen = current_buflen(state); 3831 u8 *next_buf = alt_buf(state); 3832 int *next_buflen = alt_buflen(state); 3833 int in_len = *buflen + req->nbytes, to_hash; 3834 int qm_sg_bytes, src_nents, mapped_nents; 3835 struct ahash_edesc *edesc; 3836 int ret = 0; 3837 3838 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3839 to_hash = in_len - *next_buflen; 3840 3841 if (to_hash) { 3842 struct dpaa2_sg_entry *sg_table; 3843 3844 src_nents = sg_nents_for_len(req->src, 3845 req->nbytes - *next_buflen); 3846 if (src_nents < 0) { 3847 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3848 return src_nents; 3849 } 3850 3851 if (src_nents) { 3852 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3853 DMA_TO_DEVICE); 3854 if (!mapped_nents) { 3855 dev_err(ctx->dev, "unable to DMA map source\n"); 3856 return -ENOMEM; 3857 } 3858 } else { 3859 mapped_nents = 0; 3860 } 3861 3862 /* allocate space for base edesc and link tables */ 3863 edesc = qi_cache_zalloc(GFP_DMA | flags); 3864 if (!edesc) { 3865 dma_unmap_sg(ctx->dev, req->src, src_nents, 3866 DMA_TO_DEVICE); 3867 return -ENOMEM; 3868 } 3869 3870 edesc->src_nents = src_nents; 3871 qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); 3872 sg_table = &edesc->sgt[0]; 3873 3874 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3875 if (ret) 3876 goto unmap_ctx; 3877 3878 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); 3879 3880 if (*next_buflen) 3881 scatterwalk_map_and_copy(next_buf, req->src, 3882 to_hash - *buflen, 3883 *next_buflen, 0); 3884 3885 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3886 qm_sg_bytes, DMA_TO_DEVICE); 3887 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3888 dev_err(ctx->dev, "unable to map S/G table\n"); 3889 ret = -ENOMEM; 3890 goto unmap_ctx; 3891 } 3892 edesc->qm_sg_bytes = qm_sg_bytes; 3893 3894 state->ctx_dma_len = ctx->ctx_len; 3895 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 3896 ctx->ctx_len, DMA_FROM_DEVICE); 3897 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3898 dev_err(ctx->dev, "unable to map ctx\n"); 3899 state->ctx_dma = 0; 3900 ret = -ENOMEM; 3901 goto unmap_ctx; 3902 } 3903 3904 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3905 dpaa2_fl_set_final(in_fle, true); 3906 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3907 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3908 dpaa2_fl_set_len(in_fle, to_hash); 3909 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3910 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3911 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3912 3913 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 3914 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 3915 req_ctx->cbk = ahash_done_ctx_dst; 3916 req_ctx->ctx = &req->base; 3917 req_ctx->edesc = edesc; 3918 3919 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3920 if (ret != -EINPROGRESS && 3921 !(ret == -EBUSY && 3922 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3923 goto unmap_ctx; 3924 3925 state->update = ahash_update_ctx; 3926 state->finup = ahash_finup_ctx; 3927 state->final = ahash_final_ctx; 3928 } else if (*next_buflen) { 3929 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3930 req->nbytes, 0); 3931 *buflen = *next_buflen; 3932 *next_buflen = 0; 3933 } 3934 3935 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3936 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); 3937 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 3938 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 3939 1); 3940 3941 return ret; 3942 unmap_ctx: 3943 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 3944 qi_cache_free(edesc); 3945 return ret; 3946 } 3947 3948 static int ahash_finup_no_ctx(struct ahash_request *req) 3949 { 3950 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3951 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3952 struct caam_hash_state *state = ahash_request_ctx(req); 3953 struct caam_request *req_ctx = &state->caam_req; 3954 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3955 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3956 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3957 GFP_KERNEL : GFP_ATOMIC; 3958 int buflen = *current_buflen(state); 3959 int qm_sg_bytes, src_nents, mapped_nents; 3960 int digestsize = crypto_ahash_digestsize(ahash); 3961 struct ahash_edesc *edesc; 3962 struct dpaa2_sg_entry *sg_table; 3963 int ret; 3964 3965 src_nents = sg_nents_for_len(req->src, req->nbytes); 3966 if (src_nents < 0) { 3967 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3968 return src_nents; 3969 } 3970 3971 if (src_nents) { 3972 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3973 DMA_TO_DEVICE); 3974 if (!mapped_nents) { 3975 dev_err(ctx->dev, "unable to DMA map source\n"); 3976 return -ENOMEM; 3977 } 3978 } else { 3979 mapped_nents = 0; 3980 } 3981 3982 /* allocate space for base edesc and link tables */ 3983 edesc = qi_cache_zalloc(GFP_DMA | flags); 3984 if (!edesc) { 3985 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3986 return -ENOMEM; 3987 } 3988 3989 edesc->src_nents = src_nents; 3990 qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); 3991 sg_table = &edesc->sgt[0]; 3992 3993 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3994 if (ret) 3995 goto unmap; 3996 3997 sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); 3998 3999 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4000 DMA_TO_DEVICE); 4001 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4002 dev_err(ctx->dev, "unable to map S/G table\n"); 4003 ret = -ENOMEM; 4004 goto unmap; 4005 } 4006 edesc->qm_sg_bytes = qm_sg_bytes; 4007 4008 state->ctx_dma_len = digestsize; 4009 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4010 DMA_FROM_DEVICE); 4011 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4012 dev_err(ctx->dev, "unable to map ctx\n"); 4013 state->ctx_dma = 0; 4014 ret = -ENOMEM; 4015 goto unmap; 4016 } 4017 4018 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4019 dpaa2_fl_set_final(in_fle, true); 4020 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4021 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4022 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4023 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4024 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4025 dpaa2_fl_set_len(out_fle, digestsize); 4026 4027 req_ctx->flc = &ctx->flc[DIGEST]; 4028 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4029 req_ctx->cbk = ahash_done; 4030 req_ctx->ctx = &req->base; 4031 req_ctx->edesc = edesc; 4032 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4033 if (ret != -EINPROGRESS && 4034 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4035 goto unmap; 4036 4037 return ret; 4038 unmap: 4039 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4040 qi_cache_free(edesc); 4041 return -ENOMEM; 4042 } 4043 4044 static int ahash_update_first(struct ahash_request *req) 4045 { 4046 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4047 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4048 struct caam_hash_state *state = ahash_request_ctx(req); 4049 struct caam_request *req_ctx = &state->caam_req; 4050 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4051 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4052 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4053 GFP_KERNEL : GFP_ATOMIC; 4054 u8 *next_buf = alt_buf(state); 4055 int *next_buflen = alt_buflen(state); 4056 int to_hash; 4057 int src_nents, mapped_nents; 4058 struct ahash_edesc *edesc; 4059 int ret = 0; 4060 4061 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4062 1); 4063 to_hash = req->nbytes - *next_buflen; 4064 4065 if (to_hash) { 4066 struct dpaa2_sg_entry *sg_table; 4067 4068 src_nents = sg_nents_for_len(req->src, 4069 req->nbytes - (*next_buflen)); 4070 if (src_nents < 0) { 4071 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4072 return src_nents; 4073 } 4074 4075 if (src_nents) { 4076 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4077 DMA_TO_DEVICE); 4078 if (!mapped_nents) { 4079 dev_err(ctx->dev, "unable to map source for DMA\n"); 4080 return -ENOMEM; 4081 } 4082 } else { 4083 mapped_nents = 0; 4084 } 4085 4086 /* allocate space for base edesc and link tables */ 4087 edesc = qi_cache_zalloc(GFP_DMA | flags); 4088 if (!edesc) { 4089 dma_unmap_sg(ctx->dev, req->src, src_nents, 4090 DMA_TO_DEVICE); 4091 return -ENOMEM; 4092 } 4093 4094 edesc->src_nents = src_nents; 4095 sg_table = &edesc->sgt[0]; 4096 4097 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4098 dpaa2_fl_set_final(in_fle, true); 4099 dpaa2_fl_set_len(in_fle, to_hash); 4100 4101 if (mapped_nents > 1) { 4102 int qm_sg_bytes; 4103 4104 sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); 4105 qm_sg_bytes = mapped_nents * sizeof(*sg_table); 4106 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4107 qm_sg_bytes, 4108 DMA_TO_DEVICE); 4109 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4110 dev_err(ctx->dev, "unable to map S/G table\n"); 4111 ret = -ENOMEM; 4112 goto unmap_ctx; 4113 } 4114 edesc->qm_sg_bytes = qm_sg_bytes; 4115 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4116 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4117 } else { 4118 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4119 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4120 } 4121 4122 if (*next_buflen) 4123 scatterwalk_map_and_copy(next_buf, req->src, to_hash, 4124 *next_buflen, 0); 4125 4126 state->ctx_dma_len = ctx->ctx_len; 4127 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4128 ctx->ctx_len, DMA_FROM_DEVICE); 4129 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4130 dev_err(ctx->dev, "unable to map ctx\n"); 4131 state->ctx_dma = 0; 4132 ret = -ENOMEM; 4133 goto unmap_ctx; 4134 } 4135 4136 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4137 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4138 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4139 4140 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4141 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4142 req_ctx->cbk = ahash_done_ctx_dst; 4143 req_ctx->ctx = &req->base; 4144 req_ctx->edesc = edesc; 4145 4146 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4147 if (ret != -EINPROGRESS && 4148 !(ret == -EBUSY && req->base.flags & 4149 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4150 goto unmap_ctx; 4151 4152 state->update = ahash_update_ctx; 4153 state->finup = ahash_finup_ctx; 4154 state->final = ahash_final_ctx; 4155 } else if (*next_buflen) { 4156 state->update = ahash_update_no_ctx; 4157 state->finup = ahash_finup_no_ctx; 4158 state->final = ahash_final_no_ctx; 4159 scatterwalk_map_and_copy(next_buf, req->src, 0, 4160 req->nbytes, 0); 4161 switch_buf(state); 4162 } 4163 4164 print_hex_dump_debug("next buf@" __stringify(__LINE__)": ", 4165 DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 4166 1); 4167 4168 return ret; 4169 unmap_ctx: 4170 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4171 qi_cache_free(edesc); 4172 return ret; 4173 } 4174 4175 static int ahash_finup_first(struct ahash_request *req) 4176 { 4177 return ahash_digest(req); 4178 } 4179 4180 static int ahash_init(struct ahash_request *req) 4181 { 4182 struct caam_hash_state *state = ahash_request_ctx(req); 4183 4184 state->update = ahash_update_first; 4185 state->finup = ahash_finup_first; 4186 state->final = ahash_final_no_ctx; 4187 4188 state->ctx_dma = 0; 4189 state->ctx_dma_len = 0; 4190 state->current_buf = 0; 4191 state->buf_dma = 0; 4192 state->buflen_0 = 0; 4193 state->buflen_1 = 0; 4194 4195 return 0; 4196 } 4197 4198 static int ahash_update(struct ahash_request *req) 4199 { 4200 struct caam_hash_state *state = ahash_request_ctx(req); 4201 4202 return state->update(req); 4203 } 4204 4205 static int ahash_finup(struct ahash_request *req) 4206 { 4207 struct caam_hash_state *state = ahash_request_ctx(req); 4208 4209 return state->finup(req); 4210 } 4211 4212 static int ahash_final(struct ahash_request *req) 4213 { 4214 struct caam_hash_state *state = ahash_request_ctx(req); 4215 4216 return state->final(req); 4217 } 4218 4219 static int ahash_export(struct ahash_request *req, void *out) 4220 { 4221 struct caam_hash_state *state = ahash_request_ctx(req); 4222 struct caam_export_state *export = out; 4223 int len; 4224 u8 *buf; 4225 4226 if (state->current_buf) { 4227 buf = state->buf_1; 4228 len = state->buflen_1; 4229 } else { 4230 buf = state->buf_0; 4231 len = state->buflen_0; 4232 } 4233 4234 memcpy(export->buf, buf, len); 4235 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4236 export->buflen = len; 4237 export->update = state->update; 4238 export->final = state->final; 4239 export->finup = state->finup; 4240 4241 return 0; 4242 } 4243 4244 static int ahash_import(struct ahash_request *req, const void *in) 4245 { 4246 struct caam_hash_state *state = ahash_request_ctx(req); 4247 const struct caam_export_state *export = in; 4248 4249 memset(state, 0, sizeof(*state)); 4250 memcpy(state->buf_0, export->buf, export->buflen); 4251 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4252 state->buflen_0 = export->buflen; 4253 state->update = export->update; 4254 state->final = export->final; 4255 state->finup = export->finup; 4256 4257 return 0; 4258 } 4259 4260 struct caam_hash_template { 4261 char name[CRYPTO_MAX_ALG_NAME]; 4262 char driver_name[CRYPTO_MAX_ALG_NAME]; 4263 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4264 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4265 unsigned int blocksize; 4266 struct ahash_alg template_ahash; 4267 u32 alg_type; 4268 }; 4269 4270 /* ahash descriptors */ 4271 static struct caam_hash_template driver_hash[] = { 4272 { 4273 .name = "sha1", 4274 .driver_name = "sha1-caam-qi2", 4275 .hmac_name = "hmac(sha1)", 4276 .hmac_driver_name = "hmac-sha1-caam-qi2", 4277 .blocksize = SHA1_BLOCK_SIZE, 4278 .template_ahash = { 4279 .init = ahash_init, 4280 .update = ahash_update, 4281 .final = ahash_final, 4282 .finup = ahash_finup, 4283 .digest = ahash_digest, 4284 .export = ahash_export, 4285 .import = ahash_import, 4286 .setkey = ahash_setkey, 4287 .halg = { 4288 .digestsize = SHA1_DIGEST_SIZE, 4289 .statesize = sizeof(struct caam_export_state), 4290 }, 4291 }, 4292 .alg_type = OP_ALG_ALGSEL_SHA1, 4293 }, { 4294 .name = "sha224", 4295 .driver_name = "sha224-caam-qi2", 4296 .hmac_name = "hmac(sha224)", 4297 .hmac_driver_name = "hmac-sha224-caam-qi2", 4298 .blocksize = SHA224_BLOCK_SIZE, 4299 .template_ahash = { 4300 .init = ahash_init, 4301 .update = ahash_update, 4302 .final = ahash_final, 4303 .finup = ahash_finup, 4304 .digest = ahash_digest, 4305 .export = ahash_export, 4306 .import = ahash_import, 4307 .setkey = ahash_setkey, 4308 .halg = { 4309 .digestsize = SHA224_DIGEST_SIZE, 4310 .statesize = sizeof(struct caam_export_state), 4311 }, 4312 }, 4313 .alg_type = OP_ALG_ALGSEL_SHA224, 4314 }, { 4315 .name = "sha256", 4316 .driver_name = "sha256-caam-qi2", 4317 .hmac_name = "hmac(sha256)", 4318 .hmac_driver_name = "hmac-sha256-caam-qi2", 4319 .blocksize = SHA256_BLOCK_SIZE, 4320 .template_ahash = { 4321 .init = ahash_init, 4322 .update = ahash_update, 4323 .final = ahash_final, 4324 .finup = ahash_finup, 4325 .digest = ahash_digest, 4326 .export = ahash_export, 4327 .import = ahash_import, 4328 .setkey = ahash_setkey, 4329 .halg = { 4330 .digestsize = SHA256_DIGEST_SIZE, 4331 .statesize = sizeof(struct caam_export_state), 4332 }, 4333 }, 4334 .alg_type = OP_ALG_ALGSEL_SHA256, 4335 }, { 4336 .name = "sha384", 4337 .driver_name = "sha384-caam-qi2", 4338 .hmac_name = "hmac(sha384)", 4339 .hmac_driver_name = "hmac-sha384-caam-qi2", 4340 .blocksize = SHA384_BLOCK_SIZE, 4341 .template_ahash = { 4342 .init = ahash_init, 4343 .update = ahash_update, 4344 .final = ahash_final, 4345 .finup = ahash_finup, 4346 .digest = ahash_digest, 4347 .export = ahash_export, 4348 .import = ahash_import, 4349 .setkey = ahash_setkey, 4350 .halg = { 4351 .digestsize = SHA384_DIGEST_SIZE, 4352 .statesize = sizeof(struct caam_export_state), 4353 }, 4354 }, 4355 .alg_type = OP_ALG_ALGSEL_SHA384, 4356 }, { 4357 .name = "sha512", 4358 .driver_name = "sha512-caam-qi2", 4359 .hmac_name = "hmac(sha512)", 4360 .hmac_driver_name = "hmac-sha512-caam-qi2", 4361 .blocksize = SHA512_BLOCK_SIZE, 4362 .template_ahash = { 4363 .init = ahash_init, 4364 .update = ahash_update, 4365 .final = ahash_final, 4366 .finup = ahash_finup, 4367 .digest = ahash_digest, 4368 .export = ahash_export, 4369 .import = ahash_import, 4370 .setkey = ahash_setkey, 4371 .halg = { 4372 .digestsize = SHA512_DIGEST_SIZE, 4373 .statesize = sizeof(struct caam_export_state), 4374 }, 4375 }, 4376 .alg_type = OP_ALG_ALGSEL_SHA512, 4377 }, { 4378 .name = "md5", 4379 .driver_name = "md5-caam-qi2", 4380 .hmac_name = "hmac(md5)", 4381 .hmac_driver_name = "hmac-md5-caam-qi2", 4382 .blocksize = MD5_BLOCK_WORDS * 4, 4383 .template_ahash = { 4384 .init = ahash_init, 4385 .update = ahash_update, 4386 .final = ahash_final, 4387 .finup = ahash_finup, 4388 .digest = ahash_digest, 4389 .export = ahash_export, 4390 .import = ahash_import, 4391 .setkey = ahash_setkey, 4392 .halg = { 4393 .digestsize = MD5_DIGEST_SIZE, 4394 .statesize = sizeof(struct caam_export_state), 4395 }, 4396 }, 4397 .alg_type = OP_ALG_ALGSEL_MD5, 4398 } 4399 }; 4400 4401 struct caam_hash_alg { 4402 struct list_head entry; 4403 struct device *dev; 4404 int alg_type; 4405 struct ahash_alg ahash_alg; 4406 }; 4407 4408 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4409 { 4410 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4411 struct crypto_alg *base = tfm->__crt_alg; 4412 struct hash_alg_common *halg = 4413 container_of(base, struct hash_alg_common, base); 4414 struct ahash_alg *alg = 4415 container_of(halg, struct ahash_alg, halg); 4416 struct caam_hash_alg *caam_hash = 4417 container_of(alg, struct caam_hash_alg, ahash_alg); 4418 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4419 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4420 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4421 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4422 HASH_MSG_LEN + 32, 4423 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4424 HASH_MSG_LEN + 64, 4425 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4426 dma_addr_t dma_addr; 4427 int i; 4428 4429 ctx->dev = caam_hash->dev; 4430 4431 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4432 DMA_BIDIRECTIONAL, 4433 DMA_ATTR_SKIP_CPU_SYNC); 4434 if (dma_mapping_error(ctx->dev, dma_addr)) { 4435 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4436 return -ENOMEM; 4437 } 4438 4439 for (i = 0; i < HASH_NUM_OP; i++) 4440 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4441 4442 /* copy descriptor header template value */ 4443 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4444 4445 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4446 OP_ALG_ALGSEL_SUBMASK) >> 4447 OP_ALG_ALGSEL_SHIFT]; 4448 4449 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4450 sizeof(struct caam_hash_state)); 4451 4452 return ahash_set_sh_desc(ahash); 4453 } 4454 4455 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4456 { 4457 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4458 4459 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4460 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4461 } 4462 4463 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4464 struct caam_hash_template *template, bool keyed) 4465 { 4466 struct caam_hash_alg *t_alg; 4467 struct ahash_alg *halg; 4468 struct crypto_alg *alg; 4469 4470 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4471 if (!t_alg) 4472 return ERR_PTR(-ENOMEM); 4473 4474 t_alg->ahash_alg = template->template_ahash; 4475 halg = &t_alg->ahash_alg; 4476 alg = &halg->halg.base; 4477 4478 if (keyed) { 4479 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4480 template->hmac_name); 4481 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4482 template->hmac_driver_name); 4483 } else { 4484 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4485 template->name); 4486 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4487 template->driver_name); 4488 t_alg->ahash_alg.setkey = NULL; 4489 } 4490 alg->cra_module = THIS_MODULE; 4491 alg->cra_init = caam_hash_cra_init; 4492 alg->cra_exit = caam_hash_cra_exit; 4493 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4494 alg->cra_priority = CAAM_CRA_PRIORITY; 4495 alg->cra_blocksize = template->blocksize; 4496 alg->cra_alignmask = 0; 4497 alg->cra_flags = CRYPTO_ALG_ASYNC; 4498 4499 t_alg->alg_type = template->alg_type; 4500 t_alg->dev = dev; 4501 4502 return t_alg; 4503 } 4504 4505 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4506 { 4507 struct dpaa2_caam_priv_per_cpu *ppriv; 4508 4509 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4510 napi_schedule_irqoff(&ppriv->napi); 4511 } 4512 4513 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4514 { 4515 struct device *dev = priv->dev; 4516 struct dpaa2_io_notification_ctx *nctx; 4517 struct dpaa2_caam_priv_per_cpu *ppriv; 4518 int err, i = 0, cpu; 4519 4520 for_each_online_cpu(cpu) { 4521 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4522 ppriv->priv = priv; 4523 nctx = &ppriv->nctx; 4524 nctx->is_cdan = 0; 4525 nctx->id = ppriv->rsp_fqid; 4526 nctx->desired_cpu = cpu; 4527 nctx->cb = dpaa2_caam_fqdan_cb; 4528 4529 /* Register notification callbacks */ 4530 ppriv->dpio = dpaa2_io_service_select(cpu); 4531 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4532 if (unlikely(err)) { 4533 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4534 nctx->cb = NULL; 4535 /* 4536 * If no affine DPIO for this core, there's probably 4537 * none available for next cores either. Signal we want 4538 * to retry later, in case the DPIO devices weren't 4539 * probed yet. 4540 */ 4541 err = -EPROBE_DEFER; 4542 goto err; 4543 } 4544 4545 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4546 dev); 4547 if (unlikely(!ppriv->store)) { 4548 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4549 err = -ENOMEM; 4550 goto err; 4551 } 4552 4553 if (++i == priv->num_pairs) 4554 break; 4555 } 4556 4557 return 0; 4558 4559 err: 4560 for_each_online_cpu(cpu) { 4561 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4562 if (!ppriv->nctx.cb) 4563 break; 4564 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4565 } 4566 4567 for_each_online_cpu(cpu) { 4568 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4569 if (!ppriv->store) 4570 break; 4571 dpaa2_io_store_destroy(ppriv->store); 4572 } 4573 4574 return err; 4575 } 4576 4577 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4578 { 4579 struct dpaa2_caam_priv_per_cpu *ppriv; 4580 int i = 0, cpu; 4581 4582 for_each_online_cpu(cpu) { 4583 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4584 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4585 priv->dev); 4586 dpaa2_io_store_destroy(ppriv->store); 4587 4588 if (++i == priv->num_pairs) 4589 return; 4590 } 4591 } 4592 4593 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4594 { 4595 struct dpseci_rx_queue_cfg rx_queue_cfg; 4596 struct device *dev = priv->dev; 4597 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4598 struct dpaa2_caam_priv_per_cpu *ppriv; 4599 int err = 0, i = 0, cpu; 4600 4601 /* Configure Rx queues */ 4602 for_each_online_cpu(cpu) { 4603 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4604 4605 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4606 DPSECI_QUEUE_OPT_USER_CTX; 4607 rx_queue_cfg.order_preservation_en = 0; 4608 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4609 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4610 /* 4611 * Rx priority (WQ) doesn't really matter, since we use 4612 * pull mode, i.e. volatile dequeues from specific FQs 4613 */ 4614 rx_queue_cfg.dest_cfg.priority = 0; 4615 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4616 4617 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4618 &rx_queue_cfg); 4619 if (err) { 4620 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4621 err); 4622 return err; 4623 } 4624 4625 if (++i == priv->num_pairs) 4626 break; 4627 } 4628 4629 return err; 4630 } 4631 4632 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4633 { 4634 struct device *dev = priv->dev; 4635 4636 if (!priv->cscn_mem) 4637 return; 4638 4639 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4640 kfree(priv->cscn_mem); 4641 } 4642 4643 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4644 { 4645 struct device *dev = priv->dev; 4646 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4647 4648 dpaa2_dpseci_congestion_free(priv); 4649 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4650 } 4651 4652 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4653 const struct dpaa2_fd *fd) 4654 { 4655 struct caam_request *req; 4656 u32 fd_err; 4657 4658 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4659 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4660 return; 4661 } 4662 4663 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4664 if (unlikely(fd_err)) 4665 dev_err(priv->dev, "FD error: %08x\n", fd_err); 4666 4667 /* 4668 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4669 * in FD[ERR] or FD[FRC]. 4670 */ 4671 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4672 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4673 DMA_BIDIRECTIONAL); 4674 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4675 } 4676 4677 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4678 { 4679 int err; 4680 4681 /* Retry while portal is busy */ 4682 do { 4683 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4684 ppriv->store); 4685 } while (err == -EBUSY); 4686 4687 if (unlikely(err)) 4688 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4689 4690 return err; 4691 } 4692 4693 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4694 { 4695 struct dpaa2_dq *dq; 4696 int cleaned = 0, is_last; 4697 4698 do { 4699 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4700 if (unlikely(!dq)) { 4701 if (unlikely(!is_last)) { 4702 dev_dbg(ppriv->priv->dev, 4703 "FQ %d returned no valid frames\n", 4704 ppriv->rsp_fqid); 4705 /* 4706 * MUST retry until we get some sort of 4707 * valid response token (be it "empty dequeue" 4708 * or a valid frame). 4709 */ 4710 continue; 4711 } 4712 break; 4713 } 4714 4715 /* Process FD */ 4716 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4717 cleaned++; 4718 } while (!is_last); 4719 4720 return cleaned; 4721 } 4722 4723 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4724 { 4725 struct dpaa2_caam_priv_per_cpu *ppriv; 4726 struct dpaa2_caam_priv *priv; 4727 int err, cleaned = 0, store_cleaned; 4728 4729 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4730 priv = ppriv->priv; 4731 4732 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4733 return 0; 4734 4735 do { 4736 store_cleaned = dpaa2_caam_store_consume(ppriv); 4737 cleaned += store_cleaned; 4738 4739 if (store_cleaned == 0 || 4740 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4741 break; 4742 4743 /* Try to dequeue some more */ 4744 err = dpaa2_caam_pull_fq(ppriv); 4745 if (unlikely(err)) 4746 break; 4747 } while (1); 4748 4749 if (cleaned < budget) { 4750 napi_complete_done(napi, cleaned); 4751 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4752 if (unlikely(err)) 4753 dev_err(priv->dev, "Notification rearm failed: %d\n", 4754 err); 4755 } 4756 4757 return cleaned; 4758 } 4759 4760 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4761 u16 token) 4762 { 4763 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4764 struct device *dev = priv->dev; 4765 int err; 4766 4767 /* 4768 * Congestion group feature supported starting with DPSECI API v5.1 4769 * and only when object has been created with this capability. 4770 */ 4771 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4772 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4773 return 0; 4774 4775 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4776 GFP_KERNEL | GFP_DMA); 4777 if (!priv->cscn_mem) 4778 return -ENOMEM; 4779 4780 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4781 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4782 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4783 if (dma_mapping_error(dev, priv->cscn_dma)) { 4784 dev_err(dev, "Error mapping CSCN memory area\n"); 4785 err = -ENOMEM; 4786 goto err_dma_map; 4787 } 4788 4789 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4790 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4791 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4792 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4793 cong_notif_cfg.message_iova = priv->cscn_dma; 4794 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4795 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4796 DPSECI_CGN_MODE_COHERENT_WRITE; 4797 4798 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4799 &cong_notif_cfg); 4800 if (err) { 4801 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4802 goto err_set_cong; 4803 } 4804 4805 return 0; 4806 4807 err_set_cong: 4808 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4809 err_dma_map: 4810 kfree(priv->cscn_mem); 4811 4812 return err; 4813 } 4814 4815 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4816 { 4817 struct device *dev = &ls_dev->dev; 4818 struct dpaa2_caam_priv *priv; 4819 struct dpaa2_caam_priv_per_cpu *ppriv; 4820 int err, cpu; 4821 u8 i; 4822 4823 priv = dev_get_drvdata(dev); 4824 4825 priv->dev = dev; 4826 priv->dpsec_id = ls_dev->obj_desc.id; 4827 4828 /* Get a handle for the DPSECI this interface is associate with */ 4829 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4830 if (err) { 4831 dev_err(dev, "dpseci_open() failed: %d\n", err); 4832 goto err_open; 4833 } 4834 4835 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4836 &priv->minor_ver); 4837 if (err) { 4838 dev_err(dev, "dpseci_get_api_version() failed\n"); 4839 goto err_get_vers; 4840 } 4841 4842 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 4843 4844 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 4845 &priv->dpseci_attr); 4846 if (err) { 4847 dev_err(dev, "dpseci_get_attributes() failed\n"); 4848 goto err_get_vers; 4849 } 4850 4851 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 4852 &priv->sec_attr); 4853 if (err) { 4854 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 4855 goto err_get_vers; 4856 } 4857 4858 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 4859 if (err) { 4860 dev_err(dev, "setup_congestion() failed\n"); 4861 goto err_get_vers; 4862 } 4863 4864 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 4865 priv->dpseci_attr.num_tx_queues); 4866 if (priv->num_pairs > num_online_cpus()) { 4867 dev_warn(dev, "%d queues won't be used\n", 4868 priv->num_pairs - num_online_cpus()); 4869 priv->num_pairs = num_online_cpus(); 4870 } 4871 4872 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 4873 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4874 &priv->rx_queue_attr[i]); 4875 if (err) { 4876 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 4877 goto err_get_rx_queue; 4878 } 4879 } 4880 4881 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 4882 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4883 &priv->tx_queue_attr[i]); 4884 if (err) { 4885 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 4886 goto err_get_rx_queue; 4887 } 4888 } 4889 4890 i = 0; 4891 for_each_online_cpu(cpu) { 4892 u8 j; 4893 4894 j = i % priv->num_pairs; 4895 4896 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4897 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 4898 4899 /* 4900 * Allow all cores to enqueue, while only some of them 4901 * will take part in dequeuing. 4902 */ 4903 if (++i > priv->num_pairs) 4904 continue; 4905 4906 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 4907 ppriv->prio = j; 4908 4909 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 4910 priv->rx_queue_attr[j].fqid, 4911 priv->tx_queue_attr[j].fqid); 4912 4913 ppriv->net_dev.dev = *dev; 4914 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 4915 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 4916 DPAA2_CAAM_NAPI_WEIGHT); 4917 } 4918 4919 return 0; 4920 4921 err_get_rx_queue: 4922 dpaa2_dpseci_congestion_free(priv); 4923 err_get_vers: 4924 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4925 err_open: 4926 return err; 4927 } 4928 4929 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 4930 { 4931 struct device *dev = priv->dev; 4932 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4933 struct dpaa2_caam_priv_per_cpu *ppriv; 4934 int i; 4935 4936 for (i = 0; i < priv->num_pairs; i++) { 4937 ppriv = per_cpu_ptr(priv->ppriv, i); 4938 napi_enable(&ppriv->napi); 4939 } 4940 4941 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 4942 } 4943 4944 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 4945 { 4946 struct device *dev = priv->dev; 4947 struct dpaa2_caam_priv_per_cpu *ppriv; 4948 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4949 int i, err = 0, enabled; 4950 4951 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 4952 if (err) { 4953 dev_err(dev, "dpseci_disable() failed\n"); 4954 return err; 4955 } 4956 4957 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 4958 if (err) { 4959 dev_err(dev, "dpseci_is_enabled() failed\n"); 4960 return err; 4961 } 4962 4963 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 4964 4965 for (i = 0; i < priv->num_pairs; i++) { 4966 ppriv = per_cpu_ptr(priv->ppriv, i); 4967 napi_disable(&ppriv->napi); 4968 netif_napi_del(&ppriv->napi); 4969 } 4970 4971 return 0; 4972 } 4973 4974 static struct list_head hash_list; 4975 4976 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 4977 { 4978 struct device *dev; 4979 struct dpaa2_caam_priv *priv; 4980 int i, err = 0; 4981 bool registered = false; 4982 4983 /* 4984 * There is no way to get CAAM endianness - there is no direct register 4985 * space access and MC f/w does not provide this attribute. 4986 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 4987 * property. 4988 */ 4989 caam_little_end = true; 4990 4991 caam_imx = false; 4992 4993 dev = &dpseci_dev->dev; 4994 4995 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 4996 if (!priv) 4997 return -ENOMEM; 4998 4999 dev_set_drvdata(dev, priv); 5000 5001 priv->domain = iommu_get_domain_for_dev(dev); 5002 5003 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5004 0, SLAB_CACHE_DMA, NULL); 5005 if (!qi_cache) { 5006 dev_err(dev, "Can't allocate SEC cache\n"); 5007 return -ENOMEM; 5008 } 5009 5010 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5011 if (err) { 5012 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5013 goto err_dma_mask; 5014 } 5015 5016 /* Obtain a MC portal */ 5017 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5018 if (err) { 5019 if (err == -ENXIO) 5020 err = -EPROBE_DEFER; 5021 else 5022 dev_err(dev, "MC portal allocation failed\n"); 5023 5024 goto err_dma_mask; 5025 } 5026 5027 priv->ppriv = alloc_percpu(*priv->ppriv); 5028 if (!priv->ppriv) { 5029 dev_err(dev, "alloc_percpu() failed\n"); 5030 err = -ENOMEM; 5031 goto err_alloc_ppriv; 5032 } 5033 5034 /* DPSECI initialization */ 5035 err = dpaa2_dpseci_setup(dpseci_dev); 5036 if (err) { 5037 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5038 goto err_dpseci_setup; 5039 } 5040 5041 /* DPIO */ 5042 err = dpaa2_dpseci_dpio_setup(priv); 5043 if (err) { 5044 if (err != -EPROBE_DEFER) 5045 dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); 5046 goto err_dpio_setup; 5047 } 5048 5049 /* DPSECI binding to DPIO */ 5050 err = dpaa2_dpseci_bind(priv); 5051 if (err) { 5052 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5053 goto err_bind; 5054 } 5055 5056 /* DPSECI enable */ 5057 err = dpaa2_dpseci_enable(priv); 5058 if (err) { 5059 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5060 goto err_bind; 5061 } 5062 5063 /* register crypto algorithms the device supports */ 5064 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5065 struct caam_skcipher_alg *t_alg = driver_algs + i; 5066 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5067 5068 /* Skip DES algorithms if not supported by device */ 5069 if (!priv->sec_attr.des_acc_num && 5070 (alg_sel == OP_ALG_ALGSEL_3DES || 5071 alg_sel == OP_ALG_ALGSEL_DES)) 5072 continue; 5073 5074 /* Skip AES algorithms if not supported by device */ 5075 if (!priv->sec_attr.aes_acc_num && 5076 alg_sel == OP_ALG_ALGSEL_AES) 5077 continue; 5078 5079 /* Skip CHACHA20 algorithms if not supported by device */ 5080 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5081 !priv->sec_attr.ccha_acc_num) 5082 continue; 5083 5084 t_alg->caam.dev = dev; 5085 caam_skcipher_alg_init(t_alg); 5086 5087 err = crypto_register_skcipher(&t_alg->skcipher); 5088 if (err) { 5089 dev_warn(dev, "%s alg registration failed: %d\n", 5090 t_alg->skcipher.base.cra_driver_name, err); 5091 continue; 5092 } 5093 5094 t_alg->registered = true; 5095 registered = true; 5096 } 5097 5098 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5099 struct caam_aead_alg *t_alg = driver_aeads + i; 5100 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5101 OP_ALG_ALGSEL_MASK; 5102 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5103 OP_ALG_ALGSEL_MASK; 5104 5105 /* Skip DES algorithms if not supported by device */ 5106 if (!priv->sec_attr.des_acc_num && 5107 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5108 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5109 continue; 5110 5111 /* Skip AES algorithms if not supported by device */ 5112 if (!priv->sec_attr.aes_acc_num && 5113 c1_alg_sel == OP_ALG_ALGSEL_AES) 5114 continue; 5115 5116 /* Skip CHACHA20 algorithms if not supported by device */ 5117 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5118 !priv->sec_attr.ccha_acc_num) 5119 continue; 5120 5121 /* Skip POLY1305 algorithms if not supported by device */ 5122 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5123 !priv->sec_attr.ptha_acc_num) 5124 continue; 5125 5126 /* 5127 * Skip algorithms requiring message digests 5128 * if MD not supported by device. 5129 */ 5130 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5131 !priv->sec_attr.md_acc_num) 5132 continue; 5133 5134 t_alg->caam.dev = dev; 5135 caam_aead_alg_init(t_alg); 5136 5137 err = crypto_register_aead(&t_alg->aead); 5138 if (err) { 5139 dev_warn(dev, "%s alg registration failed: %d\n", 5140 t_alg->aead.base.cra_driver_name, err); 5141 continue; 5142 } 5143 5144 t_alg->registered = true; 5145 registered = true; 5146 } 5147 if (registered) 5148 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5149 5150 /* register hash algorithms the device supports */ 5151 INIT_LIST_HEAD(&hash_list); 5152 5153 /* 5154 * Skip registration of any hashing algorithms if MD block 5155 * is not present. 5156 */ 5157 if (!priv->sec_attr.md_acc_num) 5158 return 0; 5159 5160 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5161 struct caam_hash_alg *t_alg; 5162 struct caam_hash_template *alg = driver_hash + i; 5163 5164 /* register hmac version */ 5165 t_alg = caam_hash_alloc(dev, alg, true); 5166 if (IS_ERR(t_alg)) { 5167 err = PTR_ERR(t_alg); 5168 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5169 alg->driver_name, err); 5170 continue; 5171 } 5172 5173 err = crypto_register_ahash(&t_alg->ahash_alg); 5174 if (err) { 5175 dev_warn(dev, "%s alg registration failed: %d\n", 5176 t_alg->ahash_alg.halg.base.cra_driver_name, 5177 err); 5178 kfree(t_alg); 5179 } else { 5180 list_add_tail(&t_alg->entry, &hash_list); 5181 } 5182 5183 /* register unkeyed version */ 5184 t_alg = caam_hash_alloc(dev, alg, false); 5185 if (IS_ERR(t_alg)) { 5186 err = PTR_ERR(t_alg); 5187 dev_warn(dev, "%s alg allocation failed: %d\n", 5188 alg->driver_name, err); 5189 continue; 5190 } 5191 5192 err = crypto_register_ahash(&t_alg->ahash_alg); 5193 if (err) { 5194 dev_warn(dev, "%s alg registration failed: %d\n", 5195 t_alg->ahash_alg.halg.base.cra_driver_name, 5196 err); 5197 kfree(t_alg); 5198 } else { 5199 list_add_tail(&t_alg->entry, &hash_list); 5200 } 5201 } 5202 if (!list_empty(&hash_list)) 5203 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5204 5205 return err; 5206 5207 err_bind: 5208 dpaa2_dpseci_dpio_free(priv); 5209 err_dpio_setup: 5210 dpaa2_dpseci_free(priv); 5211 err_dpseci_setup: 5212 free_percpu(priv->ppriv); 5213 err_alloc_ppriv: 5214 fsl_mc_portal_free(priv->mc_io); 5215 err_dma_mask: 5216 kmem_cache_destroy(qi_cache); 5217 5218 return err; 5219 } 5220 5221 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5222 { 5223 struct device *dev; 5224 struct dpaa2_caam_priv *priv; 5225 int i; 5226 5227 dev = &ls_dev->dev; 5228 priv = dev_get_drvdata(dev); 5229 5230 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5231 struct caam_aead_alg *t_alg = driver_aeads + i; 5232 5233 if (t_alg->registered) 5234 crypto_unregister_aead(&t_alg->aead); 5235 } 5236 5237 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5238 struct caam_skcipher_alg *t_alg = driver_algs + i; 5239 5240 if (t_alg->registered) 5241 crypto_unregister_skcipher(&t_alg->skcipher); 5242 } 5243 5244 if (hash_list.next) { 5245 struct caam_hash_alg *t_hash_alg, *p; 5246 5247 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5248 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5249 list_del(&t_hash_alg->entry); 5250 kfree(t_hash_alg); 5251 } 5252 } 5253 5254 dpaa2_dpseci_disable(priv); 5255 dpaa2_dpseci_dpio_free(priv); 5256 dpaa2_dpseci_free(priv); 5257 free_percpu(priv->ppriv); 5258 fsl_mc_portal_free(priv->mc_io); 5259 kmem_cache_destroy(qi_cache); 5260 5261 return 0; 5262 } 5263 5264 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5265 { 5266 struct dpaa2_fd fd; 5267 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5268 struct dpaa2_caam_priv_per_cpu *ppriv; 5269 int err = 0, i; 5270 5271 if (IS_ERR(req)) 5272 return PTR_ERR(req); 5273 5274 if (priv->cscn_mem) { 5275 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5276 DPAA2_CSCN_SIZE, 5277 DMA_FROM_DEVICE); 5278 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5279 dev_dbg_ratelimited(dev, "Dropping request\n"); 5280 return -EBUSY; 5281 } 5282 } 5283 5284 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5285 5286 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5287 DMA_BIDIRECTIONAL); 5288 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5289 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5290 goto err_out; 5291 } 5292 5293 memset(&fd, 0, sizeof(fd)); 5294 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5295 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5296 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5297 dpaa2_fd_set_flc(&fd, req->flc_dma); 5298 5299 ppriv = this_cpu_ptr(priv->ppriv); 5300 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5301 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5302 &fd); 5303 if (err != -EBUSY) 5304 break; 5305 5306 cpu_relax(); 5307 } 5308 5309 if (unlikely(err)) { 5310 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5311 goto err_out; 5312 } 5313 5314 return -EINPROGRESS; 5315 5316 err_out: 5317 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5318 DMA_BIDIRECTIONAL); 5319 return -EIO; 5320 } 5321 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5322 5323 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5324 { 5325 .vendor = FSL_MC_VENDOR_FREESCALE, 5326 .obj_type = "dpseci", 5327 }, 5328 { .vendor = 0x0 } 5329 }; 5330 5331 static struct fsl_mc_driver dpaa2_caam_driver = { 5332 .driver = { 5333 .name = KBUILD_MODNAME, 5334 .owner = THIS_MODULE, 5335 }, 5336 .probe = dpaa2_caam_probe, 5337 .remove = dpaa2_caam_remove, 5338 .match_id_table = dpaa2_caam_match_id_table 5339 }; 5340 5341 MODULE_LICENSE("Dual BSD/GPL"); 5342 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5343 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5344 5345 module_fsl_mc_driver(dpaa2_caam_driver); 5346