1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/fsl/mc.h> 20 #include <soc/fsl/dpaa2-io.h> 21 #include <soc/fsl/dpaa2-fd.h> 22 23 #define CAAM_CRA_PRIORITY 2000 24 25 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 26 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 27 SHA512_DIGEST_SIZE * 2) 28 29 /* 30 * This is a a cache of buffers, from which the users of CAAM QI driver 31 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 32 * NOTE: A more elegant solution would be to have some headroom in the frames 33 * being processed. This can be added by the dpaa2-eth driver. This would 34 * pose a problem for userspace application processing which cannot 35 * know of this limitation. So for now, this will work. 36 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 37 */ 38 static struct kmem_cache *qi_cache; 39 40 struct caam_alg_entry { 41 struct device *dev; 42 int class1_alg_type; 43 int class2_alg_type; 44 bool rfc3686; 45 bool geniv; 46 bool nodkp; 47 }; 48 49 struct caam_aead_alg { 50 struct aead_alg aead; 51 struct caam_alg_entry caam; 52 bool registered; 53 }; 54 55 struct caam_skcipher_alg { 56 struct skcipher_alg skcipher; 57 struct caam_alg_entry caam; 58 bool registered; 59 }; 60 61 /** 62 * caam_ctx - per-session context 63 * @flc: Flow Contexts array 64 * @key: [authentication key], encryption key 65 * @flc_dma: I/O virtual addresses of the Flow Contexts 66 * @key_dma: I/O virtual address of the key 67 * @dir: DMA direction for mapping key and Flow Contexts 68 * @dev: dpseci device 69 * @adata: authentication algorithm details 70 * @cdata: encryption algorithm details 71 * @authsize: authentication tag (a.k.a. ICV / MAC) size 72 */ 73 struct caam_ctx { 74 struct caam_flc flc[NUM_OP]; 75 u8 key[CAAM_MAX_KEY_SIZE]; 76 dma_addr_t flc_dma[NUM_OP]; 77 dma_addr_t key_dma; 78 enum dma_data_direction dir; 79 struct device *dev; 80 struct alginfo adata; 81 struct alginfo cdata; 82 unsigned int authsize; 83 }; 84 85 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 86 dma_addr_t iova_addr) 87 { 88 phys_addr_t phys_addr; 89 90 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 91 iova_addr; 92 93 return phys_to_virt(phys_addr); 94 } 95 96 /* 97 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 98 * 99 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 100 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 101 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 102 * hosting 16 SG entries. 103 * 104 * @flags - flags that would be used for the equivalent kmalloc(..) call 105 * 106 * Returns a pointer to a retrieved buffer on success or NULL on failure. 107 */ 108 static inline void *qi_cache_zalloc(gfp_t flags) 109 { 110 return kmem_cache_zalloc(qi_cache, flags); 111 } 112 113 /* 114 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 115 * 116 * @obj - buffer previously allocated by qi_cache_zalloc 117 * 118 * No checking is being done, the call is a passthrough call to 119 * kmem_cache_free(...) 120 */ 121 static inline void qi_cache_free(void *obj) 122 { 123 kmem_cache_free(qi_cache, obj); 124 } 125 126 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 127 { 128 switch (crypto_tfm_alg_type(areq->tfm)) { 129 case CRYPTO_ALG_TYPE_SKCIPHER: 130 return skcipher_request_ctx(skcipher_request_cast(areq)); 131 case CRYPTO_ALG_TYPE_AEAD: 132 return aead_request_ctx(container_of(areq, struct aead_request, 133 base)); 134 case CRYPTO_ALG_TYPE_AHASH: 135 return ahash_request_ctx(ahash_request_cast(areq)); 136 default: 137 return ERR_PTR(-EINVAL); 138 } 139 } 140 141 static void caam_unmap(struct device *dev, struct scatterlist *src, 142 struct scatterlist *dst, int src_nents, 143 int dst_nents, dma_addr_t iv_dma, int ivsize, 144 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 145 int qm_sg_bytes) 146 { 147 if (dst != src) { 148 if (src_nents) 149 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 150 if (dst_nents) 151 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 152 } else { 153 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 154 } 155 156 if (iv_dma) 157 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 158 159 if (qm_sg_bytes) 160 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 161 } 162 163 static int aead_set_sh_desc(struct crypto_aead *aead) 164 { 165 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 166 typeof(*alg), aead); 167 struct caam_ctx *ctx = crypto_aead_ctx(aead); 168 unsigned int ivsize = crypto_aead_ivsize(aead); 169 struct device *dev = ctx->dev; 170 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 171 struct caam_flc *flc; 172 u32 *desc; 173 u32 ctx1_iv_off = 0; 174 u32 *nonce = NULL; 175 unsigned int data_len[2]; 176 u32 inl_mask; 177 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 178 OP_ALG_AAI_CTR_MOD128); 179 const bool is_rfc3686 = alg->caam.rfc3686; 180 181 if (!ctx->cdata.keylen || !ctx->authsize) 182 return 0; 183 184 /* 185 * AES-CTR needs to load IV in CONTEXT1 reg 186 * at an offset of 128bits (16bytes) 187 * CONTEXT1[255:128] = IV 188 */ 189 if (ctr_mode) 190 ctx1_iv_off = 16; 191 192 /* 193 * RFC3686 specific: 194 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 195 */ 196 if (is_rfc3686) { 197 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 198 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 199 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 200 } 201 202 /* 203 * In case |user key| > |derived key|, using DKP<imm,imm> would result 204 * in invalid opcodes (last bytes of user key) in the resulting 205 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 206 * addresses are needed. 207 */ 208 ctx->adata.key_virt = ctx->key; 209 ctx->adata.key_dma = ctx->key_dma; 210 211 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 212 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 213 214 data_len[0] = ctx->adata.keylen_pad; 215 data_len[1] = ctx->cdata.keylen; 216 217 /* aead_encrypt shared descriptor */ 218 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 219 DESC_QI_AEAD_ENC_LEN) + 220 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 221 DESC_JOB_IO_LEN, data_len, &inl_mask, 222 ARRAY_SIZE(data_len)) < 0) 223 return -EINVAL; 224 225 ctx->adata.key_inline = !!(inl_mask & 1); 226 ctx->cdata.key_inline = !!(inl_mask & 2); 227 228 flc = &ctx->flc[ENCRYPT]; 229 desc = flc->sh_desc; 230 231 if (alg->caam.geniv) 232 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 233 ivsize, ctx->authsize, is_rfc3686, 234 nonce, ctx1_iv_off, true, 235 priv->sec_attr.era); 236 else 237 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 238 ivsize, ctx->authsize, is_rfc3686, nonce, 239 ctx1_iv_off, true, priv->sec_attr.era); 240 241 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 242 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 243 sizeof(flc->flc) + desc_bytes(desc), 244 ctx->dir); 245 246 /* aead_decrypt shared descriptor */ 247 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 248 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 249 DESC_JOB_IO_LEN, data_len, &inl_mask, 250 ARRAY_SIZE(data_len)) < 0) 251 return -EINVAL; 252 253 ctx->adata.key_inline = !!(inl_mask & 1); 254 ctx->cdata.key_inline = !!(inl_mask & 2); 255 256 flc = &ctx->flc[DECRYPT]; 257 desc = flc->sh_desc; 258 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 259 ivsize, ctx->authsize, alg->caam.geniv, 260 is_rfc3686, nonce, ctx1_iv_off, true, 261 priv->sec_attr.era); 262 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 263 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 264 sizeof(flc->flc) + desc_bytes(desc), 265 ctx->dir); 266 267 return 0; 268 } 269 270 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 271 { 272 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 273 274 ctx->authsize = authsize; 275 aead_set_sh_desc(authenc); 276 277 return 0; 278 } 279 280 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 281 unsigned int keylen) 282 { 283 struct caam_ctx *ctx = crypto_aead_ctx(aead); 284 struct device *dev = ctx->dev; 285 struct crypto_authenc_keys keys; 286 287 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 288 goto badkey; 289 290 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 291 keys.authkeylen + keys.enckeylen, keys.enckeylen, 292 keys.authkeylen); 293 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 294 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 295 296 ctx->adata.keylen = keys.authkeylen; 297 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 298 OP_ALG_ALGSEL_MASK); 299 300 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 301 goto badkey; 302 303 memcpy(ctx->key, keys.authkey, keys.authkeylen); 304 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 305 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 306 keys.enckeylen, ctx->dir); 307 print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", 308 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 309 ctx->adata.keylen_pad + keys.enckeylen, 1); 310 311 ctx->cdata.keylen = keys.enckeylen; 312 313 memzero_explicit(&keys, sizeof(keys)); 314 return aead_set_sh_desc(aead); 315 badkey: 316 memzero_explicit(&keys, sizeof(keys)); 317 return -EINVAL; 318 } 319 320 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 321 unsigned int keylen) 322 { 323 struct crypto_authenc_keys keys; 324 int err; 325 326 err = crypto_authenc_extractkeys(&keys, key, keylen); 327 if (unlikely(err)) 328 goto out; 329 330 err = -EINVAL; 331 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 332 goto out; 333 334 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 335 aead_setkey(aead, key, keylen); 336 337 out: 338 memzero_explicit(&keys, sizeof(keys)); 339 return err; 340 } 341 342 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 343 bool encrypt) 344 { 345 struct crypto_aead *aead = crypto_aead_reqtfm(req); 346 struct caam_request *req_ctx = aead_request_ctx(req); 347 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 348 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 349 struct caam_ctx *ctx = crypto_aead_ctx(aead); 350 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 351 typeof(*alg), aead); 352 struct device *dev = ctx->dev; 353 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 354 GFP_KERNEL : GFP_ATOMIC; 355 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 356 int src_len, dst_len = 0; 357 struct aead_edesc *edesc; 358 dma_addr_t qm_sg_dma, iv_dma = 0; 359 int ivsize = 0; 360 unsigned int authsize = ctx->authsize; 361 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 362 int in_len, out_len; 363 struct dpaa2_sg_entry *sg_table; 364 365 /* allocate space for base edesc, link tables and IV */ 366 edesc = qi_cache_zalloc(GFP_DMA | flags); 367 if (unlikely(!edesc)) { 368 dev_err(dev, "could not allocate extended descriptor\n"); 369 return ERR_PTR(-ENOMEM); 370 } 371 372 if (unlikely(req->dst != req->src)) { 373 src_len = req->assoclen + req->cryptlen; 374 dst_len = src_len + (encrypt ? authsize : (-authsize)); 375 376 src_nents = sg_nents_for_len(req->src, src_len); 377 if (unlikely(src_nents < 0)) { 378 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 379 src_len); 380 qi_cache_free(edesc); 381 return ERR_PTR(src_nents); 382 } 383 384 dst_nents = sg_nents_for_len(req->dst, dst_len); 385 if (unlikely(dst_nents < 0)) { 386 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 387 dst_len); 388 qi_cache_free(edesc); 389 return ERR_PTR(dst_nents); 390 } 391 392 if (src_nents) { 393 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 394 DMA_TO_DEVICE); 395 if (unlikely(!mapped_src_nents)) { 396 dev_err(dev, "unable to map source\n"); 397 qi_cache_free(edesc); 398 return ERR_PTR(-ENOMEM); 399 } 400 } else { 401 mapped_src_nents = 0; 402 } 403 404 if (dst_nents) { 405 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 406 DMA_FROM_DEVICE); 407 if (unlikely(!mapped_dst_nents)) { 408 dev_err(dev, "unable to map destination\n"); 409 dma_unmap_sg(dev, req->src, src_nents, 410 DMA_TO_DEVICE); 411 qi_cache_free(edesc); 412 return ERR_PTR(-ENOMEM); 413 } 414 } else { 415 mapped_dst_nents = 0; 416 } 417 } else { 418 src_len = req->assoclen + req->cryptlen + 419 (encrypt ? authsize : 0); 420 421 src_nents = sg_nents_for_len(req->src, src_len); 422 if (unlikely(src_nents < 0)) { 423 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 424 src_len); 425 qi_cache_free(edesc); 426 return ERR_PTR(src_nents); 427 } 428 429 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 430 DMA_BIDIRECTIONAL); 431 if (unlikely(!mapped_src_nents)) { 432 dev_err(dev, "unable to map source\n"); 433 qi_cache_free(edesc); 434 return ERR_PTR(-ENOMEM); 435 } 436 } 437 438 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 439 ivsize = crypto_aead_ivsize(aead); 440 441 /* 442 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 443 * Input is not contiguous. 444 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 445 * the end of the table by allocating more S/G entries. Logic: 446 * if (src != dst && output S/G) 447 * pad output S/G, if needed 448 * else if (src == dst && S/G) 449 * overlapping S/Gs; pad one of them 450 * else if (input S/G) ... 451 * pad input S/G, if needed 452 */ 453 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 454 if (mapped_dst_nents > 1) 455 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 456 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 457 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 458 1 + !!ivsize + 459 pad_sg_nents(mapped_src_nents)); 460 else 461 qm_sg_nents = pad_sg_nents(qm_sg_nents); 462 463 sg_table = &edesc->sgt[0]; 464 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 465 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 466 CAAM_QI_MEMCACHE_SIZE)) { 467 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 468 qm_sg_nents, ivsize); 469 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 470 0, DMA_NONE, 0, 0); 471 qi_cache_free(edesc); 472 return ERR_PTR(-ENOMEM); 473 } 474 475 if (ivsize) { 476 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 477 478 /* Make sure IV is located in a DMAable area */ 479 memcpy(iv, req->iv, ivsize); 480 481 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 482 if (dma_mapping_error(dev, iv_dma)) { 483 dev_err(dev, "unable to map IV\n"); 484 caam_unmap(dev, req->src, req->dst, src_nents, 485 dst_nents, 0, 0, DMA_NONE, 0, 0); 486 qi_cache_free(edesc); 487 return ERR_PTR(-ENOMEM); 488 } 489 } 490 491 edesc->src_nents = src_nents; 492 edesc->dst_nents = dst_nents; 493 edesc->iv_dma = iv_dma; 494 495 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 496 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 497 /* 498 * The associated data comes already with the IV but we need 499 * to skip it when we authenticate or encrypt... 500 */ 501 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 502 else 503 edesc->assoclen = cpu_to_caam32(req->assoclen); 504 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 505 DMA_TO_DEVICE); 506 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 507 dev_err(dev, "unable to map assoclen\n"); 508 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 509 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 510 qi_cache_free(edesc); 511 return ERR_PTR(-ENOMEM); 512 } 513 514 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 515 qm_sg_index++; 516 if (ivsize) { 517 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 518 qm_sg_index++; 519 } 520 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 521 qm_sg_index += mapped_src_nents; 522 523 if (mapped_dst_nents > 1) 524 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 525 526 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 527 if (dma_mapping_error(dev, qm_sg_dma)) { 528 dev_err(dev, "unable to map S/G table\n"); 529 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 530 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 531 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 532 qi_cache_free(edesc); 533 return ERR_PTR(-ENOMEM); 534 } 535 536 edesc->qm_sg_dma = qm_sg_dma; 537 edesc->qm_sg_bytes = qm_sg_bytes; 538 539 out_len = req->assoclen + req->cryptlen + 540 (encrypt ? ctx->authsize : (-ctx->authsize)); 541 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 542 543 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 544 dpaa2_fl_set_final(in_fle, true); 545 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 546 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 547 dpaa2_fl_set_len(in_fle, in_len); 548 549 if (req->dst == req->src) { 550 if (mapped_src_nents == 1) { 551 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 552 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 553 } else { 554 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 555 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 556 (1 + !!ivsize) * sizeof(*sg_table)); 557 } 558 } else if (!mapped_dst_nents) { 559 /* 560 * crypto engine requires the output entry to be present when 561 * "frame list" FD is used. 562 * Since engine does not support FMT=2'b11 (unused entry type), 563 * leaving out_fle zeroized is the best option. 564 */ 565 goto skip_out_fle; 566 } else if (mapped_dst_nents == 1) { 567 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 568 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 569 } else { 570 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 571 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 572 sizeof(*sg_table)); 573 } 574 575 dpaa2_fl_set_len(out_fle, out_len); 576 577 skip_out_fle: 578 return edesc; 579 } 580 581 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 582 { 583 struct caam_ctx *ctx = crypto_aead_ctx(aead); 584 unsigned int ivsize = crypto_aead_ivsize(aead); 585 struct device *dev = ctx->dev; 586 struct caam_flc *flc; 587 u32 *desc; 588 589 if (!ctx->cdata.keylen || !ctx->authsize) 590 return 0; 591 592 flc = &ctx->flc[ENCRYPT]; 593 desc = flc->sh_desc; 594 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 595 ctx->authsize, true, true); 596 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 597 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 598 sizeof(flc->flc) + desc_bytes(desc), 599 ctx->dir); 600 601 flc = &ctx->flc[DECRYPT]; 602 desc = flc->sh_desc; 603 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 604 ctx->authsize, false, true); 605 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 606 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 607 sizeof(flc->flc) + desc_bytes(desc), 608 ctx->dir); 609 610 return 0; 611 } 612 613 static int chachapoly_setauthsize(struct crypto_aead *aead, 614 unsigned int authsize) 615 { 616 struct caam_ctx *ctx = crypto_aead_ctx(aead); 617 618 if (authsize != POLY1305_DIGEST_SIZE) 619 return -EINVAL; 620 621 ctx->authsize = authsize; 622 return chachapoly_set_sh_desc(aead); 623 } 624 625 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 626 unsigned int keylen) 627 { 628 struct caam_ctx *ctx = crypto_aead_ctx(aead); 629 unsigned int ivsize = crypto_aead_ivsize(aead); 630 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 631 632 if (keylen != CHACHA_KEY_SIZE + saltlen) 633 return -EINVAL; 634 635 ctx->cdata.key_virt = key; 636 ctx->cdata.keylen = keylen - saltlen; 637 638 return chachapoly_set_sh_desc(aead); 639 } 640 641 static int gcm_set_sh_desc(struct crypto_aead *aead) 642 { 643 struct caam_ctx *ctx = crypto_aead_ctx(aead); 644 struct device *dev = ctx->dev; 645 unsigned int ivsize = crypto_aead_ivsize(aead); 646 struct caam_flc *flc; 647 u32 *desc; 648 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 649 ctx->cdata.keylen; 650 651 if (!ctx->cdata.keylen || !ctx->authsize) 652 return 0; 653 654 /* 655 * AES GCM encrypt shared descriptor 656 * Job Descriptor and Shared Descriptor 657 * must fit into the 64-word Descriptor h/w Buffer 658 */ 659 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 660 ctx->cdata.key_inline = true; 661 ctx->cdata.key_virt = ctx->key; 662 } else { 663 ctx->cdata.key_inline = false; 664 ctx->cdata.key_dma = ctx->key_dma; 665 } 666 667 flc = &ctx->flc[ENCRYPT]; 668 desc = flc->sh_desc; 669 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 670 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 671 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 672 sizeof(flc->flc) + desc_bytes(desc), 673 ctx->dir); 674 675 /* 676 * Job Descriptor and Shared Descriptors 677 * must all fit into the 64-word Descriptor h/w Buffer 678 */ 679 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 680 ctx->cdata.key_inline = true; 681 ctx->cdata.key_virt = ctx->key; 682 } else { 683 ctx->cdata.key_inline = false; 684 ctx->cdata.key_dma = ctx->key_dma; 685 } 686 687 flc = &ctx->flc[DECRYPT]; 688 desc = flc->sh_desc; 689 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 690 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 691 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 692 sizeof(flc->flc) + desc_bytes(desc), 693 ctx->dir); 694 695 return 0; 696 } 697 698 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 699 { 700 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 701 int err; 702 703 err = crypto_gcm_check_authsize(authsize); 704 if (err) 705 return err; 706 707 ctx->authsize = authsize; 708 gcm_set_sh_desc(authenc); 709 710 return 0; 711 } 712 713 static int gcm_setkey(struct crypto_aead *aead, 714 const u8 *key, unsigned int keylen) 715 { 716 struct caam_ctx *ctx = crypto_aead_ctx(aead); 717 struct device *dev = ctx->dev; 718 int ret; 719 720 ret = aes_check_keylen(keylen); 721 if (ret) 722 return ret; 723 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 724 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 725 726 memcpy(ctx->key, key, keylen); 727 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 728 ctx->cdata.keylen = keylen; 729 730 return gcm_set_sh_desc(aead); 731 } 732 733 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 734 { 735 struct caam_ctx *ctx = crypto_aead_ctx(aead); 736 struct device *dev = ctx->dev; 737 unsigned int ivsize = crypto_aead_ivsize(aead); 738 struct caam_flc *flc; 739 u32 *desc; 740 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 741 ctx->cdata.keylen; 742 743 if (!ctx->cdata.keylen || !ctx->authsize) 744 return 0; 745 746 ctx->cdata.key_virt = ctx->key; 747 748 /* 749 * RFC4106 encrypt shared descriptor 750 * Job Descriptor and Shared Descriptor 751 * must fit into the 64-word Descriptor h/w Buffer 752 */ 753 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 754 ctx->cdata.key_inline = true; 755 } else { 756 ctx->cdata.key_inline = false; 757 ctx->cdata.key_dma = ctx->key_dma; 758 } 759 760 flc = &ctx->flc[ENCRYPT]; 761 desc = flc->sh_desc; 762 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 763 true); 764 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 765 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 766 sizeof(flc->flc) + desc_bytes(desc), 767 ctx->dir); 768 769 /* 770 * Job Descriptor and Shared Descriptors 771 * must all fit into the 64-word Descriptor h/w Buffer 772 */ 773 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 774 ctx->cdata.key_inline = true; 775 } else { 776 ctx->cdata.key_inline = false; 777 ctx->cdata.key_dma = ctx->key_dma; 778 } 779 780 flc = &ctx->flc[DECRYPT]; 781 desc = flc->sh_desc; 782 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 783 true); 784 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 785 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 786 sizeof(flc->flc) + desc_bytes(desc), 787 ctx->dir); 788 789 return 0; 790 } 791 792 static int rfc4106_setauthsize(struct crypto_aead *authenc, 793 unsigned int authsize) 794 { 795 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 796 int err; 797 798 err = crypto_rfc4106_check_authsize(authsize); 799 if (err) 800 return err; 801 802 ctx->authsize = authsize; 803 rfc4106_set_sh_desc(authenc); 804 805 return 0; 806 } 807 808 static int rfc4106_setkey(struct crypto_aead *aead, 809 const u8 *key, unsigned int keylen) 810 { 811 struct caam_ctx *ctx = crypto_aead_ctx(aead); 812 struct device *dev = ctx->dev; 813 int ret; 814 815 ret = aes_check_keylen(keylen - 4); 816 if (ret) 817 return ret; 818 819 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 820 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 821 822 memcpy(ctx->key, key, keylen); 823 /* 824 * The last four bytes of the key material are used as the salt value 825 * in the nonce. Update the AES key length. 826 */ 827 ctx->cdata.keylen = keylen - 4; 828 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 829 ctx->dir); 830 831 return rfc4106_set_sh_desc(aead); 832 } 833 834 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 835 { 836 struct caam_ctx *ctx = crypto_aead_ctx(aead); 837 struct device *dev = ctx->dev; 838 unsigned int ivsize = crypto_aead_ivsize(aead); 839 struct caam_flc *flc; 840 u32 *desc; 841 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 842 ctx->cdata.keylen; 843 844 if (!ctx->cdata.keylen || !ctx->authsize) 845 return 0; 846 847 ctx->cdata.key_virt = ctx->key; 848 849 /* 850 * RFC4543 encrypt shared descriptor 851 * Job Descriptor and Shared Descriptor 852 * must fit into the 64-word Descriptor h/w Buffer 853 */ 854 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 855 ctx->cdata.key_inline = true; 856 } else { 857 ctx->cdata.key_inline = false; 858 ctx->cdata.key_dma = ctx->key_dma; 859 } 860 861 flc = &ctx->flc[ENCRYPT]; 862 desc = flc->sh_desc; 863 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 864 true); 865 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 866 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 867 sizeof(flc->flc) + desc_bytes(desc), 868 ctx->dir); 869 870 /* 871 * Job Descriptor and Shared Descriptors 872 * must all fit into the 64-word Descriptor h/w Buffer 873 */ 874 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 875 ctx->cdata.key_inline = true; 876 } else { 877 ctx->cdata.key_inline = false; 878 ctx->cdata.key_dma = ctx->key_dma; 879 } 880 881 flc = &ctx->flc[DECRYPT]; 882 desc = flc->sh_desc; 883 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 884 true); 885 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 886 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 887 sizeof(flc->flc) + desc_bytes(desc), 888 ctx->dir); 889 890 return 0; 891 } 892 893 static int rfc4543_setauthsize(struct crypto_aead *authenc, 894 unsigned int authsize) 895 { 896 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 897 898 if (authsize != 16) 899 return -EINVAL; 900 901 ctx->authsize = authsize; 902 rfc4543_set_sh_desc(authenc); 903 904 return 0; 905 } 906 907 static int rfc4543_setkey(struct crypto_aead *aead, 908 const u8 *key, unsigned int keylen) 909 { 910 struct caam_ctx *ctx = crypto_aead_ctx(aead); 911 struct device *dev = ctx->dev; 912 int ret; 913 914 ret = aes_check_keylen(keylen - 4); 915 if (ret) 916 return ret; 917 918 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 919 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 920 921 memcpy(ctx->key, key, keylen); 922 /* 923 * The last four bytes of the key material are used as the salt value 924 * in the nonce. Update the AES key length. 925 */ 926 ctx->cdata.keylen = keylen - 4; 927 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 928 ctx->dir); 929 930 return rfc4543_set_sh_desc(aead); 931 } 932 933 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 934 unsigned int keylen, const u32 ctx1_iv_off) 935 { 936 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 937 struct caam_skcipher_alg *alg = 938 container_of(crypto_skcipher_alg(skcipher), 939 struct caam_skcipher_alg, skcipher); 940 struct device *dev = ctx->dev; 941 struct caam_flc *flc; 942 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 943 u32 *desc; 944 const bool is_rfc3686 = alg->caam.rfc3686; 945 946 print_hex_dump_debug("key in @" __stringify(__LINE__)": ", 947 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 948 949 ctx->cdata.keylen = keylen; 950 ctx->cdata.key_virt = key; 951 ctx->cdata.key_inline = true; 952 953 /* skcipher_encrypt shared descriptor */ 954 flc = &ctx->flc[ENCRYPT]; 955 desc = flc->sh_desc; 956 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 957 ctx1_iv_off); 958 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 959 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 960 sizeof(flc->flc) + desc_bytes(desc), 961 ctx->dir); 962 963 /* skcipher_decrypt shared descriptor */ 964 flc = &ctx->flc[DECRYPT]; 965 desc = flc->sh_desc; 966 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 967 ctx1_iv_off); 968 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 969 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 970 sizeof(flc->flc) + desc_bytes(desc), 971 ctx->dir); 972 973 return 0; 974 } 975 976 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 977 const u8 *key, unsigned int keylen) 978 { 979 int err; 980 981 err = aes_check_keylen(keylen); 982 if (err) 983 return err; 984 985 return skcipher_setkey(skcipher, key, keylen, 0); 986 } 987 988 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 989 const u8 *key, unsigned int keylen) 990 { 991 u32 ctx1_iv_off; 992 int err; 993 994 /* 995 * RFC3686 specific: 996 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 997 * | *key = {KEY, NONCE} 998 */ 999 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1000 keylen -= CTR_RFC3686_NONCE_SIZE; 1001 1002 err = aes_check_keylen(keylen); 1003 if (err) 1004 return err; 1005 1006 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1007 } 1008 1009 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1010 const u8 *key, unsigned int keylen) 1011 { 1012 u32 ctx1_iv_off; 1013 int err; 1014 1015 /* 1016 * AES-CTR needs to load IV in CONTEXT1 reg 1017 * at an offset of 128bits (16bytes) 1018 * CONTEXT1[255:128] = IV 1019 */ 1020 ctx1_iv_off = 16; 1021 1022 err = aes_check_keylen(keylen); 1023 if (err) 1024 return err; 1025 1026 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1027 } 1028 1029 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1030 const u8 *key, unsigned int keylen) 1031 { 1032 if (keylen != CHACHA_KEY_SIZE) 1033 return -EINVAL; 1034 1035 return skcipher_setkey(skcipher, key, keylen, 0); 1036 } 1037 1038 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1039 const u8 *key, unsigned int keylen) 1040 { 1041 return verify_skcipher_des_key(skcipher, key) ?: 1042 skcipher_setkey(skcipher, key, keylen, 0); 1043 } 1044 1045 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1046 const u8 *key, unsigned int keylen) 1047 { 1048 return verify_skcipher_des3_key(skcipher, key) ?: 1049 skcipher_setkey(skcipher, key, keylen, 0); 1050 } 1051 1052 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1053 unsigned int keylen) 1054 { 1055 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1056 struct device *dev = ctx->dev; 1057 struct caam_flc *flc; 1058 u32 *desc; 1059 1060 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { 1061 dev_err(dev, "key size mismatch\n"); 1062 return -EINVAL; 1063 } 1064 1065 ctx->cdata.keylen = keylen; 1066 ctx->cdata.key_virt = key; 1067 ctx->cdata.key_inline = true; 1068 1069 /* xts_skcipher_encrypt shared descriptor */ 1070 flc = &ctx->flc[ENCRYPT]; 1071 desc = flc->sh_desc; 1072 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1073 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1074 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1075 sizeof(flc->flc) + desc_bytes(desc), 1076 ctx->dir); 1077 1078 /* xts_skcipher_decrypt shared descriptor */ 1079 flc = &ctx->flc[DECRYPT]; 1080 desc = flc->sh_desc; 1081 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1082 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1083 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1084 sizeof(flc->flc) + desc_bytes(desc), 1085 ctx->dir); 1086 1087 return 0; 1088 } 1089 1090 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1091 { 1092 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1093 struct caam_request *req_ctx = skcipher_request_ctx(req); 1094 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1095 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1096 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1097 struct device *dev = ctx->dev; 1098 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1099 GFP_KERNEL : GFP_ATOMIC; 1100 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1101 struct skcipher_edesc *edesc; 1102 dma_addr_t iv_dma; 1103 u8 *iv; 1104 int ivsize = crypto_skcipher_ivsize(skcipher); 1105 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1106 struct dpaa2_sg_entry *sg_table; 1107 1108 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1109 if (unlikely(src_nents < 0)) { 1110 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1111 req->cryptlen); 1112 return ERR_PTR(src_nents); 1113 } 1114 1115 if (unlikely(req->dst != req->src)) { 1116 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1117 if (unlikely(dst_nents < 0)) { 1118 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1119 req->cryptlen); 1120 return ERR_PTR(dst_nents); 1121 } 1122 1123 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1124 DMA_TO_DEVICE); 1125 if (unlikely(!mapped_src_nents)) { 1126 dev_err(dev, "unable to map source\n"); 1127 return ERR_PTR(-ENOMEM); 1128 } 1129 1130 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1131 DMA_FROM_DEVICE); 1132 if (unlikely(!mapped_dst_nents)) { 1133 dev_err(dev, "unable to map destination\n"); 1134 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1135 return ERR_PTR(-ENOMEM); 1136 } 1137 } else { 1138 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1139 DMA_BIDIRECTIONAL); 1140 if (unlikely(!mapped_src_nents)) { 1141 dev_err(dev, "unable to map source\n"); 1142 return ERR_PTR(-ENOMEM); 1143 } 1144 } 1145 1146 qm_sg_ents = 1 + mapped_src_nents; 1147 dst_sg_idx = qm_sg_ents; 1148 1149 /* 1150 * Input, output HW S/G tables: [IV, src][dst, IV] 1151 * IV entries point to the same buffer 1152 * If src == dst, S/G entries are reused (S/G tables overlap) 1153 * 1154 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1155 * the end of the table by allocating more S/G entries. 1156 */ 1157 if (req->src != req->dst) 1158 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1159 else 1160 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1161 1162 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1163 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1164 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1165 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1166 qm_sg_ents, ivsize); 1167 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1168 0, DMA_NONE, 0, 0); 1169 return ERR_PTR(-ENOMEM); 1170 } 1171 1172 /* allocate space for base edesc, link tables and IV */ 1173 edesc = qi_cache_zalloc(GFP_DMA | flags); 1174 if (unlikely(!edesc)) { 1175 dev_err(dev, "could not allocate extended descriptor\n"); 1176 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1177 0, DMA_NONE, 0, 0); 1178 return ERR_PTR(-ENOMEM); 1179 } 1180 1181 /* Make sure IV is located in a DMAable area */ 1182 sg_table = &edesc->sgt[0]; 1183 iv = (u8 *)(sg_table + qm_sg_ents); 1184 memcpy(iv, req->iv, ivsize); 1185 1186 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1187 if (dma_mapping_error(dev, iv_dma)) { 1188 dev_err(dev, "unable to map IV\n"); 1189 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1190 0, DMA_NONE, 0, 0); 1191 qi_cache_free(edesc); 1192 return ERR_PTR(-ENOMEM); 1193 } 1194 1195 edesc->src_nents = src_nents; 1196 edesc->dst_nents = dst_nents; 1197 edesc->iv_dma = iv_dma; 1198 edesc->qm_sg_bytes = qm_sg_bytes; 1199 1200 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1201 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1202 1203 if (req->src != req->dst) 1204 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1205 1206 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1207 ivsize, 0); 1208 1209 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1210 DMA_TO_DEVICE); 1211 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1212 dev_err(dev, "unable to map S/G table\n"); 1213 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1214 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1215 qi_cache_free(edesc); 1216 return ERR_PTR(-ENOMEM); 1217 } 1218 1219 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1220 dpaa2_fl_set_final(in_fle, true); 1221 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1222 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1223 1224 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1225 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1226 1227 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1228 1229 if (req->src == req->dst) 1230 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1231 sizeof(*sg_table)); 1232 else 1233 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1234 sizeof(*sg_table)); 1235 1236 return edesc; 1237 } 1238 1239 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1240 struct aead_request *req) 1241 { 1242 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1243 int ivsize = crypto_aead_ivsize(aead); 1244 1245 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1246 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1247 edesc->qm_sg_bytes); 1248 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1249 } 1250 1251 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1252 struct skcipher_request *req) 1253 { 1254 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1255 int ivsize = crypto_skcipher_ivsize(skcipher); 1256 1257 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1258 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1259 edesc->qm_sg_bytes); 1260 } 1261 1262 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1263 { 1264 struct crypto_async_request *areq = cbk_ctx; 1265 struct aead_request *req = container_of(areq, struct aead_request, 1266 base); 1267 struct caam_request *req_ctx = to_caam_req(areq); 1268 struct aead_edesc *edesc = req_ctx->edesc; 1269 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1270 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1271 int ecode = 0; 1272 1273 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1274 1275 if (unlikely(status)) 1276 ecode = caam_qi2_strstatus(ctx->dev, status); 1277 1278 aead_unmap(ctx->dev, edesc, req); 1279 qi_cache_free(edesc); 1280 aead_request_complete(req, ecode); 1281 } 1282 1283 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1284 { 1285 struct crypto_async_request *areq = cbk_ctx; 1286 struct aead_request *req = container_of(areq, struct aead_request, 1287 base); 1288 struct caam_request *req_ctx = to_caam_req(areq); 1289 struct aead_edesc *edesc = req_ctx->edesc; 1290 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1291 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1292 int ecode = 0; 1293 1294 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1295 1296 if (unlikely(status)) 1297 ecode = caam_qi2_strstatus(ctx->dev, status); 1298 1299 aead_unmap(ctx->dev, edesc, req); 1300 qi_cache_free(edesc); 1301 aead_request_complete(req, ecode); 1302 } 1303 1304 static int aead_encrypt(struct aead_request *req) 1305 { 1306 struct aead_edesc *edesc; 1307 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1308 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1309 struct caam_request *caam_req = aead_request_ctx(req); 1310 int ret; 1311 1312 /* allocate extended descriptor */ 1313 edesc = aead_edesc_alloc(req, true); 1314 if (IS_ERR(edesc)) 1315 return PTR_ERR(edesc); 1316 1317 caam_req->flc = &ctx->flc[ENCRYPT]; 1318 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1319 caam_req->cbk = aead_encrypt_done; 1320 caam_req->ctx = &req->base; 1321 caam_req->edesc = edesc; 1322 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1323 if (ret != -EINPROGRESS && 1324 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1325 aead_unmap(ctx->dev, edesc, req); 1326 qi_cache_free(edesc); 1327 } 1328 1329 return ret; 1330 } 1331 1332 static int aead_decrypt(struct aead_request *req) 1333 { 1334 struct aead_edesc *edesc; 1335 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1336 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1337 struct caam_request *caam_req = aead_request_ctx(req); 1338 int ret; 1339 1340 /* allocate extended descriptor */ 1341 edesc = aead_edesc_alloc(req, false); 1342 if (IS_ERR(edesc)) 1343 return PTR_ERR(edesc); 1344 1345 caam_req->flc = &ctx->flc[DECRYPT]; 1346 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1347 caam_req->cbk = aead_decrypt_done; 1348 caam_req->ctx = &req->base; 1349 caam_req->edesc = edesc; 1350 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1351 if (ret != -EINPROGRESS && 1352 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1353 aead_unmap(ctx->dev, edesc, req); 1354 qi_cache_free(edesc); 1355 } 1356 1357 return ret; 1358 } 1359 1360 static int ipsec_gcm_encrypt(struct aead_request *req) 1361 { 1362 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1363 } 1364 1365 static int ipsec_gcm_decrypt(struct aead_request *req) 1366 { 1367 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1368 } 1369 1370 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1371 { 1372 struct crypto_async_request *areq = cbk_ctx; 1373 struct skcipher_request *req = skcipher_request_cast(areq); 1374 struct caam_request *req_ctx = to_caam_req(areq); 1375 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1376 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1377 struct skcipher_edesc *edesc = req_ctx->edesc; 1378 int ecode = 0; 1379 int ivsize = crypto_skcipher_ivsize(skcipher); 1380 1381 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1382 1383 if (unlikely(status)) 1384 ecode = caam_qi2_strstatus(ctx->dev, status); 1385 1386 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1387 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1388 edesc->src_nents > 1 ? 100 : ivsize, 1); 1389 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1390 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1391 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1392 1393 skcipher_unmap(ctx->dev, edesc, req); 1394 1395 /* 1396 * The crypto API expects us to set the IV (req->iv) to the last 1397 * ciphertext block (CBC mode) or last counter (CTR mode). 1398 * This is used e.g. by the CTS mode. 1399 */ 1400 if (!ecode) 1401 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1402 ivsize); 1403 1404 qi_cache_free(edesc); 1405 skcipher_request_complete(req, ecode); 1406 } 1407 1408 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1409 { 1410 struct crypto_async_request *areq = cbk_ctx; 1411 struct skcipher_request *req = skcipher_request_cast(areq); 1412 struct caam_request *req_ctx = to_caam_req(areq); 1413 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1414 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1415 struct skcipher_edesc *edesc = req_ctx->edesc; 1416 int ecode = 0; 1417 int ivsize = crypto_skcipher_ivsize(skcipher); 1418 1419 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1420 1421 if (unlikely(status)) 1422 ecode = caam_qi2_strstatus(ctx->dev, status); 1423 1424 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1425 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1426 edesc->src_nents > 1 ? 100 : ivsize, 1); 1427 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1428 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1429 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1430 1431 skcipher_unmap(ctx->dev, edesc, req); 1432 1433 /* 1434 * The crypto API expects us to set the IV (req->iv) to the last 1435 * ciphertext block (CBC mode) or last counter (CTR mode). 1436 * This is used e.g. by the CTS mode. 1437 */ 1438 if (!ecode) 1439 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1440 ivsize); 1441 1442 qi_cache_free(edesc); 1443 skcipher_request_complete(req, ecode); 1444 } 1445 1446 static int skcipher_encrypt(struct skcipher_request *req) 1447 { 1448 struct skcipher_edesc *edesc; 1449 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1450 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1451 struct caam_request *caam_req = skcipher_request_ctx(req); 1452 int ret; 1453 1454 if (!req->cryptlen) 1455 return 0; 1456 1457 /* allocate extended descriptor */ 1458 edesc = skcipher_edesc_alloc(req); 1459 if (IS_ERR(edesc)) 1460 return PTR_ERR(edesc); 1461 1462 caam_req->flc = &ctx->flc[ENCRYPT]; 1463 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1464 caam_req->cbk = skcipher_encrypt_done; 1465 caam_req->ctx = &req->base; 1466 caam_req->edesc = edesc; 1467 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1468 if (ret != -EINPROGRESS && 1469 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1470 skcipher_unmap(ctx->dev, edesc, req); 1471 qi_cache_free(edesc); 1472 } 1473 1474 return ret; 1475 } 1476 1477 static int skcipher_decrypt(struct skcipher_request *req) 1478 { 1479 struct skcipher_edesc *edesc; 1480 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1481 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); 1482 struct caam_request *caam_req = skcipher_request_ctx(req); 1483 int ret; 1484 1485 if (!req->cryptlen) 1486 return 0; 1487 /* allocate extended descriptor */ 1488 edesc = skcipher_edesc_alloc(req); 1489 if (IS_ERR(edesc)) 1490 return PTR_ERR(edesc); 1491 1492 caam_req->flc = &ctx->flc[DECRYPT]; 1493 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1494 caam_req->cbk = skcipher_decrypt_done; 1495 caam_req->ctx = &req->base; 1496 caam_req->edesc = edesc; 1497 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1498 if (ret != -EINPROGRESS && 1499 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1500 skcipher_unmap(ctx->dev, edesc, req); 1501 qi_cache_free(edesc); 1502 } 1503 1504 return ret; 1505 } 1506 1507 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1508 bool uses_dkp) 1509 { 1510 dma_addr_t dma_addr; 1511 int i; 1512 1513 /* copy descriptor header template value */ 1514 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1515 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1516 1517 ctx->dev = caam->dev; 1518 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1519 1520 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1521 offsetof(struct caam_ctx, flc_dma), 1522 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1523 if (dma_mapping_error(ctx->dev, dma_addr)) { 1524 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1525 return -ENOMEM; 1526 } 1527 1528 for (i = 0; i < NUM_OP; i++) 1529 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1530 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1531 1532 return 0; 1533 } 1534 1535 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1536 { 1537 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1538 struct caam_skcipher_alg *caam_alg = 1539 container_of(alg, typeof(*caam_alg), skcipher); 1540 1541 crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request)); 1542 return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false); 1543 } 1544 1545 static int caam_cra_init_aead(struct crypto_aead *tfm) 1546 { 1547 struct aead_alg *alg = crypto_aead_alg(tfm); 1548 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1549 aead); 1550 1551 crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); 1552 return caam_cra_init(crypto_aead_ctx(tfm), &caam_alg->caam, 1553 !caam_alg->caam.nodkp); 1554 } 1555 1556 static void caam_exit_common(struct caam_ctx *ctx) 1557 { 1558 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1559 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1560 DMA_ATTR_SKIP_CPU_SYNC); 1561 } 1562 1563 static void caam_cra_exit(struct crypto_skcipher *tfm) 1564 { 1565 caam_exit_common(crypto_skcipher_ctx(tfm)); 1566 } 1567 1568 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1569 { 1570 caam_exit_common(crypto_aead_ctx(tfm)); 1571 } 1572 1573 static struct caam_skcipher_alg driver_algs[] = { 1574 { 1575 .skcipher = { 1576 .base = { 1577 .cra_name = "cbc(aes)", 1578 .cra_driver_name = "cbc-aes-caam-qi2", 1579 .cra_blocksize = AES_BLOCK_SIZE, 1580 }, 1581 .setkey = aes_skcipher_setkey, 1582 .encrypt = skcipher_encrypt, 1583 .decrypt = skcipher_decrypt, 1584 .min_keysize = AES_MIN_KEY_SIZE, 1585 .max_keysize = AES_MAX_KEY_SIZE, 1586 .ivsize = AES_BLOCK_SIZE, 1587 }, 1588 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1589 }, 1590 { 1591 .skcipher = { 1592 .base = { 1593 .cra_name = "cbc(des3_ede)", 1594 .cra_driver_name = "cbc-3des-caam-qi2", 1595 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1596 }, 1597 .setkey = des3_skcipher_setkey, 1598 .encrypt = skcipher_encrypt, 1599 .decrypt = skcipher_decrypt, 1600 .min_keysize = DES3_EDE_KEY_SIZE, 1601 .max_keysize = DES3_EDE_KEY_SIZE, 1602 .ivsize = DES3_EDE_BLOCK_SIZE, 1603 }, 1604 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1605 }, 1606 { 1607 .skcipher = { 1608 .base = { 1609 .cra_name = "cbc(des)", 1610 .cra_driver_name = "cbc-des-caam-qi2", 1611 .cra_blocksize = DES_BLOCK_SIZE, 1612 }, 1613 .setkey = des_skcipher_setkey, 1614 .encrypt = skcipher_encrypt, 1615 .decrypt = skcipher_decrypt, 1616 .min_keysize = DES_KEY_SIZE, 1617 .max_keysize = DES_KEY_SIZE, 1618 .ivsize = DES_BLOCK_SIZE, 1619 }, 1620 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1621 }, 1622 { 1623 .skcipher = { 1624 .base = { 1625 .cra_name = "ctr(aes)", 1626 .cra_driver_name = "ctr-aes-caam-qi2", 1627 .cra_blocksize = 1, 1628 }, 1629 .setkey = ctr_skcipher_setkey, 1630 .encrypt = skcipher_encrypt, 1631 .decrypt = skcipher_decrypt, 1632 .min_keysize = AES_MIN_KEY_SIZE, 1633 .max_keysize = AES_MAX_KEY_SIZE, 1634 .ivsize = AES_BLOCK_SIZE, 1635 .chunksize = AES_BLOCK_SIZE, 1636 }, 1637 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1638 OP_ALG_AAI_CTR_MOD128, 1639 }, 1640 { 1641 .skcipher = { 1642 .base = { 1643 .cra_name = "rfc3686(ctr(aes))", 1644 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1645 .cra_blocksize = 1, 1646 }, 1647 .setkey = rfc3686_skcipher_setkey, 1648 .encrypt = skcipher_encrypt, 1649 .decrypt = skcipher_decrypt, 1650 .min_keysize = AES_MIN_KEY_SIZE + 1651 CTR_RFC3686_NONCE_SIZE, 1652 .max_keysize = AES_MAX_KEY_SIZE + 1653 CTR_RFC3686_NONCE_SIZE, 1654 .ivsize = CTR_RFC3686_IV_SIZE, 1655 .chunksize = AES_BLOCK_SIZE, 1656 }, 1657 .caam = { 1658 .class1_alg_type = OP_ALG_ALGSEL_AES | 1659 OP_ALG_AAI_CTR_MOD128, 1660 .rfc3686 = true, 1661 }, 1662 }, 1663 { 1664 .skcipher = { 1665 .base = { 1666 .cra_name = "xts(aes)", 1667 .cra_driver_name = "xts-aes-caam-qi2", 1668 .cra_blocksize = AES_BLOCK_SIZE, 1669 }, 1670 .setkey = xts_skcipher_setkey, 1671 .encrypt = skcipher_encrypt, 1672 .decrypt = skcipher_decrypt, 1673 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1674 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1675 .ivsize = AES_BLOCK_SIZE, 1676 }, 1677 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1678 }, 1679 { 1680 .skcipher = { 1681 .base = { 1682 .cra_name = "chacha20", 1683 .cra_driver_name = "chacha20-caam-qi2", 1684 .cra_blocksize = 1, 1685 }, 1686 .setkey = chacha20_skcipher_setkey, 1687 .encrypt = skcipher_encrypt, 1688 .decrypt = skcipher_decrypt, 1689 .min_keysize = CHACHA_KEY_SIZE, 1690 .max_keysize = CHACHA_KEY_SIZE, 1691 .ivsize = CHACHA_IV_SIZE, 1692 }, 1693 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1694 }, 1695 }; 1696 1697 static struct caam_aead_alg driver_aeads[] = { 1698 { 1699 .aead = { 1700 .base = { 1701 .cra_name = "rfc4106(gcm(aes))", 1702 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1703 .cra_blocksize = 1, 1704 }, 1705 .setkey = rfc4106_setkey, 1706 .setauthsize = rfc4106_setauthsize, 1707 .encrypt = ipsec_gcm_encrypt, 1708 .decrypt = ipsec_gcm_decrypt, 1709 .ivsize = 8, 1710 .maxauthsize = AES_BLOCK_SIZE, 1711 }, 1712 .caam = { 1713 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1714 .nodkp = true, 1715 }, 1716 }, 1717 { 1718 .aead = { 1719 .base = { 1720 .cra_name = "rfc4543(gcm(aes))", 1721 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1722 .cra_blocksize = 1, 1723 }, 1724 .setkey = rfc4543_setkey, 1725 .setauthsize = rfc4543_setauthsize, 1726 .encrypt = ipsec_gcm_encrypt, 1727 .decrypt = ipsec_gcm_decrypt, 1728 .ivsize = 8, 1729 .maxauthsize = AES_BLOCK_SIZE, 1730 }, 1731 .caam = { 1732 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1733 .nodkp = true, 1734 }, 1735 }, 1736 /* Galois Counter Mode */ 1737 { 1738 .aead = { 1739 .base = { 1740 .cra_name = "gcm(aes)", 1741 .cra_driver_name = "gcm-aes-caam-qi2", 1742 .cra_blocksize = 1, 1743 }, 1744 .setkey = gcm_setkey, 1745 .setauthsize = gcm_setauthsize, 1746 .encrypt = aead_encrypt, 1747 .decrypt = aead_decrypt, 1748 .ivsize = 12, 1749 .maxauthsize = AES_BLOCK_SIZE, 1750 }, 1751 .caam = { 1752 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1753 .nodkp = true, 1754 } 1755 }, 1756 /* single-pass ipsec_esp descriptor */ 1757 { 1758 .aead = { 1759 .base = { 1760 .cra_name = "authenc(hmac(md5),cbc(aes))", 1761 .cra_driver_name = "authenc-hmac-md5-" 1762 "cbc-aes-caam-qi2", 1763 .cra_blocksize = AES_BLOCK_SIZE, 1764 }, 1765 .setkey = aead_setkey, 1766 .setauthsize = aead_setauthsize, 1767 .encrypt = aead_encrypt, 1768 .decrypt = aead_decrypt, 1769 .ivsize = AES_BLOCK_SIZE, 1770 .maxauthsize = MD5_DIGEST_SIZE, 1771 }, 1772 .caam = { 1773 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1774 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1775 OP_ALG_AAI_HMAC_PRECOMP, 1776 } 1777 }, 1778 { 1779 .aead = { 1780 .base = { 1781 .cra_name = "echainiv(authenc(hmac(md5)," 1782 "cbc(aes)))", 1783 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1784 "cbc-aes-caam-qi2", 1785 .cra_blocksize = AES_BLOCK_SIZE, 1786 }, 1787 .setkey = aead_setkey, 1788 .setauthsize = aead_setauthsize, 1789 .encrypt = aead_encrypt, 1790 .decrypt = aead_decrypt, 1791 .ivsize = AES_BLOCK_SIZE, 1792 .maxauthsize = MD5_DIGEST_SIZE, 1793 }, 1794 .caam = { 1795 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1796 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1797 OP_ALG_AAI_HMAC_PRECOMP, 1798 .geniv = true, 1799 } 1800 }, 1801 { 1802 .aead = { 1803 .base = { 1804 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1805 .cra_driver_name = "authenc-hmac-sha1-" 1806 "cbc-aes-caam-qi2", 1807 .cra_blocksize = AES_BLOCK_SIZE, 1808 }, 1809 .setkey = aead_setkey, 1810 .setauthsize = aead_setauthsize, 1811 .encrypt = aead_encrypt, 1812 .decrypt = aead_decrypt, 1813 .ivsize = AES_BLOCK_SIZE, 1814 .maxauthsize = SHA1_DIGEST_SIZE, 1815 }, 1816 .caam = { 1817 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1818 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1819 OP_ALG_AAI_HMAC_PRECOMP, 1820 } 1821 }, 1822 { 1823 .aead = { 1824 .base = { 1825 .cra_name = "echainiv(authenc(hmac(sha1)," 1826 "cbc(aes)))", 1827 .cra_driver_name = "echainiv-authenc-" 1828 "hmac-sha1-cbc-aes-caam-qi2", 1829 .cra_blocksize = AES_BLOCK_SIZE, 1830 }, 1831 .setkey = aead_setkey, 1832 .setauthsize = aead_setauthsize, 1833 .encrypt = aead_encrypt, 1834 .decrypt = aead_decrypt, 1835 .ivsize = AES_BLOCK_SIZE, 1836 .maxauthsize = SHA1_DIGEST_SIZE, 1837 }, 1838 .caam = { 1839 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1840 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1841 OP_ALG_AAI_HMAC_PRECOMP, 1842 .geniv = true, 1843 }, 1844 }, 1845 { 1846 .aead = { 1847 .base = { 1848 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1849 .cra_driver_name = "authenc-hmac-sha224-" 1850 "cbc-aes-caam-qi2", 1851 .cra_blocksize = AES_BLOCK_SIZE, 1852 }, 1853 .setkey = aead_setkey, 1854 .setauthsize = aead_setauthsize, 1855 .encrypt = aead_encrypt, 1856 .decrypt = aead_decrypt, 1857 .ivsize = AES_BLOCK_SIZE, 1858 .maxauthsize = SHA224_DIGEST_SIZE, 1859 }, 1860 .caam = { 1861 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1862 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1863 OP_ALG_AAI_HMAC_PRECOMP, 1864 } 1865 }, 1866 { 1867 .aead = { 1868 .base = { 1869 .cra_name = "echainiv(authenc(hmac(sha224)," 1870 "cbc(aes)))", 1871 .cra_driver_name = "echainiv-authenc-" 1872 "hmac-sha224-cbc-aes-caam-qi2", 1873 .cra_blocksize = AES_BLOCK_SIZE, 1874 }, 1875 .setkey = aead_setkey, 1876 .setauthsize = aead_setauthsize, 1877 .encrypt = aead_encrypt, 1878 .decrypt = aead_decrypt, 1879 .ivsize = AES_BLOCK_SIZE, 1880 .maxauthsize = SHA224_DIGEST_SIZE, 1881 }, 1882 .caam = { 1883 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1884 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1885 OP_ALG_AAI_HMAC_PRECOMP, 1886 .geniv = true, 1887 } 1888 }, 1889 { 1890 .aead = { 1891 .base = { 1892 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1893 .cra_driver_name = "authenc-hmac-sha256-" 1894 "cbc-aes-caam-qi2", 1895 .cra_blocksize = AES_BLOCK_SIZE, 1896 }, 1897 .setkey = aead_setkey, 1898 .setauthsize = aead_setauthsize, 1899 .encrypt = aead_encrypt, 1900 .decrypt = aead_decrypt, 1901 .ivsize = AES_BLOCK_SIZE, 1902 .maxauthsize = SHA256_DIGEST_SIZE, 1903 }, 1904 .caam = { 1905 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1906 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1907 OP_ALG_AAI_HMAC_PRECOMP, 1908 } 1909 }, 1910 { 1911 .aead = { 1912 .base = { 1913 .cra_name = "echainiv(authenc(hmac(sha256)," 1914 "cbc(aes)))", 1915 .cra_driver_name = "echainiv-authenc-" 1916 "hmac-sha256-cbc-aes-" 1917 "caam-qi2", 1918 .cra_blocksize = AES_BLOCK_SIZE, 1919 }, 1920 .setkey = aead_setkey, 1921 .setauthsize = aead_setauthsize, 1922 .encrypt = aead_encrypt, 1923 .decrypt = aead_decrypt, 1924 .ivsize = AES_BLOCK_SIZE, 1925 .maxauthsize = SHA256_DIGEST_SIZE, 1926 }, 1927 .caam = { 1928 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1929 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 1930 OP_ALG_AAI_HMAC_PRECOMP, 1931 .geniv = true, 1932 } 1933 }, 1934 { 1935 .aead = { 1936 .base = { 1937 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1938 .cra_driver_name = "authenc-hmac-sha384-" 1939 "cbc-aes-caam-qi2", 1940 .cra_blocksize = AES_BLOCK_SIZE, 1941 }, 1942 .setkey = aead_setkey, 1943 .setauthsize = aead_setauthsize, 1944 .encrypt = aead_encrypt, 1945 .decrypt = aead_decrypt, 1946 .ivsize = AES_BLOCK_SIZE, 1947 .maxauthsize = SHA384_DIGEST_SIZE, 1948 }, 1949 .caam = { 1950 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1951 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1952 OP_ALG_AAI_HMAC_PRECOMP, 1953 } 1954 }, 1955 { 1956 .aead = { 1957 .base = { 1958 .cra_name = "echainiv(authenc(hmac(sha384)," 1959 "cbc(aes)))", 1960 .cra_driver_name = "echainiv-authenc-" 1961 "hmac-sha384-cbc-aes-" 1962 "caam-qi2", 1963 .cra_blocksize = AES_BLOCK_SIZE, 1964 }, 1965 .setkey = aead_setkey, 1966 .setauthsize = aead_setauthsize, 1967 .encrypt = aead_encrypt, 1968 .decrypt = aead_decrypt, 1969 .ivsize = AES_BLOCK_SIZE, 1970 .maxauthsize = SHA384_DIGEST_SIZE, 1971 }, 1972 .caam = { 1973 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1974 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 1975 OP_ALG_AAI_HMAC_PRECOMP, 1976 .geniv = true, 1977 } 1978 }, 1979 { 1980 .aead = { 1981 .base = { 1982 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1983 .cra_driver_name = "authenc-hmac-sha512-" 1984 "cbc-aes-caam-qi2", 1985 .cra_blocksize = AES_BLOCK_SIZE, 1986 }, 1987 .setkey = aead_setkey, 1988 .setauthsize = aead_setauthsize, 1989 .encrypt = aead_encrypt, 1990 .decrypt = aead_decrypt, 1991 .ivsize = AES_BLOCK_SIZE, 1992 .maxauthsize = SHA512_DIGEST_SIZE, 1993 }, 1994 .caam = { 1995 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1996 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 1997 OP_ALG_AAI_HMAC_PRECOMP, 1998 } 1999 }, 2000 { 2001 .aead = { 2002 .base = { 2003 .cra_name = "echainiv(authenc(hmac(sha512)," 2004 "cbc(aes)))", 2005 .cra_driver_name = "echainiv-authenc-" 2006 "hmac-sha512-cbc-aes-" 2007 "caam-qi2", 2008 .cra_blocksize = AES_BLOCK_SIZE, 2009 }, 2010 .setkey = aead_setkey, 2011 .setauthsize = aead_setauthsize, 2012 .encrypt = aead_encrypt, 2013 .decrypt = aead_decrypt, 2014 .ivsize = AES_BLOCK_SIZE, 2015 .maxauthsize = SHA512_DIGEST_SIZE, 2016 }, 2017 .caam = { 2018 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2019 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2020 OP_ALG_AAI_HMAC_PRECOMP, 2021 .geniv = true, 2022 } 2023 }, 2024 { 2025 .aead = { 2026 .base = { 2027 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2028 .cra_driver_name = "authenc-hmac-md5-" 2029 "cbc-des3_ede-caam-qi2", 2030 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2031 }, 2032 .setkey = des3_aead_setkey, 2033 .setauthsize = aead_setauthsize, 2034 .encrypt = aead_encrypt, 2035 .decrypt = aead_decrypt, 2036 .ivsize = DES3_EDE_BLOCK_SIZE, 2037 .maxauthsize = MD5_DIGEST_SIZE, 2038 }, 2039 .caam = { 2040 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2041 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2042 OP_ALG_AAI_HMAC_PRECOMP, 2043 } 2044 }, 2045 { 2046 .aead = { 2047 .base = { 2048 .cra_name = "echainiv(authenc(hmac(md5)," 2049 "cbc(des3_ede)))", 2050 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2051 "cbc-des3_ede-caam-qi2", 2052 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2053 }, 2054 .setkey = des3_aead_setkey, 2055 .setauthsize = aead_setauthsize, 2056 .encrypt = aead_encrypt, 2057 .decrypt = aead_decrypt, 2058 .ivsize = DES3_EDE_BLOCK_SIZE, 2059 .maxauthsize = MD5_DIGEST_SIZE, 2060 }, 2061 .caam = { 2062 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2063 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2064 OP_ALG_AAI_HMAC_PRECOMP, 2065 .geniv = true, 2066 } 2067 }, 2068 { 2069 .aead = { 2070 .base = { 2071 .cra_name = "authenc(hmac(sha1)," 2072 "cbc(des3_ede))", 2073 .cra_driver_name = "authenc-hmac-sha1-" 2074 "cbc-des3_ede-caam-qi2", 2075 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2076 }, 2077 .setkey = des3_aead_setkey, 2078 .setauthsize = aead_setauthsize, 2079 .encrypt = aead_encrypt, 2080 .decrypt = aead_decrypt, 2081 .ivsize = DES3_EDE_BLOCK_SIZE, 2082 .maxauthsize = SHA1_DIGEST_SIZE, 2083 }, 2084 .caam = { 2085 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2086 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2087 OP_ALG_AAI_HMAC_PRECOMP, 2088 }, 2089 }, 2090 { 2091 .aead = { 2092 .base = { 2093 .cra_name = "echainiv(authenc(hmac(sha1)," 2094 "cbc(des3_ede)))", 2095 .cra_driver_name = "echainiv-authenc-" 2096 "hmac-sha1-" 2097 "cbc-des3_ede-caam-qi2", 2098 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2099 }, 2100 .setkey = des3_aead_setkey, 2101 .setauthsize = aead_setauthsize, 2102 .encrypt = aead_encrypt, 2103 .decrypt = aead_decrypt, 2104 .ivsize = DES3_EDE_BLOCK_SIZE, 2105 .maxauthsize = SHA1_DIGEST_SIZE, 2106 }, 2107 .caam = { 2108 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2109 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2110 OP_ALG_AAI_HMAC_PRECOMP, 2111 .geniv = true, 2112 } 2113 }, 2114 { 2115 .aead = { 2116 .base = { 2117 .cra_name = "authenc(hmac(sha224)," 2118 "cbc(des3_ede))", 2119 .cra_driver_name = "authenc-hmac-sha224-" 2120 "cbc-des3_ede-caam-qi2", 2121 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2122 }, 2123 .setkey = des3_aead_setkey, 2124 .setauthsize = aead_setauthsize, 2125 .encrypt = aead_encrypt, 2126 .decrypt = aead_decrypt, 2127 .ivsize = DES3_EDE_BLOCK_SIZE, 2128 .maxauthsize = SHA224_DIGEST_SIZE, 2129 }, 2130 .caam = { 2131 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2132 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2133 OP_ALG_AAI_HMAC_PRECOMP, 2134 }, 2135 }, 2136 { 2137 .aead = { 2138 .base = { 2139 .cra_name = "echainiv(authenc(hmac(sha224)," 2140 "cbc(des3_ede)))", 2141 .cra_driver_name = "echainiv-authenc-" 2142 "hmac-sha224-" 2143 "cbc-des3_ede-caam-qi2", 2144 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2145 }, 2146 .setkey = des3_aead_setkey, 2147 .setauthsize = aead_setauthsize, 2148 .encrypt = aead_encrypt, 2149 .decrypt = aead_decrypt, 2150 .ivsize = DES3_EDE_BLOCK_SIZE, 2151 .maxauthsize = SHA224_DIGEST_SIZE, 2152 }, 2153 .caam = { 2154 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2155 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2156 OP_ALG_AAI_HMAC_PRECOMP, 2157 .geniv = true, 2158 } 2159 }, 2160 { 2161 .aead = { 2162 .base = { 2163 .cra_name = "authenc(hmac(sha256)," 2164 "cbc(des3_ede))", 2165 .cra_driver_name = "authenc-hmac-sha256-" 2166 "cbc-des3_ede-caam-qi2", 2167 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2168 }, 2169 .setkey = des3_aead_setkey, 2170 .setauthsize = aead_setauthsize, 2171 .encrypt = aead_encrypt, 2172 .decrypt = aead_decrypt, 2173 .ivsize = DES3_EDE_BLOCK_SIZE, 2174 .maxauthsize = SHA256_DIGEST_SIZE, 2175 }, 2176 .caam = { 2177 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2178 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2179 OP_ALG_AAI_HMAC_PRECOMP, 2180 }, 2181 }, 2182 { 2183 .aead = { 2184 .base = { 2185 .cra_name = "echainiv(authenc(hmac(sha256)," 2186 "cbc(des3_ede)))", 2187 .cra_driver_name = "echainiv-authenc-" 2188 "hmac-sha256-" 2189 "cbc-des3_ede-caam-qi2", 2190 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2191 }, 2192 .setkey = des3_aead_setkey, 2193 .setauthsize = aead_setauthsize, 2194 .encrypt = aead_encrypt, 2195 .decrypt = aead_decrypt, 2196 .ivsize = DES3_EDE_BLOCK_SIZE, 2197 .maxauthsize = SHA256_DIGEST_SIZE, 2198 }, 2199 .caam = { 2200 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2201 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2202 OP_ALG_AAI_HMAC_PRECOMP, 2203 .geniv = true, 2204 } 2205 }, 2206 { 2207 .aead = { 2208 .base = { 2209 .cra_name = "authenc(hmac(sha384)," 2210 "cbc(des3_ede))", 2211 .cra_driver_name = "authenc-hmac-sha384-" 2212 "cbc-des3_ede-caam-qi2", 2213 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2214 }, 2215 .setkey = des3_aead_setkey, 2216 .setauthsize = aead_setauthsize, 2217 .encrypt = aead_encrypt, 2218 .decrypt = aead_decrypt, 2219 .ivsize = DES3_EDE_BLOCK_SIZE, 2220 .maxauthsize = SHA384_DIGEST_SIZE, 2221 }, 2222 .caam = { 2223 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2224 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2225 OP_ALG_AAI_HMAC_PRECOMP, 2226 }, 2227 }, 2228 { 2229 .aead = { 2230 .base = { 2231 .cra_name = "echainiv(authenc(hmac(sha384)," 2232 "cbc(des3_ede)))", 2233 .cra_driver_name = "echainiv-authenc-" 2234 "hmac-sha384-" 2235 "cbc-des3_ede-caam-qi2", 2236 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2237 }, 2238 .setkey = des3_aead_setkey, 2239 .setauthsize = aead_setauthsize, 2240 .encrypt = aead_encrypt, 2241 .decrypt = aead_decrypt, 2242 .ivsize = DES3_EDE_BLOCK_SIZE, 2243 .maxauthsize = SHA384_DIGEST_SIZE, 2244 }, 2245 .caam = { 2246 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2247 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2248 OP_ALG_AAI_HMAC_PRECOMP, 2249 .geniv = true, 2250 } 2251 }, 2252 { 2253 .aead = { 2254 .base = { 2255 .cra_name = "authenc(hmac(sha512)," 2256 "cbc(des3_ede))", 2257 .cra_driver_name = "authenc-hmac-sha512-" 2258 "cbc-des3_ede-caam-qi2", 2259 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2260 }, 2261 .setkey = des3_aead_setkey, 2262 .setauthsize = aead_setauthsize, 2263 .encrypt = aead_encrypt, 2264 .decrypt = aead_decrypt, 2265 .ivsize = DES3_EDE_BLOCK_SIZE, 2266 .maxauthsize = SHA512_DIGEST_SIZE, 2267 }, 2268 .caam = { 2269 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2270 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2271 OP_ALG_AAI_HMAC_PRECOMP, 2272 }, 2273 }, 2274 { 2275 .aead = { 2276 .base = { 2277 .cra_name = "echainiv(authenc(hmac(sha512)," 2278 "cbc(des3_ede)))", 2279 .cra_driver_name = "echainiv-authenc-" 2280 "hmac-sha512-" 2281 "cbc-des3_ede-caam-qi2", 2282 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2283 }, 2284 .setkey = des3_aead_setkey, 2285 .setauthsize = aead_setauthsize, 2286 .encrypt = aead_encrypt, 2287 .decrypt = aead_decrypt, 2288 .ivsize = DES3_EDE_BLOCK_SIZE, 2289 .maxauthsize = SHA512_DIGEST_SIZE, 2290 }, 2291 .caam = { 2292 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2293 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2294 OP_ALG_AAI_HMAC_PRECOMP, 2295 .geniv = true, 2296 } 2297 }, 2298 { 2299 .aead = { 2300 .base = { 2301 .cra_name = "authenc(hmac(md5),cbc(des))", 2302 .cra_driver_name = "authenc-hmac-md5-" 2303 "cbc-des-caam-qi2", 2304 .cra_blocksize = DES_BLOCK_SIZE, 2305 }, 2306 .setkey = aead_setkey, 2307 .setauthsize = aead_setauthsize, 2308 .encrypt = aead_encrypt, 2309 .decrypt = aead_decrypt, 2310 .ivsize = DES_BLOCK_SIZE, 2311 .maxauthsize = MD5_DIGEST_SIZE, 2312 }, 2313 .caam = { 2314 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2315 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2316 OP_ALG_AAI_HMAC_PRECOMP, 2317 }, 2318 }, 2319 { 2320 .aead = { 2321 .base = { 2322 .cra_name = "echainiv(authenc(hmac(md5)," 2323 "cbc(des)))", 2324 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2325 "cbc-des-caam-qi2", 2326 .cra_blocksize = DES_BLOCK_SIZE, 2327 }, 2328 .setkey = aead_setkey, 2329 .setauthsize = aead_setauthsize, 2330 .encrypt = aead_encrypt, 2331 .decrypt = aead_decrypt, 2332 .ivsize = DES_BLOCK_SIZE, 2333 .maxauthsize = MD5_DIGEST_SIZE, 2334 }, 2335 .caam = { 2336 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2337 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2338 OP_ALG_AAI_HMAC_PRECOMP, 2339 .geniv = true, 2340 } 2341 }, 2342 { 2343 .aead = { 2344 .base = { 2345 .cra_name = "authenc(hmac(sha1),cbc(des))", 2346 .cra_driver_name = "authenc-hmac-sha1-" 2347 "cbc-des-caam-qi2", 2348 .cra_blocksize = DES_BLOCK_SIZE, 2349 }, 2350 .setkey = aead_setkey, 2351 .setauthsize = aead_setauthsize, 2352 .encrypt = aead_encrypt, 2353 .decrypt = aead_decrypt, 2354 .ivsize = DES_BLOCK_SIZE, 2355 .maxauthsize = SHA1_DIGEST_SIZE, 2356 }, 2357 .caam = { 2358 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2359 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2360 OP_ALG_AAI_HMAC_PRECOMP, 2361 }, 2362 }, 2363 { 2364 .aead = { 2365 .base = { 2366 .cra_name = "echainiv(authenc(hmac(sha1)," 2367 "cbc(des)))", 2368 .cra_driver_name = "echainiv-authenc-" 2369 "hmac-sha1-cbc-des-caam-qi2", 2370 .cra_blocksize = DES_BLOCK_SIZE, 2371 }, 2372 .setkey = aead_setkey, 2373 .setauthsize = aead_setauthsize, 2374 .encrypt = aead_encrypt, 2375 .decrypt = aead_decrypt, 2376 .ivsize = DES_BLOCK_SIZE, 2377 .maxauthsize = SHA1_DIGEST_SIZE, 2378 }, 2379 .caam = { 2380 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2381 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2382 OP_ALG_AAI_HMAC_PRECOMP, 2383 .geniv = true, 2384 } 2385 }, 2386 { 2387 .aead = { 2388 .base = { 2389 .cra_name = "authenc(hmac(sha224),cbc(des))", 2390 .cra_driver_name = "authenc-hmac-sha224-" 2391 "cbc-des-caam-qi2", 2392 .cra_blocksize = DES_BLOCK_SIZE, 2393 }, 2394 .setkey = aead_setkey, 2395 .setauthsize = aead_setauthsize, 2396 .encrypt = aead_encrypt, 2397 .decrypt = aead_decrypt, 2398 .ivsize = DES_BLOCK_SIZE, 2399 .maxauthsize = SHA224_DIGEST_SIZE, 2400 }, 2401 .caam = { 2402 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2403 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2404 OP_ALG_AAI_HMAC_PRECOMP, 2405 }, 2406 }, 2407 { 2408 .aead = { 2409 .base = { 2410 .cra_name = "echainiv(authenc(hmac(sha224)," 2411 "cbc(des)))", 2412 .cra_driver_name = "echainiv-authenc-" 2413 "hmac-sha224-cbc-des-" 2414 "caam-qi2", 2415 .cra_blocksize = DES_BLOCK_SIZE, 2416 }, 2417 .setkey = aead_setkey, 2418 .setauthsize = aead_setauthsize, 2419 .encrypt = aead_encrypt, 2420 .decrypt = aead_decrypt, 2421 .ivsize = DES_BLOCK_SIZE, 2422 .maxauthsize = SHA224_DIGEST_SIZE, 2423 }, 2424 .caam = { 2425 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2426 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2427 OP_ALG_AAI_HMAC_PRECOMP, 2428 .geniv = true, 2429 } 2430 }, 2431 { 2432 .aead = { 2433 .base = { 2434 .cra_name = "authenc(hmac(sha256),cbc(des))", 2435 .cra_driver_name = "authenc-hmac-sha256-" 2436 "cbc-des-caam-qi2", 2437 .cra_blocksize = DES_BLOCK_SIZE, 2438 }, 2439 .setkey = aead_setkey, 2440 .setauthsize = aead_setauthsize, 2441 .encrypt = aead_encrypt, 2442 .decrypt = aead_decrypt, 2443 .ivsize = DES_BLOCK_SIZE, 2444 .maxauthsize = SHA256_DIGEST_SIZE, 2445 }, 2446 .caam = { 2447 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2448 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2449 OP_ALG_AAI_HMAC_PRECOMP, 2450 }, 2451 }, 2452 { 2453 .aead = { 2454 .base = { 2455 .cra_name = "echainiv(authenc(hmac(sha256)," 2456 "cbc(des)))", 2457 .cra_driver_name = "echainiv-authenc-" 2458 "hmac-sha256-cbc-des-" 2459 "caam-qi2", 2460 .cra_blocksize = DES_BLOCK_SIZE, 2461 }, 2462 .setkey = aead_setkey, 2463 .setauthsize = aead_setauthsize, 2464 .encrypt = aead_encrypt, 2465 .decrypt = aead_decrypt, 2466 .ivsize = DES_BLOCK_SIZE, 2467 .maxauthsize = SHA256_DIGEST_SIZE, 2468 }, 2469 .caam = { 2470 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2471 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2472 OP_ALG_AAI_HMAC_PRECOMP, 2473 .geniv = true, 2474 }, 2475 }, 2476 { 2477 .aead = { 2478 .base = { 2479 .cra_name = "authenc(hmac(sha384),cbc(des))", 2480 .cra_driver_name = "authenc-hmac-sha384-" 2481 "cbc-des-caam-qi2", 2482 .cra_blocksize = DES_BLOCK_SIZE, 2483 }, 2484 .setkey = aead_setkey, 2485 .setauthsize = aead_setauthsize, 2486 .encrypt = aead_encrypt, 2487 .decrypt = aead_decrypt, 2488 .ivsize = DES_BLOCK_SIZE, 2489 .maxauthsize = SHA384_DIGEST_SIZE, 2490 }, 2491 .caam = { 2492 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2493 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2494 OP_ALG_AAI_HMAC_PRECOMP, 2495 }, 2496 }, 2497 { 2498 .aead = { 2499 .base = { 2500 .cra_name = "echainiv(authenc(hmac(sha384)," 2501 "cbc(des)))", 2502 .cra_driver_name = "echainiv-authenc-" 2503 "hmac-sha384-cbc-des-" 2504 "caam-qi2", 2505 .cra_blocksize = DES_BLOCK_SIZE, 2506 }, 2507 .setkey = aead_setkey, 2508 .setauthsize = aead_setauthsize, 2509 .encrypt = aead_encrypt, 2510 .decrypt = aead_decrypt, 2511 .ivsize = DES_BLOCK_SIZE, 2512 .maxauthsize = SHA384_DIGEST_SIZE, 2513 }, 2514 .caam = { 2515 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2516 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2517 OP_ALG_AAI_HMAC_PRECOMP, 2518 .geniv = true, 2519 } 2520 }, 2521 { 2522 .aead = { 2523 .base = { 2524 .cra_name = "authenc(hmac(sha512),cbc(des))", 2525 .cra_driver_name = "authenc-hmac-sha512-" 2526 "cbc-des-caam-qi2", 2527 .cra_blocksize = DES_BLOCK_SIZE, 2528 }, 2529 .setkey = aead_setkey, 2530 .setauthsize = aead_setauthsize, 2531 .encrypt = aead_encrypt, 2532 .decrypt = aead_decrypt, 2533 .ivsize = DES_BLOCK_SIZE, 2534 .maxauthsize = SHA512_DIGEST_SIZE, 2535 }, 2536 .caam = { 2537 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2538 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2539 OP_ALG_AAI_HMAC_PRECOMP, 2540 } 2541 }, 2542 { 2543 .aead = { 2544 .base = { 2545 .cra_name = "echainiv(authenc(hmac(sha512)," 2546 "cbc(des)))", 2547 .cra_driver_name = "echainiv-authenc-" 2548 "hmac-sha512-cbc-des-" 2549 "caam-qi2", 2550 .cra_blocksize = DES_BLOCK_SIZE, 2551 }, 2552 .setkey = aead_setkey, 2553 .setauthsize = aead_setauthsize, 2554 .encrypt = aead_encrypt, 2555 .decrypt = aead_decrypt, 2556 .ivsize = DES_BLOCK_SIZE, 2557 .maxauthsize = SHA512_DIGEST_SIZE, 2558 }, 2559 .caam = { 2560 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2561 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2562 OP_ALG_AAI_HMAC_PRECOMP, 2563 .geniv = true, 2564 } 2565 }, 2566 { 2567 .aead = { 2568 .base = { 2569 .cra_name = "authenc(hmac(md5)," 2570 "rfc3686(ctr(aes)))", 2571 .cra_driver_name = "authenc-hmac-md5-" 2572 "rfc3686-ctr-aes-caam-qi2", 2573 .cra_blocksize = 1, 2574 }, 2575 .setkey = aead_setkey, 2576 .setauthsize = aead_setauthsize, 2577 .encrypt = aead_encrypt, 2578 .decrypt = aead_decrypt, 2579 .ivsize = CTR_RFC3686_IV_SIZE, 2580 .maxauthsize = MD5_DIGEST_SIZE, 2581 }, 2582 .caam = { 2583 .class1_alg_type = OP_ALG_ALGSEL_AES | 2584 OP_ALG_AAI_CTR_MOD128, 2585 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2586 OP_ALG_AAI_HMAC_PRECOMP, 2587 .rfc3686 = true, 2588 }, 2589 }, 2590 { 2591 .aead = { 2592 .base = { 2593 .cra_name = "seqiv(authenc(" 2594 "hmac(md5),rfc3686(ctr(aes))))", 2595 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2596 "rfc3686-ctr-aes-caam-qi2", 2597 .cra_blocksize = 1, 2598 }, 2599 .setkey = aead_setkey, 2600 .setauthsize = aead_setauthsize, 2601 .encrypt = aead_encrypt, 2602 .decrypt = aead_decrypt, 2603 .ivsize = CTR_RFC3686_IV_SIZE, 2604 .maxauthsize = MD5_DIGEST_SIZE, 2605 }, 2606 .caam = { 2607 .class1_alg_type = OP_ALG_ALGSEL_AES | 2608 OP_ALG_AAI_CTR_MOD128, 2609 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2610 OP_ALG_AAI_HMAC_PRECOMP, 2611 .rfc3686 = true, 2612 .geniv = true, 2613 }, 2614 }, 2615 { 2616 .aead = { 2617 .base = { 2618 .cra_name = "authenc(hmac(sha1)," 2619 "rfc3686(ctr(aes)))", 2620 .cra_driver_name = "authenc-hmac-sha1-" 2621 "rfc3686-ctr-aes-caam-qi2", 2622 .cra_blocksize = 1, 2623 }, 2624 .setkey = aead_setkey, 2625 .setauthsize = aead_setauthsize, 2626 .encrypt = aead_encrypt, 2627 .decrypt = aead_decrypt, 2628 .ivsize = CTR_RFC3686_IV_SIZE, 2629 .maxauthsize = SHA1_DIGEST_SIZE, 2630 }, 2631 .caam = { 2632 .class1_alg_type = OP_ALG_ALGSEL_AES | 2633 OP_ALG_AAI_CTR_MOD128, 2634 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2635 OP_ALG_AAI_HMAC_PRECOMP, 2636 .rfc3686 = true, 2637 }, 2638 }, 2639 { 2640 .aead = { 2641 .base = { 2642 .cra_name = "seqiv(authenc(" 2643 "hmac(sha1),rfc3686(ctr(aes))))", 2644 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2645 "rfc3686-ctr-aes-caam-qi2", 2646 .cra_blocksize = 1, 2647 }, 2648 .setkey = aead_setkey, 2649 .setauthsize = aead_setauthsize, 2650 .encrypt = aead_encrypt, 2651 .decrypt = aead_decrypt, 2652 .ivsize = CTR_RFC3686_IV_SIZE, 2653 .maxauthsize = SHA1_DIGEST_SIZE, 2654 }, 2655 .caam = { 2656 .class1_alg_type = OP_ALG_ALGSEL_AES | 2657 OP_ALG_AAI_CTR_MOD128, 2658 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2659 OP_ALG_AAI_HMAC_PRECOMP, 2660 .rfc3686 = true, 2661 .geniv = true, 2662 }, 2663 }, 2664 { 2665 .aead = { 2666 .base = { 2667 .cra_name = "authenc(hmac(sha224)," 2668 "rfc3686(ctr(aes)))", 2669 .cra_driver_name = "authenc-hmac-sha224-" 2670 "rfc3686-ctr-aes-caam-qi2", 2671 .cra_blocksize = 1, 2672 }, 2673 .setkey = aead_setkey, 2674 .setauthsize = aead_setauthsize, 2675 .encrypt = aead_encrypt, 2676 .decrypt = aead_decrypt, 2677 .ivsize = CTR_RFC3686_IV_SIZE, 2678 .maxauthsize = SHA224_DIGEST_SIZE, 2679 }, 2680 .caam = { 2681 .class1_alg_type = OP_ALG_ALGSEL_AES | 2682 OP_ALG_AAI_CTR_MOD128, 2683 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2684 OP_ALG_AAI_HMAC_PRECOMP, 2685 .rfc3686 = true, 2686 }, 2687 }, 2688 { 2689 .aead = { 2690 .base = { 2691 .cra_name = "seqiv(authenc(" 2692 "hmac(sha224),rfc3686(ctr(aes))))", 2693 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2694 "rfc3686-ctr-aes-caam-qi2", 2695 .cra_blocksize = 1, 2696 }, 2697 .setkey = aead_setkey, 2698 .setauthsize = aead_setauthsize, 2699 .encrypt = aead_encrypt, 2700 .decrypt = aead_decrypt, 2701 .ivsize = CTR_RFC3686_IV_SIZE, 2702 .maxauthsize = SHA224_DIGEST_SIZE, 2703 }, 2704 .caam = { 2705 .class1_alg_type = OP_ALG_ALGSEL_AES | 2706 OP_ALG_AAI_CTR_MOD128, 2707 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2708 OP_ALG_AAI_HMAC_PRECOMP, 2709 .rfc3686 = true, 2710 .geniv = true, 2711 }, 2712 }, 2713 { 2714 .aead = { 2715 .base = { 2716 .cra_name = "authenc(hmac(sha256)," 2717 "rfc3686(ctr(aes)))", 2718 .cra_driver_name = "authenc-hmac-sha256-" 2719 "rfc3686-ctr-aes-caam-qi2", 2720 .cra_blocksize = 1, 2721 }, 2722 .setkey = aead_setkey, 2723 .setauthsize = aead_setauthsize, 2724 .encrypt = aead_encrypt, 2725 .decrypt = aead_decrypt, 2726 .ivsize = CTR_RFC3686_IV_SIZE, 2727 .maxauthsize = SHA256_DIGEST_SIZE, 2728 }, 2729 .caam = { 2730 .class1_alg_type = OP_ALG_ALGSEL_AES | 2731 OP_ALG_AAI_CTR_MOD128, 2732 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2733 OP_ALG_AAI_HMAC_PRECOMP, 2734 .rfc3686 = true, 2735 }, 2736 }, 2737 { 2738 .aead = { 2739 .base = { 2740 .cra_name = "seqiv(authenc(hmac(sha256)," 2741 "rfc3686(ctr(aes))))", 2742 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2743 "rfc3686-ctr-aes-caam-qi2", 2744 .cra_blocksize = 1, 2745 }, 2746 .setkey = aead_setkey, 2747 .setauthsize = aead_setauthsize, 2748 .encrypt = aead_encrypt, 2749 .decrypt = aead_decrypt, 2750 .ivsize = CTR_RFC3686_IV_SIZE, 2751 .maxauthsize = SHA256_DIGEST_SIZE, 2752 }, 2753 .caam = { 2754 .class1_alg_type = OP_ALG_ALGSEL_AES | 2755 OP_ALG_AAI_CTR_MOD128, 2756 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2757 OP_ALG_AAI_HMAC_PRECOMP, 2758 .rfc3686 = true, 2759 .geniv = true, 2760 }, 2761 }, 2762 { 2763 .aead = { 2764 .base = { 2765 .cra_name = "authenc(hmac(sha384)," 2766 "rfc3686(ctr(aes)))", 2767 .cra_driver_name = "authenc-hmac-sha384-" 2768 "rfc3686-ctr-aes-caam-qi2", 2769 .cra_blocksize = 1, 2770 }, 2771 .setkey = aead_setkey, 2772 .setauthsize = aead_setauthsize, 2773 .encrypt = aead_encrypt, 2774 .decrypt = aead_decrypt, 2775 .ivsize = CTR_RFC3686_IV_SIZE, 2776 .maxauthsize = SHA384_DIGEST_SIZE, 2777 }, 2778 .caam = { 2779 .class1_alg_type = OP_ALG_ALGSEL_AES | 2780 OP_ALG_AAI_CTR_MOD128, 2781 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2782 OP_ALG_AAI_HMAC_PRECOMP, 2783 .rfc3686 = true, 2784 }, 2785 }, 2786 { 2787 .aead = { 2788 .base = { 2789 .cra_name = "seqiv(authenc(hmac(sha384)," 2790 "rfc3686(ctr(aes))))", 2791 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2792 "rfc3686-ctr-aes-caam-qi2", 2793 .cra_blocksize = 1, 2794 }, 2795 .setkey = aead_setkey, 2796 .setauthsize = aead_setauthsize, 2797 .encrypt = aead_encrypt, 2798 .decrypt = aead_decrypt, 2799 .ivsize = CTR_RFC3686_IV_SIZE, 2800 .maxauthsize = SHA384_DIGEST_SIZE, 2801 }, 2802 .caam = { 2803 .class1_alg_type = OP_ALG_ALGSEL_AES | 2804 OP_ALG_AAI_CTR_MOD128, 2805 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2806 OP_ALG_AAI_HMAC_PRECOMP, 2807 .rfc3686 = true, 2808 .geniv = true, 2809 }, 2810 }, 2811 { 2812 .aead = { 2813 .base = { 2814 .cra_name = "rfc7539(chacha20,poly1305)", 2815 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2816 "caam-qi2", 2817 .cra_blocksize = 1, 2818 }, 2819 .setkey = chachapoly_setkey, 2820 .setauthsize = chachapoly_setauthsize, 2821 .encrypt = aead_encrypt, 2822 .decrypt = aead_decrypt, 2823 .ivsize = CHACHAPOLY_IV_SIZE, 2824 .maxauthsize = POLY1305_DIGEST_SIZE, 2825 }, 2826 .caam = { 2827 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2828 OP_ALG_AAI_AEAD, 2829 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2830 OP_ALG_AAI_AEAD, 2831 .nodkp = true, 2832 }, 2833 }, 2834 { 2835 .aead = { 2836 .base = { 2837 .cra_name = "rfc7539esp(chacha20,poly1305)", 2838 .cra_driver_name = "rfc7539esp-chacha20-" 2839 "poly1305-caam-qi2", 2840 .cra_blocksize = 1, 2841 }, 2842 .setkey = chachapoly_setkey, 2843 .setauthsize = chachapoly_setauthsize, 2844 .encrypt = aead_encrypt, 2845 .decrypt = aead_decrypt, 2846 .ivsize = 8, 2847 .maxauthsize = POLY1305_DIGEST_SIZE, 2848 }, 2849 .caam = { 2850 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2851 OP_ALG_AAI_AEAD, 2852 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2853 OP_ALG_AAI_AEAD, 2854 .nodkp = true, 2855 }, 2856 }, 2857 { 2858 .aead = { 2859 .base = { 2860 .cra_name = "authenc(hmac(sha512)," 2861 "rfc3686(ctr(aes)))", 2862 .cra_driver_name = "authenc-hmac-sha512-" 2863 "rfc3686-ctr-aes-caam-qi2", 2864 .cra_blocksize = 1, 2865 }, 2866 .setkey = aead_setkey, 2867 .setauthsize = aead_setauthsize, 2868 .encrypt = aead_encrypt, 2869 .decrypt = aead_decrypt, 2870 .ivsize = CTR_RFC3686_IV_SIZE, 2871 .maxauthsize = SHA512_DIGEST_SIZE, 2872 }, 2873 .caam = { 2874 .class1_alg_type = OP_ALG_ALGSEL_AES | 2875 OP_ALG_AAI_CTR_MOD128, 2876 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2877 OP_ALG_AAI_HMAC_PRECOMP, 2878 .rfc3686 = true, 2879 }, 2880 }, 2881 { 2882 .aead = { 2883 .base = { 2884 .cra_name = "seqiv(authenc(hmac(sha512)," 2885 "rfc3686(ctr(aes))))", 2886 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2887 "rfc3686-ctr-aes-caam-qi2", 2888 .cra_blocksize = 1, 2889 }, 2890 .setkey = aead_setkey, 2891 .setauthsize = aead_setauthsize, 2892 .encrypt = aead_encrypt, 2893 .decrypt = aead_decrypt, 2894 .ivsize = CTR_RFC3686_IV_SIZE, 2895 .maxauthsize = SHA512_DIGEST_SIZE, 2896 }, 2897 .caam = { 2898 .class1_alg_type = OP_ALG_ALGSEL_AES | 2899 OP_ALG_AAI_CTR_MOD128, 2900 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2901 OP_ALG_AAI_HMAC_PRECOMP, 2902 .rfc3686 = true, 2903 .geniv = true, 2904 }, 2905 }, 2906 }; 2907 2908 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 2909 { 2910 struct skcipher_alg *alg = &t_alg->skcipher; 2911 2912 alg->base.cra_module = THIS_MODULE; 2913 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2914 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2915 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2916 2917 alg->init = caam_cra_init_skcipher; 2918 alg->exit = caam_cra_exit; 2919 } 2920 2921 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 2922 { 2923 struct aead_alg *alg = &t_alg->aead; 2924 2925 alg->base.cra_module = THIS_MODULE; 2926 alg->base.cra_priority = CAAM_CRA_PRIORITY; 2927 alg->base.cra_ctxsize = sizeof(struct caam_ctx); 2928 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 2929 2930 alg->init = caam_cra_init_aead; 2931 alg->exit = caam_cra_exit_aead; 2932 } 2933 2934 /* max hash key is max split key size */ 2935 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 2936 2937 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 2938 2939 /* caam context sizes for hashes: running digest + 8 */ 2940 #define HASH_MSG_LEN 8 2941 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 2942 2943 enum hash_optype { 2944 UPDATE = 0, 2945 UPDATE_FIRST, 2946 FINALIZE, 2947 DIGEST, 2948 HASH_NUM_OP 2949 }; 2950 2951 /** 2952 * caam_hash_ctx - ahash per-session context 2953 * @flc: Flow Contexts array 2954 * @key: authentication key 2955 * @flc_dma: I/O virtual addresses of the Flow Contexts 2956 * @dev: dpseci device 2957 * @ctx_len: size of Context Register 2958 * @adata: hashing algorithm details 2959 */ 2960 struct caam_hash_ctx { 2961 struct caam_flc flc[HASH_NUM_OP]; 2962 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2963 dma_addr_t flc_dma[HASH_NUM_OP]; 2964 struct device *dev; 2965 int ctx_len; 2966 struct alginfo adata; 2967 }; 2968 2969 /* ahash state */ 2970 struct caam_hash_state { 2971 struct caam_request caam_req; 2972 dma_addr_t buf_dma; 2973 dma_addr_t ctx_dma; 2974 int ctx_dma_len; 2975 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 2976 int buflen; 2977 int next_buflen; 2978 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 2979 int (*update)(struct ahash_request *req); 2980 int (*final)(struct ahash_request *req); 2981 int (*finup)(struct ahash_request *req); 2982 }; 2983 2984 struct caam_export_state { 2985 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 2986 u8 caam_ctx[MAX_CTX_LEN]; 2987 int buflen; 2988 int (*update)(struct ahash_request *req); 2989 int (*final)(struct ahash_request *req); 2990 int (*finup)(struct ahash_request *req); 2991 }; 2992 2993 /* Map current buffer in state (if length > 0) and put it in link table */ 2994 static inline int buf_map_to_qm_sg(struct device *dev, 2995 struct dpaa2_sg_entry *qm_sg, 2996 struct caam_hash_state *state) 2997 { 2998 int buflen = state->buflen; 2999 3000 if (!buflen) 3001 return 0; 3002 3003 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3004 DMA_TO_DEVICE); 3005 if (dma_mapping_error(dev, state->buf_dma)) { 3006 dev_err(dev, "unable to map buf\n"); 3007 state->buf_dma = 0; 3008 return -ENOMEM; 3009 } 3010 3011 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3012 3013 return 0; 3014 } 3015 3016 /* Map state->caam_ctx, and add it to link table */ 3017 static inline int ctx_map_to_qm_sg(struct device *dev, 3018 struct caam_hash_state *state, int ctx_len, 3019 struct dpaa2_sg_entry *qm_sg, u32 flag) 3020 { 3021 state->ctx_dma_len = ctx_len; 3022 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3023 if (dma_mapping_error(dev, state->ctx_dma)) { 3024 dev_err(dev, "unable to map ctx\n"); 3025 state->ctx_dma = 0; 3026 return -ENOMEM; 3027 } 3028 3029 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3030 3031 return 0; 3032 } 3033 3034 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3035 { 3036 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3037 int digestsize = crypto_ahash_digestsize(ahash); 3038 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3039 struct caam_flc *flc; 3040 u32 *desc; 3041 3042 /* ahash_update shared descriptor */ 3043 flc = &ctx->flc[UPDATE]; 3044 desc = flc->sh_desc; 3045 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3046 ctx->ctx_len, true, priv->sec_attr.era); 3047 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3048 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3049 desc_bytes(desc), DMA_BIDIRECTIONAL); 3050 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3051 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3052 1); 3053 3054 /* ahash_update_first shared descriptor */ 3055 flc = &ctx->flc[UPDATE_FIRST]; 3056 desc = flc->sh_desc; 3057 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3058 ctx->ctx_len, false, priv->sec_attr.era); 3059 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3060 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3061 desc_bytes(desc), DMA_BIDIRECTIONAL); 3062 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3063 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3064 1); 3065 3066 /* ahash_final shared descriptor */ 3067 flc = &ctx->flc[FINALIZE]; 3068 desc = flc->sh_desc; 3069 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3070 ctx->ctx_len, true, priv->sec_attr.era); 3071 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3072 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3073 desc_bytes(desc), DMA_BIDIRECTIONAL); 3074 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3075 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3076 1); 3077 3078 /* ahash_digest shared descriptor */ 3079 flc = &ctx->flc[DIGEST]; 3080 desc = flc->sh_desc; 3081 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3082 ctx->ctx_len, false, priv->sec_attr.era); 3083 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3084 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3085 desc_bytes(desc), DMA_BIDIRECTIONAL); 3086 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3087 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3088 1); 3089 3090 return 0; 3091 } 3092 3093 struct split_key_sh_result { 3094 struct completion completion; 3095 int err; 3096 struct device *dev; 3097 }; 3098 3099 static void split_key_sh_done(void *cbk_ctx, u32 err) 3100 { 3101 struct split_key_sh_result *res = cbk_ctx; 3102 3103 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3104 3105 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3106 complete(&res->completion); 3107 } 3108 3109 /* Digest hash size if it is too large */ 3110 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3111 u32 digestsize) 3112 { 3113 struct caam_request *req_ctx; 3114 u32 *desc; 3115 struct split_key_sh_result result; 3116 dma_addr_t key_dma; 3117 struct caam_flc *flc; 3118 dma_addr_t flc_dma; 3119 int ret = -ENOMEM; 3120 struct dpaa2_fl_entry *in_fle, *out_fle; 3121 3122 req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); 3123 if (!req_ctx) 3124 return -ENOMEM; 3125 3126 in_fle = &req_ctx->fd_flt[1]; 3127 out_fle = &req_ctx->fd_flt[0]; 3128 3129 flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); 3130 if (!flc) 3131 goto err_flc; 3132 3133 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3134 if (dma_mapping_error(ctx->dev, key_dma)) { 3135 dev_err(ctx->dev, "unable to map key memory\n"); 3136 goto err_key_dma; 3137 } 3138 3139 desc = flc->sh_desc; 3140 3141 init_sh_desc(desc, 0); 3142 3143 /* descriptor to perform unkeyed hash on key_in */ 3144 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3145 OP_ALG_AS_INITFINAL); 3146 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3147 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3148 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3149 LDST_SRCDST_BYTE_CONTEXT); 3150 3151 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3152 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3153 desc_bytes(desc), DMA_TO_DEVICE); 3154 if (dma_mapping_error(ctx->dev, flc_dma)) { 3155 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3156 goto err_flc_dma; 3157 } 3158 3159 dpaa2_fl_set_final(in_fle, true); 3160 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3161 dpaa2_fl_set_addr(in_fle, key_dma); 3162 dpaa2_fl_set_len(in_fle, *keylen); 3163 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3164 dpaa2_fl_set_addr(out_fle, key_dma); 3165 dpaa2_fl_set_len(out_fle, digestsize); 3166 3167 print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", 3168 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3169 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3170 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3171 1); 3172 3173 result.err = 0; 3174 init_completion(&result.completion); 3175 result.dev = ctx->dev; 3176 3177 req_ctx->flc = flc; 3178 req_ctx->flc_dma = flc_dma; 3179 req_ctx->cbk = split_key_sh_done; 3180 req_ctx->ctx = &result; 3181 3182 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3183 if (ret == -EINPROGRESS) { 3184 /* in progress */ 3185 wait_for_completion(&result.completion); 3186 ret = result.err; 3187 print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", 3188 DUMP_PREFIX_ADDRESS, 16, 4, key, 3189 digestsize, 1); 3190 } 3191 3192 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3193 DMA_TO_DEVICE); 3194 err_flc_dma: 3195 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3196 err_key_dma: 3197 kfree(flc); 3198 err_flc: 3199 kfree(req_ctx); 3200 3201 *keylen = digestsize; 3202 3203 return ret; 3204 } 3205 3206 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3207 unsigned int keylen) 3208 { 3209 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3210 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3211 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3212 int ret; 3213 u8 *hashed_key = NULL; 3214 3215 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3216 3217 if (keylen > blocksize) { 3218 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 3219 if (!hashed_key) 3220 return -ENOMEM; 3221 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3222 if (ret) 3223 goto bad_free_key; 3224 key = hashed_key; 3225 } 3226 3227 ctx->adata.keylen = keylen; 3228 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3229 OP_ALG_ALGSEL_MASK); 3230 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3231 goto bad_free_key; 3232 3233 ctx->adata.key_virt = key; 3234 ctx->adata.key_inline = true; 3235 3236 /* 3237 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3238 * in invalid opcodes (last bytes of user key) in the resulting 3239 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3240 * addresses are needed. 3241 */ 3242 if (keylen > ctx->adata.keylen_pad) { 3243 memcpy(ctx->key, key, keylen); 3244 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3245 ctx->adata.keylen_pad, 3246 DMA_TO_DEVICE); 3247 } 3248 3249 ret = ahash_set_sh_desc(ahash); 3250 kfree(hashed_key); 3251 return ret; 3252 bad_free_key: 3253 kfree(hashed_key); 3254 return -EINVAL; 3255 } 3256 3257 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3258 struct ahash_request *req) 3259 { 3260 struct caam_hash_state *state = ahash_request_ctx(req); 3261 3262 if (edesc->src_nents) 3263 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3264 3265 if (edesc->qm_sg_bytes) 3266 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3267 DMA_TO_DEVICE); 3268 3269 if (state->buf_dma) { 3270 dma_unmap_single(dev, state->buf_dma, state->buflen, 3271 DMA_TO_DEVICE); 3272 state->buf_dma = 0; 3273 } 3274 } 3275 3276 static inline void ahash_unmap_ctx(struct device *dev, 3277 struct ahash_edesc *edesc, 3278 struct ahash_request *req, u32 flag) 3279 { 3280 struct caam_hash_state *state = ahash_request_ctx(req); 3281 3282 if (state->ctx_dma) { 3283 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3284 state->ctx_dma = 0; 3285 } 3286 ahash_unmap(dev, edesc, req); 3287 } 3288 3289 static void ahash_done(void *cbk_ctx, u32 status) 3290 { 3291 struct crypto_async_request *areq = cbk_ctx; 3292 struct ahash_request *req = ahash_request_cast(areq); 3293 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3294 struct caam_hash_state *state = ahash_request_ctx(req); 3295 struct ahash_edesc *edesc = state->caam_req.edesc; 3296 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3297 int digestsize = crypto_ahash_digestsize(ahash); 3298 int ecode = 0; 3299 3300 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3301 3302 if (unlikely(status)) 3303 ecode = caam_qi2_strstatus(ctx->dev, status); 3304 3305 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3306 memcpy(req->result, state->caam_ctx, digestsize); 3307 qi_cache_free(edesc); 3308 3309 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3310 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3311 ctx->ctx_len, 1); 3312 3313 req->base.complete(&req->base, ecode); 3314 } 3315 3316 static void ahash_done_bi(void *cbk_ctx, u32 status) 3317 { 3318 struct crypto_async_request *areq = cbk_ctx; 3319 struct ahash_request *req = ahash_request_cast(areq); 3320 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3321 struct caam_hash_state *state = ahash_request_ctx(req); 3322 struct ahash_edesc *edesc = state->caam_req.edesc; 3323 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3324 int ecode = 0; 3325 3326 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3327 3328 if (unlikely(status)) 3329 ecode = caam_qi2_strstatus(ctx->dev, status); 3330 3331 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3332 qi_cache_free(edesc); 3333 3334 scatterwalk_map_and_copy(state->buf, req->src, 3335 req->nbytes - state->next_buflen, 3336 state->next_buflen, 0); 3337 state->buflen = state->next_buflen; 3338 3339 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3340 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3341 state->buflen, 1); 3342 3343 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3344 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3345 ctx->ctx_len, 1); 3346 if (req->result) 3347 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3348 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3349 crypto_ahash_digestsize(ahash), 1); 3350 3351 req->base.complete(&req->base, ecode); 3352 } 3353 3354 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3355 { 3356 struct crypto_async_request *areq = cbk_ctx; 3357 struct ahash_request *req = ahash_request_cast(areq); 3358 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3359 struct caam_hash_state *state = ahash_request_ctx(req); 3360 struct ahash_edesc *edesc = state->caam_req.edesc; 3361 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3362 int digestsize = crypto_ahash_digestsize(ahash); 3363 int ecode = 0; 3364 3365 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3366 3367 if (unlikely(status)) 3368 ecode = caam_qi2_strstatus(ctx->dev, status); 3369 3370 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3371 memcpy(req->result, state->caam_ctx, digestsize); 3372 qi_cache_free(edesc); 3373 3374 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3375 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3376 ctx->ctx_len, 1); 3377 3378 req->base.complete(&req->base, ecode); 3379 } 3380 3381 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3382 { 3383 struct crypto_async_request *areq = cbk_ctx; 3384 struct ahash_request *req = ahash_request_cast(areq); 3385 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3386 struct caam_hash_state *state = ahash_request_ctx(req); 3387 struct ahash_edesc *edesc = state->caam_req.edesc; 3388 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3389 int ecode = 0; 3390 3391 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3392 3393 if (unlikely(status)) 3394 ecode = caam_qi2_strstatus(ctx->dev, status); 3395 3396 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3397 qi_cache_free(edesc); 3398 3399 scatterwalk_map_and_copy(state->buf, req->src, 3400 req->nbytes - state->next_buflen, 3401 state->next_buflen, 0); 3402 state->buflen = state->next_buflen; 3403 3404 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3405 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3406 state->buflen, 1); 3407 3408 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3409 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3410 ctx->ctx_len, 1); 3411 if (req->result) 3412 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3413 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3414 crypto_ahash_digestsize(ahash), 1); 3415 3416 req->base.complete(&req->base, ecode); 3417 } 3418 3419 static int ahash_update_ctx(struct ahash_request *req) 3420 { 3421 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3422 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3423 struct caam_hash_state *state = ahash_request_ctx(req); 3424 struct caam_request *req_ctx = &state->caam_req; 3425 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3426 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3427 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3428 GFP_KERNEL : GFP_ATOMIC; 3429 u8 *buf = state->buf; 3430 int *buflen = &state->buflen; 3431 int *next_buflen = &state->next_buflen; 3432 int in_len = *buflen + req->nbytes, to_hash; 3433 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3434 struct ahash_edesc *edesc; 3435 int ret = 0; 3436 3437 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3438 to_hash = in_len - *next_buflen; 3439 3440 if (to_hash) { 3441 struct dpaa2_sg_entry *sg_table; 3442 int src_len = req->nbytes - *next_buflen; 3443 3444 src_nents = sg_nents_for_len(req->src, src_len); 3445 if (src_nents < 0) { 3446 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3447 return src_nents; 3448 } 3449 3450 if (src_nents) { 3451 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3452 DMA_TO_DEVICE); 3453 if (!mapped_nents) { 3454 dev_err(ctx->dev, "unable to DMA map source\n"); 3455 return -ENOMEM; 3456 } 3457 } else { 3458 mapped_nents = 0; 3459 } 3460 3461 /* allocate space for base edesc and link tables */ 3462 edesc = qi_cache_zalloc(GFP_DMA | flags); 3463 if (!edesc) { 3464 dma_unmap_sg(ctx->dev, req->src, src_nents, 3465 DMA_TO_DEVICE); 3466 return -ENOMEM; 3467 } 3468 3469 edesc->src_nents = src_nents; 3470 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3471 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3472 sizeof(*sg_table); 3473 sg_table = &edesc->sgt[0]; 3474 3475 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3476 DMA_BIDIRECTIONAL); 3477 if (ret) 3478 goto unmap_ctx; 3479 3480 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3481 if (ret) 3482 goto unmap_ctx; 3483 3484 if (mapped_nents) { 3485 sg_to_qm_sg_last(req->src, src_len, 3486 sg_table + qm_sg_src_index, 0); 3487 } else { 3488 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3489 true); 3490 } 3491 3492 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3493 qm_sg_bytes, DMA_TO_DEVICE); 3494 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3495 dev_err(ctx->dev, "unable to map S/G table\n"); 3496 ret = -ENOMEM; 3497 goto unmap_ctx; 3498 } 3499 edesc->qm_sg_bytes = qm_sg_bytes; 3500 3501 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3502 dpaa2_fl_set_final(in_fle, true); 3503 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3504 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3505 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3506 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3507 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3508 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3509 3510 req_ctx->flc = &ctx->flc[UPDATE]; 3511 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3512 req_ctx->cbk = ahash_done_bi; 3513 req_ctx->ctx = &req->base; 3514 req_ctx->edesc = edesc; 3515 3516 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3517 if (ret != -EINPROGRESS && 3518 !(ret == -EBUSY && 3519 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3520 goto unmap_ctx; 3521 } else if (*next_buflen) { 3522 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3523 req->nbytes, 0); 3524 *buflen = *next_buflen; 3525 3526 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3527 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3528 *buflen, 1); 3529 } 3530 3531 return ret; 3532 unmap_ctx: 3533 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3534 qi_cache_free(edesc); 3535 return ret; 3536 } 3537 3538 static int ahash_final_ctx(struct ahash_request *req) 3539 { 3540 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3541 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3542 struct caam_hash_state *state = ahash_request_ctx(req); 3543 struct caam_request *req_ctx = &state->caam_req; 3544 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3545 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3546 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3547 GFP_KERNEL : GFP_ATOMIC; 3548 int buflen = state->buflen; 3549 int qm_sg_bytes; 3550 int digestsize = crypto_ahash_digestsize(ahash); 3551 struct ahash_edesc *edesc; 3552 struct dpaa2_sg_entry *sg_table; 3553 int ret; 3554 3555 /* allocate space for base edesc and link tables */ 3556 edesc = qi_cache_zalloc(GFP_DMA | flags); 3557 if (!edesc) 3558 return -ENOMEM; 3559 3560 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3561 sg_table = &edesc->sgt[0]; 3562 3563 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3564 DMA_BIDIRECTIONAL); 3565 if (ret) 3566 goto unmap_ctx; 3567 3568 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3569 if (ret) 3570 goto unmap_ctx; 3571 3572 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3573 3574 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3575 DMA_TO_DEVICE); 3576 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3577 dev_err(ctx->dev, "unable to map S/G table\n"); 3578 ret = -ENOMEM; 3579 goto unmap_ctx; 3580 } 3581 edesc->qm_sg_bytes = qm_sg_bytes; 3582 3583 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3584 dpaa2_fl_set_final(in_fle, true); 3585 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3586 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3587 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3588 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3589 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3590 dpaa2_fl_set_len(out_fle, digestsize); 3591 3592 req_ctx->flc = &ctx->flc[FINALIZE]; 3593 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3594 req_ctx->cbk = ahash_done_ctx_src; 3595 req_ctx->ctx = &req->base; 3596 req_ctx->edesc = edesc; 3597 3598 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3599 if (ret == -EINPROGRESS || 3600 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3601 return ret; 3602 3603 unmap_ctx: 3604 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3605 qi_cache_free(edesc); 3606 return ret; 3607 } 3608 3609 static int ahash_finup_ctx(struct ahash_request *req) 3610 { 3611 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3612 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3613 struct caam_hash_state *state = ahash_request_ctx(req); 3614 struct caam_request *req_ctx = &state->caam_req; 3615 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3616 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3617 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3618 GFP_KERNEL : GFP_ATOMIC; 3619 int buflen = state->buflen; 3620 int qm_sg_bytes, qm_sg_src_index; 3621 int src_nents, mapped_nents; 3622 int digestsize = crypto_ahash_digestsize(ahash); 3623 struct ahash_edesc *edesc; 3624 struct dpaa2_sg_entry *sg_table; 3625 int ret; 3626 3627 src_nents = sg_nents_for_len(req->src, req->nbytes); 3628 if (src_nents < 0) { 3629 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3630 return src_nents; 3631 } 3632 3633 if (src_nents) { 3634 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3635 DMA_TO_DEVICE); 3636 if (!mapped_nents) { 3637 dev_err(ctx->dev, "unable to DMA map source\n"); 3638 return -ENOMEM; 3639 } 3640 } else { 3641 mapped_nents = 0; 3642 } 3643 3644 /* allocate space for base edesc and link tables */ 3645 edesc = qi_cache_zalloc(GFP_DMA | flags); 3646 if (!edesc) { 3647 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3648 return -ENOMEM; 3649 } 3650 3651 edesc->src_nents = src_nents; 3652 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3653 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3654 sizeof(*sg_table); 3655 sg_table = &edesc->sgt[0]; 3656 3657 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3658 DMA_BIDIRECTIONAL); 3659 if (ret) 3660 goto unmap_ctx; 3661 3662 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3663 if (ret) 3664 goto unmap_ctx; 3665 3666 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3667 3668 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3669 DMA_TO_DEVICE); 3670 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3671 dev_err(ctx->dev, "unable to map S/G table\n"); 3672 ret = -ENOMEM; 3673 goto unmap_ctx; 3674 } 3675 edesc->qm_sg_bytes = qm_sg_bytes; 3676 3677 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3678 dpaa2_fl_set_final(in_fle, true); 3679 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3680 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3681 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3682 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3683 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3684 dpaa2_fl_set_len(out_fle, digestsize); 3685 3686 req_ctx->flc = &ctx->flc[FINALIZE]; 3687 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3688 req_ctx->cbk = ahash_done_ctx_src; 3689 req_ctx->ctx = &req->base; 3690 req_ctx->edesc = edesc; 3691 3692 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3693 if (ret == -EINPROGRESS || 3694 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3695 return ret; 3696 3697 unmap_ctx: 3698 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3699 qi_cache_free(edesc); 3700 return ret; 3701 } 3702 3703 static int ahash_digest(struct ahash_request *req) 3704 { 3705 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3706 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3707 struct caam_hash_state *state = ahash_request_ctx(req); 3708 struct caam_request *req_ctx = &state->caam_req; 3709 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3710 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3711 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3712 GFP_KERNEL : GFP_ATOMIC; 3713 int digestsize = crypto_ahash_digestsize(ahash); 3714 int src_nents, mapped_nents; 3715 struct ahash_edesc *edesc; 3716 int ret = -ENOMEM; 3717 3718 state->buf_dma = 0; 3719 3720 src_nents = sg_nents_for_len(req->src, req->nbytes); 3721 if (src_nents < 0) { 3722 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3723 return src_nents; 3724 } 3725 3726 if (src_nents) { 3727 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3728 DMA_TO_DEVICE); 3729 if (!mapped_nents) { 3730 dev_err(ctx->dev, "unable to map source for DMA\n"); 3731 return ret; 3732 } 3733 } else { 3734 mapped_nents = 0; 3735 } 3736 3737 /* allocate space for base edesc and link tables */ 3738 edesc = qi_cache_zalloc(GFP_DMA | flags); 3739 if (!edesc) { 3740 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3741 return ret; 3742 } 3743 3744 edesc->src_nents = src_nents; 3745 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3746 3747 if (mapped_nents > 1) { 3748 int qm_sg_bytes; 3749 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3750 3751 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3752 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3753 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3754 qm_sg_bytes, DMA_TO_DEVICE); 3755 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3756 dev_err(ctx->dev, "unable to map S/G table\n"); 3757 goto unmap; 3758 } 3759 edesc->qm_sg_bytes = qm_sg_bytes; 3760 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3761 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3762 } else { 3763 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3764 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3765 } 3766 3767 state->ctx_dma_len = digestsize; 3768 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3769 DMA_FROM_DEVICE); 3770 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3771 dev_err(ctx->dev, "unable to map ctx\n"); 3772 state->ctx_dma = 0; 3773 goto unmap; 3774 } 3775 3776 dpaa2_fl_set_final(in_fle, true); 3777 dpaa2_fl_set_len(in_fle, req->nbytes); 3778 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3779 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3780 dpaa2_fl_set_len(out_fle, digestsize); 3781 3782 req_ctx->flc = &ctx->flc[DIGEST]; 3783 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3784 req_ctx->cbk = ahash_done; 3785 req_ctx->ctx = &req->base; 3786 req_ctx->edesc = edesc; 3787 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3788 if (ret == -EINPROGRESS || 3789 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3790 return ret; 3791 3792 unmap: 3793 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3794 qi_cache_free(edesc); 3795 return ret; 3796 } 3797 3798 static int ahash_final_no_ctx(struct ahash_request *req) 3799 { 3800 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3801 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3802 struct caam_hash_state *state = ahash_request_ctx(req); 3803 struct caam_request *req_ctx = &state->caam_req; 3804 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3805 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3806 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3807 GFP_KERNEL : GFP_ATOMIC; 3808 u8 *buf = state->buf; 3809 int buflen = state->buflen; 3810 int digestsize = crypto_ahash_digestsize(ahash); 3811 struct ahash_edesc *edesc; 3812 int ret = -ENOMEM; 3813 3814 /* allocate space for base edesc and link tables */ 3815 edesc = qi_cache_zalloc(GFP_DMA | flags); 3816 if (!edesc) 3817 return ret; 3818 3819 if (buflen) { 3820 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3821 DMA_TO_DEVICE); 3822 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3823 dev_err(ctx->dev, "unable to map src\n"); 3824 goto unmap; 3825 } 3826 } 3827 3828 state->ctx_dma_len = digestsize; 3829 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3830 DMA_FROM_DEVICE); 3831 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3832 dev_err(ctx->dev, "unable to map ctx\n"); 3833 state->ctx_dma = 0; 3834 goto unmap; 3835 } 3836 3837 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3838 dpaa2_fl_set_final(in_fle, true); 3839 /* 3840 * crypto engine requires the input entry to be present when 3841 * "frame list" FD is used. 3842 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3843 * in_fle zeroized (except for "Final" flag) is the best option. 3844 */ 3845 if (buflen) { 3846 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3847 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3848 dpaa2_fl_set_len(in_fle, buflen); 3849 } 3850 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3851 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3852 dpaa2_fl_set_len(out_fle, digestsize); 3853 3854 req_ctx->flc = &ctx->flc[DIGEST]; 3855 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3856 req_ctx->cbk = ahash_done; 3857 req_ctx->ctx = &req->base; 3858 req_ctx->edesc = edesc; 3859 3860 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3861 if (ret == -EINPROGRESS || 3862 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3863 return ret; 3864 3865 unmap: 3866 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3867 qi_cache_free(edesc); 3868 return ret; 3869 } 3870 3871 static int ahash_update_no_ctx(struct ahash_request *req) 3872 { 3873 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3874 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3875 struct caam_hash_state *state = ahash_request_ctx(req); 3876 struct caam_request *req_ctx = &state->caam_req; 3877 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3878 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3879 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3880 GFP_KERNEL : GFP_ATOMIC; 3881 u8 *buf = state->buf; 3882 int *buflen = &state->buflen; 3883 int *next_buflen = &state->next_buflen; 3884 int in_len = *buflen + req->nbytes, to_hash; 3885 int qm_sg_bytes, src_nents, mapped_nents; 3886 struct ahash_edesc *edesc; 3887 int ret = 0; 3888 3889 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3890 to_hash = in_len - *next_buflen; 3891 3892 if (to_hash) { 3893 struct dpaa2_sg_entry *sg_table; 3894 int src_len = req->nbytes - *next_buflen; 3895 3896 src_nents = sg_nents_for_len(req->src, src_len); 3897 if (src_nents < 0) { 3898 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3899 return src_nents; 3900 } 3901 3902 if (src_nents) { 3903 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3904 DMA_TO_DEVICE); 3905 if (!mapped_nents) { 3906 dev_err(ctx->dev, "unable to DMA map source\n"); 3907 return -ENOMEM; 3908 } 3909 } else { 3910 mapped_nents = 0; 3911 } 3912 3913 /* allocate space for base edesc and link tables */ 3914 edesc = qi_cache_zalloc(GFP_DMA | flags); 3915 if (!edesc) { 3916 dma_unmap_sg(ctx->dev, req->src, src_nents, 3917 DMA_TO_DEVICE); 3918 return -ENOMEM; 3919 } 3920 3921 edesc->src_nents = src_nents; 3922 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 3923 sizeof(*sg_table); 3924 sg_table = &edesc->sgt[0]; 3925 3926 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 3927 if (ret) 3928 goto unmap_ctx; 3929 3930 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 3931 3932 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3933 qm_sg_bytes, DMA_TO_DEVICE); 3934 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3935 dev_err(ctx->dev, "unable to map S/G table\n"); 3936 ret = -ENOMEM; 3937 goto unmap_ctx; 3938 } 3939 edesc->qm_sg_bytes = qm_sg_bytes; 3940 3941 state->ctx_dma_len = ctx->ctx_len; 3942 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 3943 ctx->ctx_len, DMA_FROM_DEVICE); 3944 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3945 dev_err(ctx->dev, "unable to map ctx\n"); 3946 state->ctx_dma = 0; 3947 ret = -ENOMEM; 3948 goto unmap_ctx; 3949 } 3950 3951 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3952 dpaa2_fl_set_final(in_fle, true); 3953 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3954 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3955 dpaa2_fl_set_len(in_fle, to_hash); 3956 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3957 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3958 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3959 3960 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 3961 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 3962 req_ctx->cbk = ahash_done_ctx_dst; 3963 req_ctx->ctx = &req->base; 3964 req_ctx->edesc = edesc; 3965 3966 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3967 if (ret != -EINPROGRESS && 3968 !(ret == -EBUSY && 3969 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3970 goto unmap_ctx; 3971 3972 state->update = ahash_update_ctx; 3973 state->finup = ahash_finup_ctx; 3974 state->final = ahash_final_ctx; 3975 } else if (*next_buflen) { 3976 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3977 req->nbytes, 0); 3978 *buflen = *next_buflen; 3979 3980 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3981 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3982 *buflen, 1); 3983 } 3984 3985 return ret; 3986 unmap_ctx: 3987 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 3988 qi_cache_free(edesc); 3989 return ret; 3990 } 3991 3992 static int ahash_finup_no_ctx(struct ahash_request *req) 3993 { 3994 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3995 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 3996 struct caam_hash_state *state = ahash_request_ctx(req); 3997 struct caam_request *req_ctx = &state->caam_req; 3998 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3999 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4000 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4001 GFP_KERNEL : GFP_ATOMIC; 4002 int buflen = state->buflen; 4003 int qm_sg_bytes, src_nents, mapped_nents; 4004 int digestsize = crypto_ahash_digestsize(ahash); 4005 struct ahash_edesc *edesc; 4006 struct dpaa2_sg_entry *sg_table; 4007 int ret; 4008 4009 src_nents = sg_nents_for_len(req->src, req->nbytes); 4010 if (src_nents < 0) { 4011 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4012 return src_nents; 4013 } 4014 4015 if (src_nents) { 4016 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4017 DMA_TO_DEVICE); 4018 if (!mapped_nents) { 4019 dev_err(ctx->dev, "unable to DMA map source\n"); 4020 return -ENOMEM; 4021 } 4022 } else { 4023 mapped_nents = 0; 4024 } 4025 4026 /* allocate space for base edesc and link tables */ 4027 edesc = qi_cache_zalloc(GFP_DMA | flags); 4028 if (!edesc) { 4029 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4030 return -ENOMEM; 4031 } 4032 4033 edesc->src_nents = src_nents; 4034 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4035 sg_table = &edesc->sgt[0]; 4036 4037 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4038 if (ret) 4039 goto unmap; 4040 4041 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4042 4043 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4044 DMA_TO_DEVICE); 4045 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4046 dev_err(ctx->dev, "unable to map S/G table\n"); 4047 ret = -ENOMEM; 4048 goto unmap; 4049 } 4050 edesc->qm_sg_bytes = qm_sg_bytes; 4051 4052 state->ctx_dma_len = digestsize; 4053 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4054 DMA_FROM_DEVICE); 4055 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4056 dev_err(ctx->dev, "unable to map ctx\n"); 4057 state->ctx_dma = 0; 4058 ret = -ENOMEM; 4059 goto unmap; 4060 } 4061 4062 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4063 dpaa2_fl_set_final(in_fle, true); 4064 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4065 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4066 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4067 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4068 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4069 dpaa2_fl_set_len(out_fle, digestsize); 4070 4071 req_ctx->flc = &ctx->flc[DIGEST]; 4072 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4073 req_ctx->cbk = ahash_done; 4074 req_ctx->ctx = &req->base; 4075 req_ctx->edesc = edesc; 4076 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4077 if (ret != -EINPROGRESS && 4078 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4079 goto unmap; 4080 4081 return ret; 4082 unmap: 4083 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4084 qi_cache_free(edesc); 4085 return -ENOMEM; 4086 } 4087 4088 static int ahash_update_first(struct ahash_request *req) 4089 { 4090 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4091 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); 4092 struct caam_hash_state *state = ahash_request_ctx(req); 4093 struct caam_request *req_ctx = &state->caam_req; 4094 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4095 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4096 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4097 GFP_KERNEL : GFP_ATOMIC; 4098 u8 *buf = state->buf; 4099 int *buflen = &state->buflen; 4100 int *next_buflen = &state->next_buflen; 4101 int to_hash; 4102 int src_nents, mapped_nents; 4103 struct ahash_edesc *edesc; 4104 int ret = 0; 4105 4106 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4107 1); 4108 to_hash = req->nbytes - *next_buflen; 4109 4110 if (to_hash) { 4111 struct dpaa2_sg_entry *sg_table; 4112 int src_len = req->nbytes - *next_buflen; 4113 4114 src_nents = sg_nents_for_len(req->src, src_len); 4115 if (src_nents < 0) { 4116 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4117 return src_nents; 4118 } 4119 4120 if (src_nents) { 4121 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4122 DMA_TO_DEVICE); 4123 if (!mapped_nents) { 4124 dev_err(ctx->dev, "unable to map source for DMA\n"); 4125 return -ENOMEM; 4126 } 4127 } else { 4128 mapped_nents = 0; 4129 } 4130 4131 /* allocate space for base edesc and link tables */ 4132 edesc = qi_cache_zalloc(GFP_DMA | flags); 4133 if (!edesc) { 4134 dma_unmap_sg(ctx->dev, req->src, src_nents, 4135 DMA_TO_DEVICE); 4136 return -ENOMEM; 4137 } 4138 4139 edesc->src_nents = src_nents; 4140 sg_table = &edesc->sgt[0]; 4141 4142 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4143 dpaa2_fl_set_final(in_fle, true); 4144 dpaa2_fl_set_len(in_fle, to_hash); 4145 4146 if (mapped_nents > 1) { 4147 int qm_sg_bytes; 4148 4149 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4150 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4151 sizeof(*sg_table); 4152 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4153 qm_sg_bytes, 4154 DMA_TO_DEVICE); 4155 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4156 dev_err(ctx->dev, "unable to map S/G table\n"); 4157 ret = -ENOMEM; 4158 goto unmap_ctx; 4159 } 4160 edesc->qm_sg_bytes = qm_sg_bytes; 4161 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4162 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4163 } else { 4164 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4165 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4166 } 4167 4168 state->ctx_dma_len = ctx->ctx_len; 4169 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4170 ctx->ctx_len, DMA_FROM_DEVICE); 4171 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4172 dev_err(ctx->dev, "unable to map ctx\n"); 4173 state->ctx_dma = 0; 4174 ret = -ENOMEM; 4175 goto unmap_ctx; 4176 } 4177 4178 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4179 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4180 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4181 4182 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4183 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4184 req_ctx->cbk = ahash_done_ctx_dst; 4185 req_ctx->ctx = &req->base; 4186 req_ctx->edesc = edesc; 4187 4188 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4189 if (ret != -EINPROGRESS && 4190 !(ret == -EBUSY && req->base.flags & 4191 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4192 goto unmap_ctx; 4193 4194 state->update = ahash_update_ctx; 4195 state->finup = ahash_finup_ctx; 4196 state->final = ahash_final_ctx; 4197 } else if (*next_buflen) { 4198 state->update = ahash_update_no_ctx; 4199 state->finup = ahash_finup_no_ctx; 4200 state->final = ahash_final_no_ctx; 4201 scatterwalk_map_and_copy(buf, req->src, 0, 4202 req->nbytes, 0); 4203 *buflen = *next_buflen; 4204 4205 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4206 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4207 *buflen, 1); 4208 } 4209 4210 return ret; 4211 unmap_ctx: 4212 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4213 qi_cache_free(edesc); 4214 return ret; 4215 } 4216 4217 static int ahash_finup_first(struct ahash_request *req) 4218 { 4219 return ahash_digest(req); 4220 } 4221 4222 static int ahash_init(struct ahash_request *req) 4223 { 4224 struct caam_hash_state *state = ahash_request_ctx(req); 4225 4226 state->update = ahash_update_first; 4227 state->finup = ahash_finup_first; 4228 state->final = ahash_final_no_ctx; 4229 4230 state->ctx_dma = 0; 4231 state->ctx_dma_len = 0; 4232 state->buf_dma = 0; 4233 state->buflen = 0; 4234 state->next_buflen = 0; 4235 4236 return 0; 4237 } 4238 4239 static int ahash_update(struct ahash_request *req) 4240 { 4241 struct caam_hash_state *state = ahash_request_ctx(req); 4242 4243 return state->update(req); 4244 } 4245 4246 static int ahash_finup(struct ahash_request *req) 4247 { 4248 struct caam_hash_state *state = ahash_request_ctx(req); 4249 4250 return state->finup(req); 4251 } 4252 4253 static int ahash_final(struct ahash_request *req) 4254 { 4255 struct caam_hash_state *state = ahash_request_ctx(req); 4256 4257 return state->final(req); 4258 } 4259 4260 static int ahash_export(struct ahash_request *req, void *out) 4261 { 4262 struct caam_hash_state *state = ahash_request_ctx(req); 4263 struct caam_export_state *export = out; 4264 u8 *buf = state->buf; 4265 int len = state->buflen; 4266 4267 memcpy(export->buf, buf, len); 4268 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4269 export->buflen = len; 4270 export->update = state->update; 4271 export->final = state->final; 4272 export->finup = state->finup; 4273 4274 return 0; 4275 } 4276 4277 static int ahash_import(struct ahash_request *req, const void *in) 4278 { 4279 struct caam_hash_state *state = ahash_request_ctx(req); 4280 const struct caam_export_state *export = in; 4281 4282 memset(state, 0, sizeof(*state)); 4283 memcpy(state->buf, export->buf, export->buflen); 4284 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4285 state->buflen = export->buflen; 4286 state->update = export->update; 4287 state->final = export->final; 4288 state->finup = export->finup; 4289 4290 return 0; 4291 } 4292 4293 struct caam_hash_template { 4294 char name[CRYPTO_MAX_ALG_NAME]; 4295 char driver_name[CRYPTO_MAX_ALG_NAME]; 4296 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4297 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4298 unsigned int blocksize; 4299 struct ahash_alg template_ahash; 4300 u32 alg_type; 4301 }; 4302 4303 /* ahash descriptors */ 4304 static struct caam_hash_template driver_hash[] = { 4305 { 4306 .name = "sha1", 4307 .driver_name = "sha1-caam-qi2", 4308 .hmac_name = "hmac(sha1)", 4309 .hmac_driver_name = "hmac-sha1-caam-qi2", 4310 .blocksize = SHA1_BLOCK_SIZE, 4311 .template_ahash = { 4312 .init = ahash_init, 4313 .update = ahash_update, 4314 .final = ahash_final, 4315 .finup = ahash_finup, 4316 .digest = ahash_digest, 4317 .export = ahash_export, 4318 .import = ahash_import, 4319 .setkey = ahash_setkey, 4320 .halg = { 4321 .digestsize = SHA1_DIGEST_SIZE, 4322 .statesize = sizeof(struct caam_export_state), 4323 }, 4324 }, 4325 .alg_type = OP_ALG_ALGSEL_SHA1, 4326 }, { 4327 .name = "sha224", 4328 .driver_name = "sha224-caam-qi2", 4329 .hmac_name = "hmac(sha224)", 4330 .hmac_driver_name = "hmac-sha224-caam-qi2", 4331 .blocksize = SHA224_BLOCK_SIZE, 4332 .template_ahash = { 4333 .init = ahash_init, 4334 .update = ahash_update, 4335 .final = ahash_final, 4336 .finup = ahash_finup, 4337 .digest = ahash_digest, 4338 .export = ahash_export, 4339 .import = ahash_import, 4340 .setkey = ahash_setkey, 4341 .halg = { 4342 .digestsize = SHA224_DIGEST_SIZE, 4343 .statesize = sizeof(struct caam_export_state), 4344 }, 4345 }, 4346 .alg_type = OP_ALG_ALGSEL_SHA224, 4347 }, { 4348 .name = "sha256", 4349 .driver_name = "sha256-caam-qi2", 4350 .hmac_name = "hmac(sha256)", 4351 .hmac_driver_name = "hmac-sha256-caam-qi2", 4352 .blocksize = SHA256_BLOCK_SIZE, 4353 .template_ahash = { 4354 .init = ahash_init, 4355 .update = ahash_update, 4356 .final = ahash_final, 4357 .finup = ahash_finup, 4358 .digest = ahash_digest, 4359 .export = ahash_export, 4360 .import = ahash_import, 4361 .setkey = ahash_setkey, 4362 .halg = { 4363 .digestsize = SHA256_DIGEST_SIZE, 4364 .statesize = sizeof(struct caam_export_state), 4365 }, 4366 }, 4367 .alg_type = OP_ALG_ALGSEL_SHA256, 4368 }, { 4369 .name = "sha384", 4370 .driver_name = "sha384-caam-qi2", 4371 .hmac_name = "hmac(sha384)", 4372 .hmac_driver_name = "hmac-sha384-caam-qi2", 4373 .blocksize = SHA384_BLOCK_SIZE, 4374 .template_ahash = { 4375 .init = ahash_init, 4376 .update = ahash_update, 4377 .final = ahash_final, 4378 .finup = ahash_finup, 4379 .digest = ahash_digest, 4380 .export = ahash_export, 4381 .import = ahash_import, 4382 .setkey = ahash_setkey, 4383 .halg = { 4384 .digestsize = SHA384_DIGEST_SIZE, 4385 .statesize = sizeof(struct caam_export_state), 4386 }, 4387 }, 4388 .alg_type = OP_ALG_ALGSEL_SHA384, 4389 }, { 4390 .name = "sha512", 4391 .driver_name = "sha512-caam-qi2", 4392 .hmac_name = "hmac(sha512)", 4393 .hmac_driver_name = "hmac-sha512-caam-qi2", 4394 .blocksize = SHA512_BLOCK_SIZE, 4395 .template_ahash = { 4396 .init = ahash_init, 4397 .update = ahash_update, 4398 .final = ahash_final, 4399 .finup = ahash_finup, 4400 .digest = ahash_digest, 4401 .export = ahash_export, 4402 .import = ahash_import, 4403 .setkey = ahash_setkey, 4404 .halg = { 4405 .digestsize = SHA512_DIGEST_SIZE, 4406 .statesize = sizeof(struct caam_export_state), 4407 }, 4408 }, 4409 .alg_type = OP_ALG_ALGSEL_SHA512, 4410 }, { 4411 .name = "md5", 4412 .driver_name = "md5-caam-qi2", 4413 .hmac_name = "hmac(md5)", 4414 .hmac_driver_name = "hmac-md5-caam-qi2", 4415 .blocksize = MD5_BLOCK_WORDS * 4, 4416 .template_ahash = { 4417 .init = ahash_init, 4418 .update = ahash_update, 4419 .final = ahash_final, 4420 .finup = ahash_finup, 4421 .digest = ahash_digest, 4422 .export = ahash_export, 4423 .import = ahash_import, 4424 .setkey = ahash_setkey, 4425 .halg = { 4426 .digestsize = MD5_DIGEST_SIZE, 4427 .statesize = sizeof(struct caam_export_state), 4428 }, 4429 }, 4430 .alg_type = OP_ALG_ALGSEL_MD5, 4431 } 4432 }; 4433 4434 struct caam_hash_alg { 4435 struct list_head entry; 4436 struct device *dev; 4437 int alg_type; 4438 struct ahash_alg ahash_alg; 4439 }; 4440 4441 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4442 { 4443 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4444 struct crypto_alg *base = tfm->__crt_alg; 4445 struct hash_alg_common *halg = 4446 container_of(base, struct hash_alg_common, base); 4447 struct ahash_alg *alg = 4448 container_of(halg, struct ahash_alg, halg); 4449 struct caam_hash_alg *caam_hash = 4450 container_of(alg, struct caam_hash_alg, ahash_alg); 4451 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4452 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4453 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4454 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4455 HASH_MSG_LEN + 32, 4456 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4457 HASH_MSG_LEN + 64, 4458 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4459 dma_addr_t dma_addr; 4460 int i; 4461 4462 ctx->dev = caam_hash->dev; 4463 4464 if (alg->setkey) { 4465 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4466 ARRAY_SIZE(ctx->key), 4467 DMA_TO_DEVICE, 4468 DMA_ATTR_SKIP_CPU_SYNC); 4469 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4470 dev_err(ctx->dev, "unable to map key\n"); 4471 return -ENOMEM; 4472 } 4473 } 4474 4475 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4476 DMA_BIDIRECTIONAL, 4477 DMA_ATTR_SKIP_CPU_SYNC); 4478 if (dma_mapping_error(ctx->dev, dma_addr)) { 4479 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4480 if (ctx->adata.key_dma) 4481 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4482 ARRAY_SIZE(ctx->key), 4483 DMA_TO_DEVICE, 4484 DMA_ATTR_SKIP_CPU_SYNC); 4485 return -ENOMEM; 4486 } 4487 4488 for (i = 0; i < HASH_NUM_OP; i++) 4489 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4490 4491 /* copy descriptor header template value */ 4492 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4493 4494 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4495 OP_ALG_ALGSEL_SUBMASK) >> 4496 OP_ALG_ALGSEL_SHIFT]; 4497 4498 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 4499 sizeof(struct caam_hash_state)); 4500 4501 return ahash_set_sh_desc(ahash); 4502 } 4503 4504 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4505 { 4506 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); 4507 4508 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4509 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4510 if (ctx->adata.key_dma) 4511 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4512 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4513 DMA_ATTR_SKIP_CPU_SYNC); 4514 } 4515 4516 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4517 struct caam_hash_template *template, bool keyed) 4518 { 4519 struct caam_hash_alg *t_alg; 4520 struct ahash_alg *halg; 4521 struct crypto_alg *alg; 4522 4523 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); 4524 if (!t_alg) 4525 return ERR_PTR(-ENOMEM); 4526 4527 t_alg->ahash_alg = template->template_ahash; 4528 halg = &t_alg->ahash_alg; 4529 alg = &halg->halg.base; 4530 4531 if (keyed) { 4532 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4533 template->hmac_name); 4534 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4535 template->hmac_driver_name); 4536 } else { 4537 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", 4538 template->name); 4539 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4540 template->driver_name); 4541 t_alg->ahash_alg.setkey = NULL; 4542 } 4543 alg->cra_module = THIS_MODULE; 4544 alg->cra_init = caam_hash_cra_init; 4545 alg->cra_exit = caam_hash_cra_exit; 4546 alg->cra_ctxsize = sizeof(struct caam_hash_ctx); 4547 alg->cra_priority = CAAM_CRA_PRIORITY; 4548 alg->cra_blocksize = template->blocksize; 4549 alg->cra_alignmask = 0; 4550 alg->cra_flags = CRYPTO_ALG_ASYNC; 4551 4552 t_alg->alg_type = template->alg_type; 4553 t_alg->dev = dev; 4554 4555 return t_alg; 4556 } 4557 4558 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4559 { 4560 struct dpaa2_caam_priv_per_cpu *ppriv; 4561 4562 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4563 napi_schedule_irqoff(&ppriv->napi); 4564 } 4565 4566 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4567 { 4568 struct device *dev = priv->dev; 4569 struct dpaa2_io_notification_ctx *nctx; 4570 struct dpaa2_caam_priv_per_cpu *ppriv; 4571 int err, i = 0, cpu; 4572 4573 for_each_online_cpu(cpu) { 4574 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4575 ppriv->priv = priv; 4576 nctx = &ppriv->nctx; 4577 nctx->is_cdan = 0; 4578 nctx->id = ppriv->rsp_fqid; 4579 nctx->desired_cpu = cpu; 4580 nctx->cb = dpaa2_caam_fqdan_cb; 4581 4582 /* Register notification callbacks */ 4583 ppriv->dpio = dpaa2_io_service_select(cpu); 4584 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4585 if (unlikely(err)) { 4586 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4587 nctx->cb = NULL; 4588 /* 4589 * If no affine DPIO for this core, there's probably 4590 * none available for next cores either. Signal we want 4591 * to retry later, in case the DPIO devices weren't 4592 * probed yet. 4593 */ 4594 err = -EPROBE_DEFER; 4595 goto err; 4596 } 4597 4598 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4599 dev); 4600 if (unlikely(!ppriv->store)) { 4601 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4602 err = -ENOMEM; 4603 goto err; 4604 } 4605 4606 if (++i == priv->num_pairs) 4607 break; 4608 } 4609 4610 return 0; 4611 4612 err: 4613 for_each_online_cpu(cpu) { 4614 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4615 if (!ppriv->nctx.cb) 4616 break; 4617 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4618 } 4619 4620 for_each_online_cpu(cpu) { 4621 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4622 if (!ppriv->store) 4623 break; 4624 dpaa2_io_store_destroy(ppriv->store); 4625 } 4626 4627 return err; 4628 } 4629 4630 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4631 { 4632 struct dpaa2_caam_priv_per_cpu *ppriv; 4633 int i = 0, cpu; 4634 4635 for_each_online_cpu(cpu) { 4636 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4637 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4638 priv->dev); 4639 dpaa2_io_store_destroy(ppriv->store); 4640 4641 if (++i == priv->num_pairs) 4642 return; 4643 } 4644 } 4645 4646 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4647 { 4648 struct dpseci_rx_queue_cfg rx_queue_cfg; 4649 struct device *dev = priv->dev; 4650 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4651 struct dpaa2_caam_priv_per_cpu *ppriv; 4652 int err = 0, i = 0, cpu; 4653 4654 /* Configure Rx queues */ 4655 for_each_online_cpu(cpu) { 4656 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4657 4658 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4659 DPSECI_QUEUE_OPT_USER_CTX; 4660 rx_queue_cfg.order_preservation_en = 0; 4661 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4662 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4663 /* 4664 * Rx priority (WQ) doesn't really matter, since we use 4665 * pull mode, i.e. volatile dequeues from specific FQs 4666 */ 4667 rx_queue_cfg.dest_cfg.priority = 0; 4668 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4669 4670 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4671 &rx_queue_cfg); 4672 if (err) { 4673 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4674 err); 4675 return err; 4676 } 4677 4678 if (++i == priv->num_pairs) 4679 break; 4680 } 4681 4682 return err; 4683 } 4684 4685 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4686 { 4687 struct device *dev = priv->dev; 4688 4689 if (!priv->cscn_mem) 4690 return; 4691 4692 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4693 kfree(priv->cscn_mem); 4694 } 4695 4696 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4697 { 4698 struct device *dev = priv->dev; 4699 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4700 4701 dpaa2_dpseci_congestion_free(priv); 4702 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4703 } 4704 4705 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4706 const struct dpaa2_fd *fd) 4707 { 4708 struct caam_request *req; 4709 u32 fd_err; 4710 4711 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4712 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4713 return; 4714 } 4715 4716 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4717 if (unlikely(fd_err)) 4718 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4719 4720 /* 4721 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4722 * in FD[ERR] or FD[FRC]. 4723 */ 4724 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4725 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4726 DMA_BIDIRECTIONAL); 4727 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4728 } 4729 4730 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4731 { 4732 int err; 4733 4734 /* Retry while portal is busy */ 4735 do { 4736 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4737 ppriv->store); 4738 } while (err == -EBUSY); 4739 4740 if (unlikely(err)) 4741 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4742 4743 return err; 4744 } 4745 4746 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4747 { 4748 struct dpaa2_dq *dq; 4749 int cleaned = 0, is_last; 4750 4751 do { 4752 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4753 if (unlikely(!dq)) { 4754 if (unlikely(!is_last)) { 4755 dev_dbg(ppriv->priv->dev, 4756 "FQ %d returned no valid frames\n", 4757 ppriv->rsp_fqid); 4758 /* 4759 * MUST retry until we get some sort of 4760 * valid response token (be it "empty dequeue" 4761 * or a valid frame). 4762 */ 4763 continue; 4764 } 4765 break; 4766 } 4767 4768 /* Process FD */ 4769 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4770 cleaned++; 4771 } while (!is_last); 4772 4773 return cleaned; 4774 } 4775 4776 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4777 { 4778 struct dpaa2_caam_priv_per_cpu *ppriv; 4779 struct dpaa2_caam_priv *priv; 4780 int err, cleaned = 0, store_cleaned; 4781 4782 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4783 priv = ppriv->priv; 4784 4785 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4786 return 0; 4787 4788 do { 4789 store_cleaned = dpaa2_caam_store_consume(ppriv); 4790 cleaned += store_cleaned; 4791 4792 if (store_cleaned == 0 || 4793 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4794 break; 4795 4796 /* Try to dequeue some more */ 4797 err = dpaa2_caam_pull_fq(ppriv); 4798 if (unlikely(err)) 4799 break; 4800 } while (1); 4801 4802 if (cleaned < budget) { 4803 napi_complete_done(napi, cleaned); 4804 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4805 if (unlikely(err)) 4806 dev_err(priv->dev, "Notification rearm failed: %d\n", 4807 err); 4808 } 4809 4810 return cleaned; 4811 } 4812 4813 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4814 u16 token) 4815 { 4816 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4817 struct device *dev = priv->dev; 4818 int err; 4819 4820 /* 4821 * Congestion group feature supported starting with DPSECI API v5.1 4822 * and only when object has been created with this capability. 4823 */ 4824 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4825 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4826 return 0; 4827 4828 priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, 4829 GFP_KERNEL | GFP_DMA); 4830 if (!priv->cscn_mem) 4831 return -ENOMEM; 4832 4833 priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); 4834 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, 4835 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4836 if (dma_mapping_error(dev, priv->cscn_dma)) { 4837 dev_err(dev, "Error mapping CSCN memory area\n"); 4838 err = -ENOMEM; 4839 goto err_dma_map; 4840 } 4841 4842 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4843 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4844 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4845 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4846 cong_notif_cfg.message_iova = priv->cscn_dma; 4847 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4848 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4849 DPSECI_CGN_MODE_COHERENT_WRITE; 4850 4851 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4852 &cong_notif_cfg); 4853 if (err) { 4854 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4855 goto err_set_cong; 4856 } 4857 4858 return 0; 4859 4860 err_set_cong: 4861 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4862 err_dma_map: 4863 kfree(priv->cscn_mem); 4864 4865 return err; 4866 } 4867 4868 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 4869 { 4870 struct device *dev = &ls_dev->dev; 4871 struct dpaa2_caam_priv *priv; 4872 struct dpaa2_caam_priv_per_cpu *ppriv; 4873 int err, cpu; 4874 u8 i; 4875 4876 priv = dev_get_drvdata(dev); 4877 4878 priv->dev = dev; 4879 priv->dpsec_id = ls_dev->obj_desc.id; 4880 4881 /* Get a handle for the DPSECI this interface is associate with */ 4882 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 4883 if (err) { 4884 dev_err(dev, "dpseci_open() failed: %d\n", err); 4885 goto err_open; 4886 } 4887 4888 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 4889 &priv->minor_ver); 4890 if (err) { 4891 dev_err(dev, "dpseci_get_api_version() failed\n"); 4892 goto err_get_vers; 4893 } 4894 4895 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 4896 4897 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 4898 &priv->dpseci_attr); 4899 if (err) { 4900 dev_err(dev, "dpseci_get_attributes() failed\n"); 4901 goto err_get_vers; 4902 } 4903 4904 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 4905 &priv->sec_attr); 4906 if (err) { 4907 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 4908 goto err_get_vers; 4909 } 4910 4911 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 4912 if (err) { 4913 dev_err(dev, "setup_congestion() failed\n"); 4914 goto err_get_vers; 4915 } 4916 4917 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 4918 priv->dpseci_attr.num_tx_queues); 4919 if (priv->num_pairs > num_online_cpus()) { 4920 dev_warn(dev, "%d queues won't be used\n", 4921 priv->num_pairs - num_online_cpus()); 4922 priv->num_pairs = num_online_cpus(); 4923 } 4924 4925 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 4926 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4927 &priv->rx_queue_attr[i]); 4928 if (err) { 4929 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 4930 goto err_get_rx_queue; 4931 } 4932 } 4933 4934 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 4935 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4936 &priv->tx_queue_attr[i]); 4937 if (err) { 4938 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 4939 goto err_get_rx_queue; 4940 } 4941 } 4942 4943 i = 0; 4944 for_each_online_cpu(cpu) { 4945 u8 j; 4946 4947 j = i % priv->num_pairs; 4948 4949 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4950 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 4951 4952 /* 4953 * Allow all cores to enqueue, while only some of them 4954 * will take part in dequeuing. 4955 */ 4956 if (++i > priv->num_pairs) 4957 continue; 4958 4959 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 4960 ppriv->prio = j; 4961 4962 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 4963 priv->rx_queue_attr[j].fqid, 4964 priv->tx_queue_attr[j].fqid); 4965 4966 ppriv->net_dev.dev = *dev; 4967 INIT_LIST_HEAD(&ppriv->net_dev.napi_list); 4968 netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, 4969 DPAA2_CAAM_NAPI_WEIGHT); 4970 } 4971 4972 return 0; 4973 4974 err_get_rx_queue: 4975 dpaa2_dpseci_congestion_free(priv); 4976 err_get_vers: 4977 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4978 err_open: 4979 return err; 4980 } 4981 4982 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 4983 { 4984 struct device *dev = priv->dev; 4985 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4986 struct dpaa2_caam_priv_per_cpu *ppriv; 4987 int i; 4988 4989 for (i = 0; i < priv->num_pairs; i++) { 4990 ppriv = per_cpu_ptr(priv->ppriv, i); 4991 napi_enable(&ppriv->napi); 4992 } 4993 4994 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 4995 } 4996 4997 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 4998 { 4999 struct device *dev = priv->dev; 5000 struct dpaa2_caam_priv_per_cpu *ppriv; 5001 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5002 int i, err = 0, enabled; 5003 5004 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5005 if (err) { 5006 dev_err(dev, "dpseci_disable() failed\n"); 5007 return err; 5008 } 5009 5010 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5011 if (err) { 5012 dev_err(dev, "dpseci_is_enabled() failed\n"); 5013 return err; 5014 } 5015 5016 dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); 5017 5018 for (i = 0; i < priv->num_pairs; i++) { 5019 ppriv = per_cpu_ptr(priv->ppriv, i); 5020 napi_disable(&ppriv->napi); 5021 netif_napi_del(&ppriv->napi); 5022 } 5023 5024 return 0; 5025 } 5026 5027 static struct list_head hash_list; 5028 5029 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5030 { 5031 struct device *dev; 5032 struct dpaa2_caam_priv *priv; 5033 int i, err = 0; 5034 bool registered = false; 5035 5036 /* 5037 * There is no way to get CAAM endianness - there is no direct register 5038 * space access and MC f/w does not provide this attribute. 5039 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5040 * property. 5041 */ 5042 caam_little_end = true; 5043 5044 caam_imx = false; 5045 5046 dev = &dpseci_dev->dev; 5047 5048 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5049 if (!priv) 5050 return -ENOMEM; 5051 5052 dev_set_drvdata(dev, priv); 5053 5054 priv->domain = iommu_get_domain_for_dev(dev); 5055 5056 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5057 0, SLAB_CACHE_DMA, NULL); 5058 if (!qi_cache) { 5059 dev_err(dev, "Can't allocate SEC cache\n"); 5060 return -ENOMEM; 5061 } 5062 5063 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5064 if (err) { 5065 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5066 goto err_dma_mask; 5067 } 5068 5069 /* Obtain a MC portal */ 5070 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5071 if (err) { 5072 if (err == -ENXIO) 5073 err = -EPROBE_DEFER; 5074 else 5075 dev_err(dev, "MC portal allocation failed\n"); 5076 5077 goto err_dma_mask; 5078 } 5079 5080 priv->ppriv = alloc_percpu(*priv->ppriv); 5081 if (!priv->ppriv) { 5082 dev_err(dev, "alloc_percpu() failed\n"); 5083 err = -ENOMEM; 5084 goto err_alloc_ppriv; 5085 } 5086 5087 /* DPSECI initialization */ 5088 err = dpaa2_dpseci_setup(dpseci_dev); 5089 if (err) { 5090 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5091 goto err_dpseci_setup; 5092 } 5093 5094 /* DPIO */ 5095 err = dpaa2_dpseci_dpio_setup(priv); 5096 if (err) { 5097 if (err != -EPROBE_DEFER) 5098 dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); 5099 goto err_dpio_setup; 5100 } 5101 5102 /* DPSECI binding to DPIO */ 5103 err = dpaa2_dpseci_bind(priv); 5104 if (err) { 5105 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5106 goto err_bind; 5107 } 5108 5109 /* DPSECI enable */ 5110 err = dpaa2_dpseci_enable(priv); 5111 if (err) { 5112 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5113 goto err_bind; 5114 } 5115 5116 dpaa2_dpseci_debugfs_init(priv); 5117 5118 /* register crypto algorithms the device supports */ 5119 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5120 struct caam_skcipher_alg *t_alg = driver_algs + i; 5121 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5122 5123 /* Skip DES algorithms if not supported by device */ 5124 if (!priv->sec_attr.des_acc_num && 5125 (alg_sel == OP_ALG_ALGSEL_3DES || 5126 alg_sel == OP_ALG_ALGSEL_DES)) 5127 continue; 5128 5129 /* Skip AES algorithms if not supported by device */ 5130 if (!priv->sec_attr.aes_acc_num && 5131 alg_sel == OP_ALG_ALGSEL_AES) 5132 continue; 5133 5134 /* Skip CHACHA20 algorithms if not supported by device */ 5135 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5136 !priv->sec_attr.ccha_acc_num) 5137 continue; 5138 5139 t_alg->caam.dev = dev; 5140 caam_skcipher_alg_init(t_alg); 5141 5142 err = crypto_register_skcipher(&t_alg->skcipher); 5143 if (err) { 5144 dev_warn(dev, "%s alg registration failed: %d\n", 5145 t_alg->skcipher.base.cra_driver_name, err); 5146 continue; 5147 } 5148 5149 t_alg->registered = true; 5150 registered = true; 5151 } 5152 5153 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5154 struct caam_aead_alg *t_alg = driver_aeads + i; 5155 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5156 OP_ALG_ALGSEL_MASK; 5157 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5158 OP_ALG_ALGSEL_MASK; 5159 5160 /* Skip DES algorithms if not supported by device */ 5161 if (!priv->sec_attr.des_acc_num && 5162 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5163 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5164 continue; 5165 5166 /* Skip AES algorithms if not supported by device */ 5167 if (!priv->sec_attr.aes_acc_num && 5168 c1_alg_sel == OP_ALG_ALGSEL_AES) 5169 continue; 5170 5171 /* Skip CHACHA20 algorithms if not supported by device */ 5172 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5173 !priv->sec_attr.ccha_acc_num) 5174 continue; 5175 5176 /* Skip POLY1305 algorithms if not supported by device */ 5177 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5178 !priv->sec_attr.ptha_acc_num) 5179 continue; 5180 5181 /* 5182 * Skip algorithms requiring message digests 5183 * if MD not supported by device. 5184 */ 5185 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5186 !priv->sec_attr.md_acc_num) 5187 continue; 5188 5189 t_alg->caam.dev = dev; 5190 caam_aead_alg_init(t_alg); 5191 5192 err = crypto_register_aead(&t_alg->aead); 5193 if (err) { 5194 dev_warn(dev, "%s alg registration failed: %d\n", 5195 t_alg->aead.base.cra_driver_name, err); 5196 continue; 5197 } 5198 5199 t_alg->registered = true; 5200 registered = true; 5201 } 5202 if (registered) 5203 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5204 5205 /* register hash algorithms the device supports */ 5206 INIT_LIST_HEAD(&hash_list); 5207 5208 /* 5209 * Skip registration of any hashing algorithms if MD block 5210 * is not present. 5211 */ 5212 if (!priv->sec_attr.md_acc_num) 5213 return 0; 5214 5215 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5216 struct caam_hash_alg *t_alg; 5217 struct caam_hash_template *alg = driver_hash + i; 5218 5219 /* register hmac version */ 5220 t_alg = caam_hash_alloc(dev, alg, true); 5221 if (IS_ERR(t_alg)) { 5222 err = PTR_ERR(t_alg); 5223 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5224 alg->driver_name, err); 5225 continue; 5226 } 5227 5228 err = crypto_register_ahash(&t_alg->ahash_alg); 5229 if (err) { 5230 dev_warn(dev, "%s alg registration failed: %d\n", 5231 t_alg->ahash_alg.halg.base.cra_driver_name, 5232 err); 5233 kfree(t_alg); 5234 } else { 5235 list_add_tail(&t_alg->entry, &hash_list); 5236 } 5237 5238 /* register unkeyed version */ 5239 t_alg = caam_hash_alloc(dev, alg, false); 5240 if (IS_ERR(t_alg)) { 5241 err = PTR_ERR(t_alg); 5242 dev_warn(dev, "%s alg allocation failed: %d\n", 5243 alg->driver_name, err); 5244 continue; 5245 } 5246 5247 err = crypto_register_ahash(&t_alg->ahash_alg); 5248 if (err) { 5249 dev_warn(dev, "%s alg registration failed: %d\n", 5250 t_alg->ahash_alg.halg.base.cra_driver_name, 5251 err); 5252 kfree(t_alg); 5253 } else { 5254 list_add_tail(&t_alg->entry, &hash_list); 5255 } 5256 } 5257 if (!list_empty(&hash_list)) 5258 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5259 5260 return err; 5261 5262 err_bind: 5263 dpaa2_dpseci_dpio_free(priv); 5264 err_dpio_setup: 5265 dpaa2_dpseci_free(priv); 5266 err_dpseci_setup: 5267 free_percpu(priv->ppriv); 5268 err_alloc_ppriv: 5269 fsl_mc_portal_free(priv->mc_io); 5270 err_dma_mask: 5271 kmem_cache_destroy(qi_cache); 5272 5273 return err; 5274 } 5275 5276 static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5277 { 5278 struct device *dev; 5279 struct dpaa2_caam_priv *priv; 5280 int i; 5281 5282 dev = &ls_dev->dev; 5283 priv = dev_get_drvdata(dev); 5284 5285 dpaa2_dpseci_debugfs_exit(priv); 5286 5287 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5288 struct caam_aead_alg *t_alg = driver_aeads + i; 5289 5290 if (t_alg->registered) 5291 crypto_unregister_aead(&t_alg->aead); 5292 } 5293 5294 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5295 struct caam_skcipher_alg *t_alg = driver_algs + i; 5296 5297 if (t_alg->registered) 5298 crypto_unregister_skcipher(&t_alg->skcipher); 5299 } 5300 5301 if (hash_list.next) { 5302 struct caam_hash_alg *t_hash_alg, *p; 5303 5304 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5305 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5306 list_del(&t_hash_alg->entry); 5307 kfree(t_hash_alg); 5308 } 5309 } 5310 5311 dpaa2_dpseci_disable(priv); 5312 dpaa2_dpseci_dpio_free(priv); 5313 dpaa2_dpseci_free(priv); 5314 free_percpu(priv->ppriv); 5315 fsl_mc_portal_free(priv->mc_io); 5316 kmem_cache_destroy(qi_cache); 5317 5318 return 0; 5319 } 5320 5321 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5322 { 5323 struct dpaa2_fd fd; 5324 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5325 struct dpaa2_caam_priv_per_cpu *ppriv; 5326 int err = 0, i; 5327 5328 if (IS_ERR(req)) 5329 return PTR_ERR(req); 5330 5331 if (priv->cscn_mem) { 5332 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5333 DPAA2_CSCN_SIZE, 5334 DMA_FROM_DEVICE); 5335 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { 5336 dev_dbg_ratelimited(dev, "Dropping request\n"); 5337 return -EBUSY; 5338 } 5339 } 5340 5341 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5342 5343 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5344 DMA_BIDIRECTIONAL); 5345 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5346 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5347 goto err_out; 5348 } 5349 5350 memset(&fd, 0, sizeof(fd)); 5351 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5352 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5353 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5354 dpaa2_fd_set_flc(&fd, req->flc_dma); 5355 5356 ppriv = this_cpu_ptr(priv->ppriv); 5357 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5358 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5359 &fd); 5360 if (err != -EBUSY) 5361 break; 5362 5363 cpu_relax(); 5364 } 5365 5366 if (unlikely(err)) { 5367 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5368 goto err_out; 5369 } 5370 5371 return -EINPROGRESS; 5372 5373 err_out: 5374 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5375 DMA_BIDIRECTIONAL); 5376 return -EIO; 5377 } 5378 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5379 5380 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5381 { 5382 .vendor = FSL_MC_VENDOR_FREESCALE, 5383 .obj_type = "dpseci", 5384 }, 5385 { .vendor = 0x0 } 5386 }; 5387 5388 static struct fsl_mc_driver dpaa2_caam_driver = { 5389 .driver = { 5390 .name = KBUILD_MODNAME, 5391 .owner = THIS_MODULE, 5392 }, 5393 .probe = dpaa2_caam_probe, 5394 .remove = dpaa2_caam_remove, 5395 .match_id_table = dpaa2_caam_match_id_table 5396 }; 5397 5398 MODULE_LICENSE("Dual BSD/GPL"); 5399 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5400 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5401 5402 module_fsl_mc_driver(dpaa2_caam_driver); 5403