1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) STMicroelectronics SA 2017 4 * Author: Fabien Dessenne <fabien.dessenne@st.com> 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/delay.h> 9 #include <linux/interrupt.h> 10 #include <linux/iopoll.h> 11 #include <linux/module.h> 12 #include <linux/of_device.h> 13 #include <linux/platform_device.h> 14 #include <linux/pm_runtime.h> 15 #include <linux/reset.h> 16 17 #include <crypto/aes.h> 18 #include <crypto/des.h> 19 #include <crypto/engine.h> 20 #include <crypto/scatterwalk.h> 21 #include <crypto/internal/aead.h> 22 23 #define DRIVER_NAME "stm32-cryp" 24 25 /* Bit [0] encrypt / decrypt */ 26 #define FLG_ENCRYPT BIT(0) 27 /* Bit [8..1] algo & operation mode */ 28 #define FLG_AES BIT(1) 29 #define FLG_DES BIT(2) 30 #define FLG_TDES BIT(3) 31 #define FLG_ECB BIT(4) 32 #define FLG_CBC BIT(5) 33 #define FLG_CTR BIT(6) 34 #define FLG_GCM BIT(7) 35 #define FLG_CCM BIT(8) 36 /* Mode mask = bits [15..0] */ 37 #define FLG_MODE_MASK GENMASK(15, 0) 38 /* Bit [31..16] status */ 39 #define FLG_CCM_PADDED_WA BIT(16) 40 41 /* Registers */ 42 #define CRYP_CR 0x00000000 43 #define CRYP_SR 0x00000004 44 #define CRYP_DIN 0x00000008 45 #define CRYP_DOUT 0x0000000C 46 #define CRYP_DMACR 0x00000010 47 #define CRYP_IMSCR 0x00000014 48 #define CRYP_RISR 0x00000018 49 #define CRYP_MISR 0x0000001C 50 #define CRYP_K0LR 0x00000020 51 #define CRYP_K0RR 0x00000024 52 #define CRYP_K1LR 0x00000028 53 #define CRYP_K1RR 0x0000002C 54 #define CRYP_K2LR 0x00000030 55 #define CRYP_K2RR 0x00000034 56 #define CRYP_K3LR 0x00000038 57 #define CRYP_K3RR 0x0000003C 58 #define CRYP_IV0LR 0x00000040 59 #define CRYP_IV0RR 0x00000044 60 #define CRYP_IV1LR 0x00000048 61 #define CRYP_IV1RR 0x0000004C 62 #define CRYP_CSGCMCCM0R 0x00000050 63 #define CRYP_CSGCM0R 0x00000070 64 65 /* Registers values */ 66 #define CR_DEC_NOT_ENC 0x00000004 67 #define CR_TDES_ECB 0x00000000 68 #define CR_TDES_CBC 0x00000008 69 #define CR_DES_ECB 0x00000010 70 #define CR_DES_CBC 0x00000018 71 #define CR_AES_ECB 0x00000020 72 #define CR_AES_CBC 0x00000028 73 #define CR_AES_CTR 0x00000030 74 #define CR_AES_KP 0x00000038 75 #define CR_AES_GCM 0x00080000 76 #define CR_AES_CCM 0x00080008 77 #define CR_AES_UNKNOWN 0xFFFFFFFF 78 #define CR_ALGO_MASK 0x00080038 79 #define CR_DATA32 0x00000000 80 #define CR_DATA16 0x00000040 81 #define CR_DATA8 0x00000080 82 #define CR_DATA1 0x000000C0 83 #define CR_KEY128 0x00000000 84 #define CR_KEY192 0x00000100 85 #define CR_KEY256 0x00000200 86 #define CR_FFLUSH 0x00004000 87 #define CR_CRYPEN 0x00008000 88 #define CR_PH_INIT 0x00000000 89 #define CR_PH_HEADER 0x00010000 90 #define CR_PH_PAYLOAD 0x00020000 91 #define CR_PH_FINAL 0x00030000 92 #define CR_PH_MASK 0x00030000 93 #define CR_NBPBL_SHIFT 20 94 95 #define SR_BUSY 0x00000010 96 #define SR_OFNE 0x00000004 97 98 #define IMSCR_IN BIT(0) 99 #define IMSCR_OUT BIT(1) 100 101 #define MISR_IN BIT(0) 102 #define MISR_OUT BIT(1) 103 104 /* Misc */ 105 #define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32)) 106 #define GCM_CTR_INIT 2 107 #define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset) 108 #define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset) 109 #define CRYP_AUTOSUSPEND_DELAY 50 110 111 struct stm32_cryp_caps { 112 bool swap_final; 113 bool padding_wa; 114 }; 115 116 struct stm32_cryp_ctx { 117 struct crypto_engine_ctx enginectx; 118 struct stm32_cryp *cryp; 119 int keylen; 120 u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 121 unsigned long flags; 122 }; 123 124 struct stm32_cryp_reqctx { 125 unsigned long mode; 126 }; 127 128 struct stm32_cryp { 129 struct list_head list; 130 struct device *dev; 131 void __iomem *regs; 132 struct clk *clk; 133 unsigned long flags; 134 u32 irq_status; 135 const struct stm32_cryp_caps *caps; 136 struct stm32_cryp_ctx *ctx; 137 138 struct crypto_engine *engine; 139 140 struct ablkcipher_request *req; 141 struct aead_request *areq; 142 143 size_t authsize; 144 size_t hw_blocksize; 145 146 size_t total_in; 147 size_t total_in_save; 148 size_t total_out; 149 size_t total_out_save; 150 151 struct scatterlist *in_sg; 152 struct scatterlist *out_sg; 153 struct scatterlist *out_sg_save; 154 155 struct scatterlist in_sgl; 156 struct scatterlist out_sgl; 157 bool sgs_copied; 158 159 int in_sg_len; 160 int out_sg_len; 161 162 struct scatter_walk in_walk; 163 struct scatter_walk out_walk; 164 165 u32 last_ctr[4]; 166 u32 gcm_ctr; 167 }; 168 169 struct stm32_cryp_list { 170 struct list_head dev_list; 171 spinlock_t lock; /* protect dev_list */ 172 }; 173 174 static struct stm32_cryp_list cryp_list = { 175 .dev_list = LIST_HEAD_INIT(cryp_list.dev_list), 176 .lock = __SPIN_LOCK_UNLOCKED(cryp_list.lock), 177 }; 178 179 static inline bool is_aes(struct stm32_cryp *cryp) 180 { 181 return cryp->flags & FLG_AES; 182 } 183 184 static inline bool is_des(struct stm32_cryp *cryp) 185 { 186 return cryp->flags & FLG_DES; 187 } 188 189 static inline bool is_tdes(struct stm32_cryp *cryp) 190 { 191 return cryp->flags & FLG_TDES; 192 } 193 194 static inline bool is_ecb(struct stm32_cryp *cryp) 195 { 196 return cryp->flags & FLG_ECB; 197 } 198 199 static inline bool is_cbc(struct stm32_cryp *cryp) 200 { 201 return cryp->flags & FLG_CBC; 202 } 203 204 static inline bool is_ctr(struct stm32_cryp *cryp) 205 { 206 return cryp->flags & FLG_CTR; 207 } 208 209 static inline bool is_gcm(struct stm32_cryp *cryp) 210 { 211 return cryp->flags & FLG_GCM; 212 } 213 214 static inline bool is_ccm(struct stm32_cryp *cryp) 215 { 216 return cryp->flags & FLG_CCM; 217 } 218 219 static inline bool is_encrypt(struct stm32_cryp *cryp) 220 { 221 return cryp->flags & FLG_ENCRYPT; 222 } 223 224 static inline bool is_decrypt(struct stm32_cryp *cryp) 225 { 226 return !is_encrypt(cryp); 227 } 228 229 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) 230 { 231 return readl_relaxed(cryp->regs + ofst); 232 } 233 234 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) 235 { 236 writel_relaxed(val, cryp->regs + ofst); 237 } 238 239 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) 240 { 241 u32 status; 242 243 return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status, 244 !(status & SR_BUSY), 10, 100000); 245 } 246 247 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) 248 { 249 u32 status; 250 251 return readl_relaxed_poll_timeout(cryp->regs + CRYP_CR, status, 252 !(status & CR_CRYPEN), 10, 100000); 253 } 254 255 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) 256 { 257 u32 status; 258 259 return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status, 260 status & SR_OFNE, 10, 100000); 261 } 262 263 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp); 264 265 static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx) 266 { 267 struct stm32_cryp *tmp, *cryp = NULL; 268 269 spin_lock_bh(&cryp_list.lock); 270 if (!ctx->cryp) { 271 list_for_each_entry(tmp, &cryp_list.dev_list, list) { 272 cryp = tmp; 273 break; 274 } 275 ctx->cryp = cryp; 276 } else { 277 cryp = ctx->cryp; 278 } 279 280 spin_unlock_bh(&cryp_list.lock); 281 282 return cryp; 283 } 284 285 static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total, 286 size_t align) 287 { 288 int len = 0; 289 290 if (!total) 291 return 0; 292 293 if (!IS_ALIGNED(total, align)) 294 return -EINVAL; 295 296 while (sg) { 297 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 298 return -EINVAL; 299 300 if (!IS_ALIGNED(sg->length, align)) 301 return -EINVAL; 302 303 len += sg->length; 304 sg = sg_next(sg); 305 } 306 307 if (len != total) 308 return -EINVAL; 309 310 return 0; 311 } 312 313 static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp) 314 { 315 int ret; 316 317 ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in, 318 cryp->hw_blocksize); 319 if (ret) 320 return ret; 321 322 ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out, 323 cryp->hw_blocksize); 324 325 return ret; 326 } 327 328 static void sg_copy_buf(void *buf, struct scatterlist *sg, 329 unsigned int start, unsigned int nbytes, int out) 330 { 331 struct scatter_walk walk; 332 333 if (!nbytes) 334 return; 335 336 scatterwalk_start(&walk, sg); 337 scatterwalk_advance(&walk, start); 338 scatterwalk_copychunks(buf, &walk, nbytes, out); 339 scatterwalk_done(&walk, out, 0); 340 } 341 342 static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp) 343 { 344 void *buf_in, *buf_out; 345 int pages, total_in, total_out; 346 347 if (!stm32_cryp_check_io_aligned(cryp)) { 348 cryp->sgs_copied = 0; 349 return 0; 350 } 351 352 total_in = ALIGN(cryp->total_in, cryp->hw_blocksize); 353 pages = total_in ? get_order(total_in) : 1; 354 buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); 355 356 total_out = ALIGN(cryp->total_out, cryp->hw_blocksize); 357 pages = total_out ? get_order(total_out) : 1; 358 buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); 359 360 if (!buf_in || !buf_out) { 361 dev_err(cryp->dev, "Can't allocate pages when unaligned\n"); 362 cryp->sgs_copied = 0; 363 return -EFAULT; 364 } 365 366 sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0); 367 368 sg_init_one(&cryp->in_sgl, buf_in, total_in); 369 cryp->in_sg = &cryp->in_sgl; 370 cryp->in_sg_len = 1; 371 372 sg_init_one(&cryp->out_sgl, buf_out, total_out); 373 cryp->out_sg_save = cryp->out_sg; 374 cryp->out_sg = &cryp->out_sgl; 375 cryp->out_sg_len = 1; 376 377 cryp->sgs_copied = 1; 378 379 return 0; 380 } 381 382 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv) 383 { 384 if (!iv) 385 return; 386 387 stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++)); 388 stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++)); 389 390 if (is_aes(cryp)) { 391 stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++)); 392 stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++)); 393 } 394 } 395 396 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) 397 { 398 struct ablkcipher_request *req = cryp->req; 399 u32 *tmp = req->info; 400 401 if (!tmp) 402 return; 403 404 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0LR)); 405 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV0RR)); 406 407 if (is_aes(cryp)) { 408 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1LR)); 409 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, CRYP_IV1RR)); 410 } 411 } 412 413 static void stm32_cryp_hw_write_key(struct stm32_cryp *c) 414 { 415 unsigned int i; 416 int r_id; 417 418 if (is_des(c)) { 419 stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0])); 420 stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1])); 421 } else { 422 r_id = CRYP_K3RR; 423 for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4) 424 stm32_cryp_write(c, r_id, 425 cpu_to_be32(c->ctx->key[i - 1])); 426 } 427 } 428 429 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) 430 { 431 if (is_aes(cryp) && is_ecb(cryp)) 432 return CR_AES_ECB; 433 434 if (is_aes(cryp) && is_cbc(cryp)) 435 return CR_AES_CBC; 436 437 if (is_aes(cryp) && is_ctr(cryp)) 438 return CR_AES_CTR; 439 440 if (is_aes(cryp) && is_gcm(cryp)) 441 return CR_AES_GCM; 442 443 if (is_aes(cryp) && is_ccm(cryp)) 444 return CR_AES_CCM; 445 446 if (is_des(cryp) && is_ecb(cryp)) 447 return CR_DES_ECB; 448 449 if (is_des(cryp) && is_cbc(cryp)) 450 return CR_DES_CBC; 451 452 if (is_tdes(cryp) && is_ecb(cryp)) 453 return CR_TDES_ECB; 454 455 if (is_tdes(cryp) && is_cbc(cryp)) 456 return CR_TDES_CBC; 457 458 dev_err(cryp->dev, "Unknown mode\n"); 459 return CR_AES_UNKNOWN; 460 } 461 462 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) 463 { 464 return is_encrypt(cryp) ? cryp->areq->cryptlen : 465 cryp->areq->cryptlen - cryp->authsize; 466 } 467 468 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) 469 { 470 int ret; 471 u32 iv[4]; 472 473 /* Phase 1 : init */ 474 memcpy(iv, cryp->areq->iv, 12); 475 iv[3] = cpu_to_be32(GCM_CTR_INIT); 476 cryp->gcm_ctr = GCM_CTR_INIT; 477 stm32_cryp_hw_write_iv(cryp, iv); 478 479 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN); 480 481 /* Wait for end of processing */ 482 ret = stm32_cryp_wait_enable(cryp); 483 if (ret) 484 dev_err(cryp->dev, "Timeout (gcm init)\n"); 485 486 return ret; 487 } 488 489 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) 490 { 491 int ret; 492 u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE]; 493 u32 *d; 494 unsigned int i, textlen; 495 496 /* Phase 1 : init. Firstly set the CTR value to 1 (not 0) */ 497 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 498 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 499 iv[AES_BLOCK_SIZE - 1] = 1; 500 stm32_cryp_hw_write_iv(cryp, (u32 *)iv); 501 502 /* Build B0 */ 503 memcpy(b0, iv, AES_BLOCK_SIZE); 504 505 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); 506 507 if (cryp->areq->assoclen) 508 b0[0] |= 0x40; 509 510 textlen = stm32_cryp_get_input_text_len(cryp); 511 512 b0[AES_BLOCK_SIZE - 2] = textlen >> 8; 513 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF; 514 515 /* Enable HW */ 516 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_PH_INIT | CR_CRYPEN); 517 518 /* Write B0 */ 519 d = (u32 *)b0; 520 521 for (i = 0; i < AES_BLOCK_32; i++) { 522 if (!cryp->caps->padding_wa) 523 *d = cpu_to_be32(*d); 524 stm32_cryp_write(cryp, CRYP_DIN, *d++); 525 } 526 527 /* Wait for end of processing */ 528 ret = stm32_cryp_wait_enable(cryp); 529 if (ret) 530 dev_err(cryp->dev, "Timeout (ccm init)\n"); 531 532 return ret; 533 } 534 535 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) 536 { 537 int ret; 538 u32 cfg, hw_mode; 539 540 pm_runtime_get_sync(cryp->dev); 541 542 /* Disable interrupt */ 543 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 544 545 /* Set key */ 546 stm32_cryp_hw_write_key(cryp); 547 548 /* Set configuration */ 549 cfg = CR_DATA8 | CR_FFLUSH; 550 551 switch (cryp->ctx->keylen) { 552 case AES_KEYSIZE_128: 553 cfg |= CR_KEY128; 554 break; 555 556 case AES_KEYSIZE_192: 557 cfg |= CR_KEY192; 558 break; 559 560 default: 561 case AES_KEYSIZE_256: 562 cfg |= CR_KEY256; 563 break; 564 } 565 566 hw_mode = stm32_cryp_get_hw_mode(cryp); 567 if (hw_mode == CR_AES_UNKNOWN) 568 return -EINVAL; 569 570 /* AES ECB/CBC decrypt: run key preparation first */ 571 if (is_decrypt(cryp) && 572 ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) { 573 stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN); 574 575 /* Wait for end of processing */ 576 ret = stm32_cryp_wait_busy(cryp); 577 if (ret) { 578 dev_err(cryp->dev, "Timeout (key preparation)\n"); 579 return ret; 580 } 581 } 582 583 cfg |= hw_mode; 584 585 if (is_decrypt(cryp)) 586 cfg |= CR_DEC_NOT_ENC; 587 588 /* Apply config and flush (valid when CRYPEN = 0) */ 589 stm32_cryp_write(cryp, CRYP_CR, cfg); 590 591 switch (hw_mode) { 592 case CR_AES_GCM: 593 case CR_AES_CCM: 594 /* Phase 1 : init */ 595 if (hw_mode == CR_AES_CCM) 596 ret = stm32_cryp_ccm_init(cryp, cfg); 597 else 598 ret = stm32_cryp_gcm_init(cryp, cfg); 599 600 if (ret) 601 return ret; 602 603 /* Phase 2 : header (authenticated data) */ 604 if (cryp->areq->assoclen) { 605 cfg |= CR_PH_HEADER; 606 } else if (stm32_cryp_get_input_text_len(cryp)) { 607 cfg |= CR_PH_PAYLOAD; 608 stm32_cryp_write(cryp, CRYP_CR, cfg); 609 } else { 610 cfg |= CR_PH_INIT; 611 } 612 613 break; 614 615 case CR_DES_CBC: 616 case CR_TDES_CBC: 617 case CR_AES_CBC: 618 case CR_AES_CTR: 619 stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info); 620 break; 621 622 default: 623 break; 624 } 625 626 /* Enable now */ 627 cfg |= CR_CRYPEN; 628 629 stm32_cryp_write(cryp, CRYP_CR, cfg); 630 631 cryp->flags &= ~FLG_CCM_PADDED_WA; 632 633 return 0; 634 } 635 636 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) 637 { 638 if (!err && (is_gcm(cryp) || is_ccm(cryp))) 639 /* Phase 4 : output tag */ 640 err = stm32_cryp_read_auth_tag(cryp); 641 642 if (!err && (!(is_gcm(cryp) || is_ccm(cryp)))) 643 stm32_cryp_get_iv(cryp); 644 645 if (cryp->sgs_copied) { 646 void *buf_in, *buf_out; 647 int pages, len; 648 649 buf_in = sg_virt(&cryp->in_sgl); 650 buf_out = sg_virt(&cryp->out_sgl); 651 652 sg_copy_buf(buf_out, cryp->out_sg_save, 0, 653 cryp->total_out_save, 1); 654 655 len = ALIGN(cryp->total_in_save, cryp->hw_blocksize); 656 pages = len ? get_order(len) : 1; 657 free_pages((unsigned long)buf_in, pages); 658 659 len = ALIGN(cryp->total_out_save, cryp->hw_blocksize); 660 pages = len ? get_order(len) : 1; 661 free_pages((unsigned long)buf_out, pages); 662 } 663 664 pm_runtime_mark_last_busy(cryp->dev); 665 pm_runtime_put_autosuspend(cryp->dev); 666 667 if (is_gcm(cryp) || is_ccm(cryp)) 668 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); 669 else 670 crypto_finalize_ablkcipher_request(cryp->engine, cryp->req, 671 err); 672 673 memset(cryp->ctx->key, 0, cryp->ctx->keylen); 674 } 675 676 static int stm32_cryp_cpu_start(struct stm32_cryp *cryp) 677 { 678 /* Enable interrupt and let the IRQ handler do everything */ 679 stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT); 680 681 return 0; 682 } 683 684 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq); 685 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine, 686 void *areq); 687 688 static int stm32_cryp_cra_init(struct crypto_tfm *tfm) 689 { 690 struct stm32_cryp_ctx *ctx = crypto_tfm_ctx(tfm); 691 692 tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx); 693 694 ctx->enginectx.op.do_one_request = stm32_cryp_cipher_one_req; 695 ctx->enginectx.op.prepare_request = stm32_cryp_prepare_cipher_req; 696 ctx->enginectx.op.unprepare_request = NULL; 697 return 0; 698 } 699 700 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); 701 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, 702 void *areq); 703 704 static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) 705 { 706 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 707 708 tfm->reqsize = sizeof(struct stm32_cryp_reqctx); 709 710 ctx->enginectx.op.do_one_request = stm32_cryp_aead_one_req; 711 ctx->enginectx.op.prepare_request = stm32_cryp_prepare_aead_req; 712 ctx->enginectx.op.unprepare_request = NULL; 713 714 return 0; 715 } 716 717 static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode) 718 { 719 struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx( 720 crypto_ablkcipher_reqtfm(req)); 721 struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req); 722 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 723 724 if (!cryp) 725 return -ENODEV; 726 727 rctx->mode = mode; 728 729 return crypto_transfer_ablkcipher_request_to_engine(cryp->engine, req); 730 } 731 732 static int stm32_cryp_aead_crypt(struct aead_request *req, unsigned long mode) 733 { 734 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 735 struct stm32_cryp_reqctx *rctx = aead_request_ctx(req); 736 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); 737 738 if (!cryp) 739 return -ENODEV; 740 741 rctx->mode = mode; 742 743 return crypto_transfer_aead_request_to_engine(cryp->engine, req); 744 } 745 746 static int stm32_cryp_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 747 unsigned int keylen) 748 { 749 struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm); 750 751 memcpy(ctx->key, key, keylen); 752 ctx->keylen = keylen; 753 754 return 0; 755 } 756 757 static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 758 unsigned int keylen) 759 { 760 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 761 keylen != AES_KEYSIZE_256) 762 return -EINVAL; 763 else 764 return stm32_cryp_setkey(tfm, key, keylen); 765 } 766 767 static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 768 unsigned int keylen) 769 { 770 u32 tmp[DES_EXPKEY_WORDS]; 771 772 if (keylen != DES_KEY_SIZE) 773 return -EINVAL; 774 775 if ((crypto_ablkcipher_get_flags(tfm) & 776 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) && 777 unlikely(!des_ekey(tmp, key))) { 778 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); 779 return -EINVAL; 780 } 781 782 return stm32_cryp_setkey(tfm, key, keylen); 783 } 784 785 static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 786 unsigned int keylen) 787 { 788 u32 flags; 789 int err; 790 791 flags = crypto_ablkcipher_get_flags(tfm); 792 err = __des3_verify_key(&flags, key); 793 if (unlikely(err)) { 794 crypto_ablkcipher_set_flags(tfm, flags); 795 return err; 796 } 797 798 return stm32_cryp_setkey(tfm, key, keylen); 799 } 800 801 static int stm32_cryp_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, 802 unsigned int keylen) 803 { 804 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(tfm); 805 806 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 807 keylen != AES_KEYSIZE_256) 808 return -EINVAL; 809 810 memcpy(ctx->key, key, keylen); 811 ctx->keylen = keylen; 812 813 return 0; 814 } 815 816 static int stm32_cryp_aes_gcm_setauthsize(struct crypto_aead *tfm, 817 unsigned int authsize) 818 { 819 return authsize == AES_BLOCK_SIZE ? 0 : -EINVAL; 820 } 821 822 static int stm32_cryp_aes_ccm_setauthsize(struct crypto_aead *tfm, 823 unsigned int authsize) 824 { 825 switch (authsize) { 826 case 4: 827 case 6: 828 case 8: 829 case 10: 830 case 12: 831 case 14: 832 case 16: 833 break; 834 default: 835 return -EINVAL; 836 } 837 838 return 0; 839 } 840 841 static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req) 842 { 843 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT); 844 } 845 846 static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req) 847 { 848 return stm32_cryp_crypt(req, FLG_AES | FLG_ECB); 849 } 850 851 static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req) 852 { 853 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT); 854 } 855 856 static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req) 857 { 858 return stm32_cryp_crypt(req, FLG_AES | FLG_CBC); 859 } 860 861 static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req) 862 { 863 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT); 864 } 865 866 static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req) 867 { 868 return stm32_cryp_crypt(req, FLG_AES | FLG_CTR); 869 } 870 871 static int stm32_cryp_aes_gcm_encrypt(struct aead_request *req) 872 { 873 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM | FLG_ENCRYPT); 874 } 875 876 static int stm32_cryp_aes_gcm_decrypt(struct aead_request *req) 877 { 878 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_GCM); 879 } 880 881 static int stm32_cryp_aes_ccm_encrypt(struct aead_request *req) 882 { 883 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM | FLG_ENCRYPT); 884 } 885 886 static int stm32_cryp_aes_ccm_decrypt(struct aead_request *req) 887 { 888 return stm32_cryp_aead_crypt(req, FLG_AES | FLG_CCM); 889 } 890 891 static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req) 892 { 893 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT); 894 } 895 896 static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req) 897 { 898 return stm32_cryp_crypt(req, FLG_DES | FLG_ECB); 899 } 900 901 static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req) 902 { 903 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT); 904 } 905 906 static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req) 907 { 908 return stm32_cryp_crypt(req, FLG_DES | FLG_CBC); 909 } 910 911 static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req) 912 { 913 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT); 914 } 915 916 static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req) 917 { 918 return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB); 919 } 920 921 static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req) 922 { 923 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT); 924 } 925 926 static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req) 927 { 928 return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC); 929 } 930 931 static int stm32_cryp_prepare_req(struct ablkcipher_request *req, 932 struct aead_request *areq) 933 { 934 struct stm32_cryp_ctx *ctx; 935 struct stm32_cryp *cryp; 936 struct stm32_cryp_reqctx *rctx; 937 int ret; 938 939 if (!req && !areq) 940 return -EINVAL; 941 942 ctx = req ? crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)) : 943 crypto_aead_ctx(crypto_aead_reqtfm(areq)); 944 945 cryp = ctx->cryp; 946 947 if (!cryp) 948 return -ENODEV; 949 950 rctx = req ? ablkcipher_request_ctx(req) : aead_request_ctx(areq); 951 rctx->mode &= FLG_MODE_MASK; 952 953 ctx->cryp = cryp; 954 955 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; 956 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; 957 cryp->ctx = ctx; 958 959 if (req) { 960 cryp->req = req; 961 cryp->areq = NULL; 962 cryp->total_in = req->nbytes; 963 cryp->total_out = cryp->total_in; 964 } else { 965 /* 966 * Length of input and output data: 967 * Encryption case: 968 * INPUT = AssocData || PlainText 969 * <- assoclen -> <- cryptlen -> 970 * <------- total_in -----------> 971 * 972 * OUTPUT = AssocData || CipherText || AuthTag 973 * <- assoclen -> <- cryptlen -> <- authsize -> 974 * <---------------- total_out -----------------> 975 * 976 * Decryption case: 977 * INPUT = AssocData || CipherText || AuthTag 978 * <- assoclen -> <--------- cryptlen ---------> 979 * <- authsize -> 980 * <---------------- total_in ------------------> 981 * 982 * OUTPUT = AssocData || PlainText 983 * <- assoclen -> <- crypten - authsize -> 984 * <---------- total_out -----------------> 985 */ 986 cryp->areq = areq; 987 cryp->req = NULL; 988 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); 989 cryp->total_in = areq->assoclen + areq->cryptlen; 990 if (is_encrypt(cryp)) 991 /* Append auth tag to output */ 992 cryp->total_out = cryp->total_in + cryp->authsize; 993 else 994 /* No auth tag in output */ 995 cryp->total_out = cryp->total_in - cryp->authsize; 996 } 997 998 cryp->total_in_save = cryp->total_in; 999 cryp->total_out_save = cryp->total_out; 1000 1001 cryp->in_sg = req ? req->src : areq->src; 1002 cryp->out_sg = req ? req->dst : areq->dst; 1003 cryp->out_sg_save = cryp->out_sg; 1004 1005 cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in); 1006 if (cryp->in_sg_len < 0) { 1007 dev_err(cryp->dev, "Cannot get in_sg_len\n"); 1008 ret = cryp->in_sg_len; 1009 return ret; 1010 } 1011 1012 cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out); 1013 if (cryp->out_sg_len < 0) { 1014 dev_err(cryp->dev, "Cannot get out_sg_len\n"); 1015 ret = cryp->out_sg_len; 1016 return ret; 1017 } 1018 1019 ret = stm32_cryp_copy_sgs(cryp); 1020 if (ret) 1021 return ret; 1022 1023 scatterwalk_start(&cryp->in_walk, cryp->in_sg); 1024 scatterwalk_start(&cryp->out_walk, cryp->out_sg); 1025 1026 if (is_gcm(cryp) || is_ccm(cryp)) { 1027 /* In output, jump after assoc data */ 1028 scatterwalk_advance(&cryp->out_walk, cryp->areq->assoclen); 1029 cryp->total_out -= cryp->areq->assoclen; 1030 } 1031 1032 ret = stm32_cryp_hw_init(cryp); 1033 return ret; 1034 } 1035 1036 static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine, 1037 void *areq) 1038 { 1039 struct ablkcipher_request *req = container_of(areq, 1040 struct ablkcipher_request, 1041 base); 1042 1043 return stm32_cryp_prepare_req(req, NULL); 1044 } 1045 1046 static int stm32_cryp_cipher_one_req(struct crypto_engine *engine, void *areq) 1047 { 1048 struct ablkcipher_request *req = container_of(areq, 1049 struct ablkcipher_request, 1050 base); 1051 struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx( 1052 crypto_ablkcipher_reqtfm(req)); 1053 struct stm32_cryp *cryp = ctx->cryp; 1054 1055 if (!cryp) 1056 return -ENODEV; 1057 1058 return stm32_cryp_cpu_start(cryp); 1059 } 1060 1061 static int stm32_cryp_prepare_aead_req(struct crypto_engine *engine, void *areq) 1062 { 1063 struct aead_request *req = container_of(areq, struct aead_request, 1064 base); 1065 1066 return stm32_cryp_prepare_req(NULL, req); 1067 } 1068 1069 static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq) 1070 { 1071 struct aead_request *req = container_of(areq, struct aead_request, 1072 base); 1073 struct stm32_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1074 struct stm32_cryp *cryp = ctx->cryp; 1075 1076 if (!cryp) 1077 return -ENODEV; 1078 1079 if (unlikely(!cryp->areq->assoclen && 1080 !stm32_cryp_get_input_text_len(cryp))) { 1081 /* No input data to process: get tag and finish */ 1082 stm32_cryp_finish_req(cryp, 0); 1083 return 0; 1084 } 1085 1086 return stm32_cryp_cpu_start(cryp); 1087 } 1088 1089 static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst, 1090 unsigned int n) 1091 { 1092 scatterwalk_advance(&cryp->out_walk, n); 1093 1094 if (unlikely(cryp->out_sg->length == _walked_out)) { 1095 cryp->out_sg = sg_next(cryp->out_sg); 1096 if (cryp->out_sg) { 1097 scatterwalk_start(&cryp->out_walk, cryp->out_sg); 1098 return (sg_virt(cryp->out_sg) + _walked_out); 1099 } 1100 } 1101 1102 return (u32 *)((u8 *)dst + n); 1103 } 1104 1105 static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src, 1106 unsigned int n) 1107 { 1108 scatterwalk_advance(&cryp->in_walk, n); 1109 1110 if (unlikely(cryp->in_sg->length == _walked_in)) { 1111 cryp->in_sg = sg_next(cryp->in_sg); 1112 if (cryp->in_sg) { 1113 scatterwalk_start(&cryp->in_walk, cryp->in_sg); 1114 return (sg_virt(cryp->in_sg) + _walked_in); 1115 } 1116 } 1117 1118 return (u32 *)((u8 *)src + n); 1119 } 1120 1121 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) 1122 { 1123 u32 cfg, size_bit, *dst, d32; 1124 u8 *d8; 1125 unsigned int i, j; 1126 int ret = 0; 1127 1128 /* Update Config */ 1129 cfg = stm32_cryp_read(cryp, CRYP_CR); 1130 1131 cfg &= ~CR_PH_MASK; 1132 cfg |= CR_PH_FINAL; 1133 cfg &= ~CR_DEC_NOT_ENC; 1134 cfg |= CR_CRYPEN; 1135 1136 stm32_cryp_write(cryp, CRYP_CR, cfg); 1137 1138 if (is_gcm(cryp)) { 1139 /* GCM: write aad and payload size (in bits) */ 1140 size_bit = cryp->areq->assoclen * 8; 1141 if (cryp->caps->swap_final) 1142 size_bit = cpu_to_be32(size_bit); 1143 1144 stm32_cryp_write(cryp, CRYP_DIN, 0); 1145 stm32_cryp_write(cryp, CRYP_DIN, size_bit); 1146 1147 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : 1148 cryp->areq->cryptlen - AES_BLOCK_SIZE; 1149 size_bit *= 8; 1150 if (cryp->caps->swap_final) 1151 size_bit = cpu_to_be32(size_bit); 1152 1153 stm32_cryp_write(cryp, CRYP_DIN, 0); 1154 stm32_cryp_write(cryp, CRYP_DIN, size_bit); 1155 } else { 1156 /* CCM: write CTR0 */ 1157 u8 iv[AES_BLOCK_SIZE]; 1158 u32 *iv32 = (u32 *)iv; 1159 1160 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); 1161 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1); 1162 1163 for (i = 0; i < AES_BLOCK_32; i++) { 1164 if (!cryp->caps->padding_wa) 1165 *iv32 = cpu_to_be32(*iv32); 1166 stm32_cryp_write(cryp, CRYP_DIN, *iv32++); 1167 } 1168 } 1169 1170 /* Wait for output data */ 1171 ret = stm32_cryp_wait_output(cryp); 1172 if (ret) { 1173 dev_err(cryp->dev, "Timeout (read tag)\n"); 1174 return ret; 1175 } 1176 1177 if (is_encrypt(cryp)) { 1178 /* Get and write tag */ 1179 dst = sg_virt(cryp->out_sg) + _walked_out; 1180 1181 for (i = 0; i < AES_BLOCK_32; i++) { 1182 if (cryp->total_out >= sizeof(u32)) { 1183 /* Read a full u32 */ 1184 *dst = stm32_cryp_read(cryp, CRYP_DOUT); 1185 1186 dst = stm32_cryp_next_out(cryp, dst, 1187 sizeof(u32)); 1188 cryp->total_out -= sizeof(u32); 1189 } else if (!cryp->total_out) { 1190 /* Empty fifo out (data from input padding) */ 1191 stm32_cryp_read(cryp, CRYP_DOUT); 1192 } else { 1193 /* Read less than an u32 */ 1194 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1195 d8 = (u8 *)&d32; 1196 1197 for (j = 0; j < cryp->total_out; j++) { 1198 *((u8 *)dst) = *(d8++); 1199 dst = stm32_cryp_next_out(cryp, dst, 1); 1200 } 1201 cryp->total_out = 0; 1202 } 1203 } 1204 } else { 1205 /* Get and check tag */ 1206 u32 in_tag[AES_BLOCK_32], out_tag[AES_BLOCK_32]; 1207 1208 scatterwalk_map_and_copy(in_tag, cryp->in_sg, 1209 cryp->total_in_save - cryp->authsize, 1210 cryp->authsize, 0); 1211 1212 for (i = 0; i < AES_BLOCK_32; i++) 1213 out_tag[i] = stm32_cryp_read(cryp, CRYP_DOUT); 1214 1215 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) 1216 ret = -EBADMSG; 1217 } 1218 1219 /* Disable cryp */ 1220 cfg &= ~CR_CRYPEN; 1221 stm32_cryp_write(cryp, CRYP_CR, cfg); 1222 1223 return ret; 1224 } 1225 1226 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) 1227 { 1228 u32 cr; 1229 1230 if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) { 1231 cryp->last_ctr[3] = 0; 1232 cryp->last_ctr[2]++; 1233 if (!cryp->last_ctr[2]) { 1234 cryp->last_ctr[1]++; 1235 if (!cryp->last_ctr[1]) 1236 cryp->last_ctr[0]++; 1237 } 1238 1239 cr = stm32_cryp_read(cryp, CRYP_CR); 1240 stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); 1241 1242 stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr); 1243 1244 stm32_cryp_write(cryp, CRYP_CR, cr); 1245 } 1246 1247 cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR); 1248 cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR); 1249 cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR); 1250 cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR); 1251 } 1252 1253 static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp) 1254 { 1255 unsigned int i, j; 1256 u32 d32, *dst; 1257 u8 *d8; 1258 size_t tag_size; 1259 1260 /* Do no read tag now (if any) */ 1261 if (is_encrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) 1262 tag_size = cryp->authsize; 1263 else 1264 tag_size = 0; 1265 1266 dst = sg_virt(cryp->out_sg) + _walked_out; 1267 1268 for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { 1269 if (likely(cryp->total_out - tag_size >= sizeof(u32))) { 1270 /* Read a full u32 */ 1271 *dst = stm32_cryp_read(cryp, CRYP_DOUT); 1272 1273 dst = stm32_cryp_next_out(cryp, dst, sizeof(u32)); 1274 cryp->total_out -= sizeof(u32); 1275 } else if (cryp->total_out == tag_size) { 1276 /* Empty fifo out (data from input padding) */ 1277 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1278 } else { 1279 /* Read less than an u32 */ 1280 d32 = stm32_cryp_read(cryp, CRYP_DOUT); 1281 d8 = (u8 *)&d32; 1282 1283 for (j = 0; j < cryp->total_out - tag_size; j++) { 1284 *((u8 *)dst) = *(d8++); 1285 dst = stm32_cryp_next_out(cryp, dst, 1); 1286 } 1287 cryp->total_out = tag_size; 1288 } 1289 } 1290 1291 return !(cryp->total_out - tag_size) || !cryp->total_in; 1292 } 1293 1294 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) 1295 { 1296 unsigned int i, j; 1297 u32 *src; 1298 u8 d8[4]; 1299 size_t tag_size; 1300 1301 /* Do no write tag (if any) */ 1302 if (is_decrypt(cryp) && (is_gcm(cryp) || is_ccm(cryp))) 1303 tag_size = cryp->authsize; 1304 else 1305 tag_size = 0; 1306 1307 src = sg_virt(cryp->in_sg) + _walked_in; 1308 1309 for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) { 1310 if (likely(cryp->total_in - tag_size >= sizeof(u32))) { 1311 /* Write a full u32 */ 1312 stm32_cryp_write(cryp, CRYP_DIN, *src); 1313 1314 src = stm32_cryp_next_in(cryp, src, sizeof(u32)); 1315 cryp->total_in -= sizeof(u32); 1316 } else if (cryp->total_in == tag_size) { 1317 /* Write padding data */ 1318 stm32_cryp_write(cryp, CRYP_DIN, 0); 1319 } else { 1320 /* Write less than an u32 */ 1321 memset(d8, 0, sizeof(u32)); 1322 for (j = 0; j < cryp->total_in - tag_size; j++) { 1323 d8[j] = *((u8 *)src); 1324 src = stm32_cryp_next_in(cryp, src, 1); 1325 } 1326 1327 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1328 cryp->total_in = tag_size; 1329 } 1330 } 1331 } 1332 1333 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) 1334 { 1335 int err; 1336 u32 cfg, tmp[AES_BLOCK_32]; 1337 size_t total_in_ori = cryp->total_in; 1338 struct scatterlist *out_sg_ori = cryp->out_sg; 1339 unsigned int i; 1340 1341 /* 'Special workaround' procedure described in the datasheet */ 1342 1343 /* a) disable ip */ 1344 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1345 cfg = stm32_cryp_read(cryp, CRYP_CR); 1346 cfg &= ~CR_CRYPEN; 1347 stm32_cryp_write(cryp, CRYP_CR, cfg); 1348 1349 /* b) Update IV1R */ 1350 stm32_cryp_write(cryp, CRYP_IV1RR, cryp->gcm_ctr - 2); 1351 1352 /* c) change mode to CTR */ 1353 cfg &= ~CR_ALGO_MASK; 1354 cfg |= CR_AES_CTR; 1355 stm32_cryp_write(cryp, CRYP_CR, cfg); 1356 1357 /* a) enable IP */ 1358 cfg |= CR_CRYPEN; 1359 stm32_cryp_write(cryp, CRYP_CR, cfg); 1360 1361 /* b) pad and write the last block */ 1362 stm32_cryp_irq_write_block(cryp); 1363 cryp->total_in = total_in_ori; 1364 err = stm32_cryp_wait_output(cryp); 1365 if (err) { 1366 dev_err(cryp->dev, "Timeout (write gcm header)\n"); 1367 return stm32_cryp_finish_req(cryp, err); 1368 } 1369 1370 /* c) get and store encrypted data */ 1371 stm32_cryp_irq_read_data(cryp); 1372 scatterwalk_map_and_copy(tmp, out_sg_ori, 1373 cryp->total_in_save - total_in_ori, 1374 total_in_ori, 0); 1375 1376 /* d) change mode back to AES GCM */ 1377 cfg &= ~CR_ALGO_MASK; 1378 cfg |= CR_AES_GCM; 1379 stm32_cryp_write(cryp, CRYP_CR, cfg); 1380 1381 /* e) change phase to Final */ 1382 cfg &= ~CR_PH_MASK; 1383 cfg |= CR_PH_FINAL; 1384 stm32_cryp_write(cryp, CRYP_CR, cfg); 1385 1386 /* f) write padded data */ 1387 for (i = 0; i < AES_BLOCK_32; i++) { 1388 if (cryp->total_in) 1389 stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); 1390 else 1391 stm32_cryp_write(cryp, CRYP_DIN, 0); 1392 1393 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); 1394 } 1395 1396 /* g) Empty fifo out */ 1397 err = stm32_cryp_wait_output(cryp); 1398 if (err) { 1399 dev_err(cryp->dev, "Timeout (write gcm header)\n"); 1400 return stm32_cryp_finish_req(cryp, err); 1401 } 1402 1403 for (i = 0; i < AES_BLOCK_32; i++) 1404 stm32_cryp_read(cryp, CRYP_DOUT); 1405 1406 /* h) run the he normal Final phase */ 1407 stm32_cryp_finish_req(cryp, 0); 1408 } 1409 1410 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) 1411 { 1412 u32 cfg, payload_bytes; 1413 1414 /* disable ip, set NPBLB and reneable ip */ 1415 cfg = stm32_cryp_read(cryp, CRYP_CR); 1416 cfg &= ~CR_CRYPEN; 1417 stm32_cryp_write(cryp, CRYP_CR, cfg); 1418 1419 payload_bytes = is_decrypt(cryp) ? cryp->total_in - cryp->authsize : 1420 cryp->total_in; 1421 cfg |= (cryp->hw_blocksize - payload_bytes) << CR_NBPBL_SHIFT; 1422 cfg |= CR_CRYPEN; 1423 stm32_cryp_write(cryp, CRYP_CR, cfg); 1424 } 1425 1426 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) 1427 { 1428 int err = 0; 1429 u32 cfg, iv1tmp; 1430 u32 cstmp1[AES_BLOCK_32], cstmp2[AES_BLOCK_32], tmp[AES_BLOCK_32]; 1431 size_t last_total_out, total_in_ori = cryp->total_in; 1432 struct scatterlist *out_sg_ori = cryp->out_sg; 1433 unsigned int i; 1434 1435 /* 'Special workaround' procedure described in the datasheet */ 1436 cryp->flags |= FLG_CCM_PADDED_WA; 1437 1438 /* a) disable ip */ 1439 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1440 1441 cfg = stm32_cryp_read(cryp, CRYP_CR); 1442 cfg &= ~CR_CRYPEN; 1443 stm32_cryp_write(cryp, CRYP_CR, cfg); 1444 1445 /* b) get IV1 from CRYP_CSGCMCCM7 */ 1446 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); 1447 1448 /* c) Load CRYP_CSGCMCCMxR */ 1449 for (i = 0; i < ARRAY_SIZE(cstmp1); i++) 1450 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1451 1452 /* d) Write IV1R */ 1453 stm32_cryp_write(cryp, CRYP_IV1RR, iv1tmp); 1454 1455 /* e) change mode to CTR */ 1456 cfg &= ~CR_ALGO_MASK; 1457 cfg |= CR_AES_CTR; 1458 stm32_cryp_write(cryp, CRYP_CR, cfg); 1459 1460 /* a) enable IP */ 1461 cfg |= CR_CRYPEN; 1462 stm32_cryp_write(cryp, CRYP_CR, cfg); 1463 1464 /* b) pad and write the last block */ 1465 stm32_cryp_irq_write_block(cryp); 1466 cryp->total_in = total_in_ori; 1467 err = stm32_cryp_wait_output(cryp); 1468 if (err) { 1469 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n"); 1470 return stm32_cryp_finish_req(cryp, err); 1471 } 1472 1473 /* c) get and store decrypted data */ 1474 last_total_out = cryp->total_out; 1475 stm32_cryp_irq_read_data(cryp); 1476 1477 memset(tmp, 0, sizeof(tmp)); 1478 scatterwalk_map_and_copy(tmp, out_sg_ori, 1479 cryp->total_out_save - last_total_out, 1480 last_total_out, 0); 1481 1482 /* d) Load again CRYP_CSGCMCCMxR */ 1483 for (i = 0; i < ARRAY_SIZE(cstmp2); i++) 1484 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); 1485 1486 /* e) change mode back to AES CCM */ 1487 cfg &= ~CR_ALGO_MASK; 1488 cfg |= CR_AES_CCM; 1489 stm32_cryp_write(cryp, CRYP_CR, cfg); 1490 1491 /* f) change phase to header */ 1492 cfg &= ~CR_PH_MASK; 1493 cfg |= CR_PH_HEADER; 1494 stm32_cryp_write(cryp, CRYP_CR, cfg); 1495 1496 /* g) XOR and write padded data */ 1497 for (i = 0; i < ARRAY_SIZE(tmp); i++) { 1498 tmp[i] ^= cstmp1[i]; 1499 tmp[i] ^= cstmp2[i]; 1500 stm32_cryp_write(cryp, CRYP_DIN, tmp[i]); 1501 } 1502 1503 /* h) wait for completion */ 1504 err = stm32_cryp_wait_busy(cryp); 1505 if (err) 1506 dev_err(cryp->dev, "Timeout (wite ccm padded data)\n"); 1507 1508 /* i) run the he normal Final phase */ 1509 stm32_cryp_finish_req(cryp, err); 1510 } 1511 1512 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) 1513 { 1514 if (unlikely(!cryp->total_in)) { 1515 dev_warn(cryp->dev, "No more data to process\n"); 1516 return; 1517 } 1518 1519 if (unlikely(cryp->total_in < AES_BLOCK_SIZE && 1520 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && 1521 is_encrypt(cryp))) { 1522 /* Padding for AES GCM encryption */ 1523 if (cryp->caps->padding_wa) 1524 /* Special case 1 */ 1525 return stm32_cryp_irq_write_gcm_padded_data(cryp); 1526 1527 /* Setting padding bytes (NBBLB) */ 1528 stm32_cryp_irq_set_npblb(cryp); 1529 } 1530 1531 if (unlikely((cryp->total_in - cryp->authsize < AES_BLOCK_SIZE) && 1532 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && 1533 is_decrypt(cryp))) { 1534 /* Padding for AES CCM decryption */ 1535 if (cryp->caps->padding_wa) 1536 /* Special case 2 */ 1537 return stm32_cryp_irq_write_ccm_padded_data(cryp); 1538 1539 /* Setting padding bytes (NBBLB) */ 1540 stm32_cryp_irq_set_npblb(cryp); 1541 } 1542 1543 if (is_aes(cryp) && is_ctr(cryp)) 1544 stm32_cryp_check_ctr_counter(cryp); 1545 1546 stm32_cryp_irq_write_block(cryp); 1547 } 1548 1549 static void stm32_cryp_irq_write_gcm_header(struct stm32_cryp *cryp) 1550 { 1551 int err; 1552 unsigned int i, j; 1553 u32 cfg, *src; 1554 1555 src = sg_virt(cryp->in_sg) + _walked_in; 1556 1557 for (i = 0; i < AES_BLOCK_32; i++) { 1558 stm32_cryp_write(cryp, CRYP_DIN, *src); 1559 1560 src = stm32_cryp_next_in(cryp, src, sizeof(u32)); 1561 cryp->total_in -= min_t(size_t, sizeof(u32), cryp->total_in); 1562 1563 /* Check if whole header written */ 1564 if ((cryp->total_in_save - cryp->total_in) == 1565 cryp->areq->assoclen) { 1566 /* Write padding if needed */ 1567 for (j = i + 1; j < AES_BLOCK_32; j++) 1568 stm32_cryp_write(cryp, CRYP_DIN, 0); 1569 1570 /* Wait for completion */ 1571 err = stm32_cryp_wait_busy(cryp); 1572 if (err) { 1573 dev_err(cryp->dev, "Timeout (gcm header)\n"); 1574 return stm32_cryp_finish_req(cryp, err); 1575 } 1576 1577 if (stm32_cryp_get_input_text_len(cryp)) { 1578 /* Phase 3 : payload */ 1579 cfg = stm32_cryp_read(cryp, CRYP_CR); 1580 cfg &= ~CR_CRYPEN; 1581 stm32_cryp_write(cryp, CRYP_CR, cfg); 1582 1583 cfg &= ~CR_PH_MASK; 1584 cfg |= CR_PH_PAYLOAD; 1585 cfg |= CR_CRYPEN; 1586 stm32_cryp_write(cryp, CRYP_CR, cfg); 1587 } else { 1588 /* Phase 4 : tag */ 1589 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1590 stm32_cryp_finish_req(cryp, 0); 1591 } 1592 1593 break; 1594 } 1595 1596 if (!cryp->total_in) 1597 break; 1598 } 1599 } 1600 1601 static void stm32_cryp_irq_write_ccm_header(struct stm32_cryp *cryp) 1602 { 1603 int err; 1604 unsigned int i = 0, j, k; 1605 u32 alen, cfg, *src; 1606 u8 d8[4]; 1607 1608 src = sg_virt(cryp->in_sg) + _walked_in; 1609 alen = cryp->areq->assoclen; 1610 1611 if (!_walked_in) { 1612 if (cryp->areq->assoclen <= 65280) { 1613 /* Write first u32 of B1 */ 1614 d8[0] = (alen >> 8) & 0xFF; 1615 d8[1] = alen & 0xFF; 1616 d8[2] = *((u8 *)src); 1617 src = stm32_cryp_next_in(cryp, src, 1); 1618 d8[3] = *((u8 *)src); 1619 src = stm32_cryp_next_in(cryp, src, 1); 1620 1621 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1622 i++; 1623 1624 cryp->total_in -= min_t(size_t, 2, cryp->total_in); 1625 } else { 1626 /* Build the two first u32 of B1 */ 1627 d8[0] = 0xFF; 1628 d8[1] = 0xFE; 1629 d8[2] = alen & 0xFF000000; 1630 d8[3] = alen & 0x00FF0000; 1631 1632 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1633 i++; 1634 1635 d8[0] = alen & 0x0000FF00; 1636 d8[1] = alen & 0x000000FF; 1637 d8[2] = *((u8 *)src); 1638 src = stm32_cryp_next_in(cryp, src, 1); 1639 d8[3] = *((u8 *)src); 1640 src = stm32_cryp_next_in(cryp, src, 1); 1641 1642 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1643 i++; 1644 1645 cryp->total_in -= min_t(size_t, 2, cryp->total_in); 1646 } 1647 } 1648 1649 /* Write next u32 */ 1650 for (; i < AES_BLOCK_32; i++) { 1651 /* Build an u32 */ 1652 memset(d8, 0, sizeof(u32)); 1653 for (k = 0; k < sizeof(u32); k++) { 1654 d8[k] = *((u8 *)src); 1655 src = stm32_cryp_next_in(cryp, src, 1); 1656 1657 cryp->total_in -= min_t(size_t, 1, cryp->total_in); 1658 if ((cryp->total_in_save - cryp->total_in) == alen) 1659 break; 1660 } 1661 1662 stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8); 1663 1664 if ((cryp->total_in_save - cryp->total_in) == alen) { 1665 /* Write padding if needed */ 1666 for (j = i + 1; j < AES_BLOCK_32; j++) 1667 stm32_cryp_write(cryp, CRYP_DIN, 0); 1668 1669 /* Wait for completion */ 1670 err = stm32_cryp_wait_busy(cryp); 1671 if (err) { 1672 dev_err(cryp->dev, "Timeout (ccm header)\n"); 1673 return stm32_cryp_finish_req(cryp, err); 1674 } 1675 1676 if (stm32_cryp_get_input_text_len(cryp)) { 1677 /* Phase 3 : payload */ 1678 cfg = stm32_cryp_read(cryp, CRYP_CR); 1679 cfg &= ~CR_CRYPEN; 1680 stm32_cryp_write(cryp, CRYP_CR, cfg); 1681 1682 cfg &= ~CR_PH_MASK; 1683 cfg |= CR_PH_PAYLOAD; 1684 cfg |= CR_CRYPEN; 1685 stm32_cryp_write(cryp, CRYP_CR, cfg); 1686 } else { 1687 /* Phase 4 : tag */ 1688 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1689 stm32_cryp_finish_req(cryp, 0); 1690 } 1691 1692 break; 1693 } 1694 } 1695 } 1696 1697 static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg) 1698 { 1699 struct stm32_cryp *cryp = arg; 1700 u32 ph; 1701 1702 if (cryp->irq_status & MISR_OUT) 1703 /* Output FIFO IRQ: read data */ 1704 if (unlikely(stm32_cryp_irq_read_data(cryp))) { 1705 /* All bytes processed, finish */ 1706 stm32_cryp_write(cryp, CRYP_IMSCR, 0); 1707 stm32_cryp_finish_req(cryp, 0); 1708 return IRQ_HANDLED; 1709 } 1710 1711 if (cryp->irq_status & MISR_IN) { 1712 if (is_gcm(cryp)) { 1713 ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; 1714 if (unlikely(ph == CR_PH_HEADER)) 1715 /* Write Header */ 1716 stm32_cryp_irq_write_gcm_header(cryp); 1717 else 1718 /* Input FIFO IRQ: write data */ 1719 stm32_cryp_irq_write_data(cryp); 1720 cryp->gcm_ctr++; 1721 } else if (is_ccm(cryp)) { 1722 ph = stm32_cryp_read(cryp, CRYP_CR) & CR_PH_MASK; 1723 if (unlikely(ph == CR_PH_HEADER)) 1724 /* Write Header */ 1725 stm32_cryp_irq_write_ccm_header(cryp); 1726 else 1727 /* Input FIFO IRQ: write data */ 1728 stm32_cryp_irq_write_data(cryp); 1729 } else { 1730 /* Input FIFO IRQ: write data */ 1731 stm32_cryp_irq_write_data(cryp); 1732 } 1733 } 1734 1735 return IRQ_HANDLED; 1736 } 1737 1738 static irqreturn_t stm32_cryp_irq(int irq, void *arg) 1739 { 1740 struct stm32_cryp *cryp = arg; 1741 1742 cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR); 1743 1744 return IRQ_WAKE_THREAD; 1745 } 1746 1747 static struct crypto_alg crypto_algs[] = { 1748 { 1749 .cra_name = "ecb(aes)", 1750 .cra_driver_name = "stm32-ecb-aes", 1751 .cra_priority = 200, 1752 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1753 CRYPTO_ALG_ASYNC, 1754 .cra_blocksize = AES_BLOCK_SIZE, 1755 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1756 .cra_alignmask = 0xf, 1757 .cra_type = &crypto_ablkcipher_type, 1758 .cra_module = THIS_MODULE, 1759 .cra_init = stm32_cryp_cra_init, 1760 .cra_ablkcipher = { 1761 .min_keysize = AES_MIN_KEY_SIZE, 1762 .max_keysize = AES_MAX_KEY_SIZE, 1763 .setkey = stm32_cryp_aes_setkey, 1764 .encrypt = stm32_cryp_aes_ecb_encrypt, 1765 .decrypt = stm32_cryp_aes_ecb_decrypt, 1766 } 1767 }, 1768 { 1769 .cra_name = "cbc(aes)", 1770 .cra_driver_name = "stm32-cbc-aes", 1771 .cra_priority = 200, 1772 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1773 CRYPTO_ALG_ASYNC, 1774 .cra_blocksize = AES_BLOCK_SIZE, 1775 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1776 .cra_alignmask = 0xf, 1777 .cra_type = &crypto_ablkcipher_type, 1778 .cra_module = THIS_MODULE, 1779 .cra_init = stm32_cryp_cra_init, 1780 .cra_ablkcipher = { 1781 .min_keysize = AES_MIN_KEY_SIZE, 1782 .max_keysize = AES_MAX_KEY_SIZE, 1783 .ivsize = AES_BLOCK_SIZE, 1784 .setkey = stm32_cryp_aes_setkey, 1785 .encrypt = stm32_cryp_aes_cbc_encrypt, 1786 .decrypt = stm32_cryp_aes_cbc_decrypt, 1787 } 1788 }, 1789 { 1790 .cra_name = "ctr(aes)", 1791 .cra_driver_name = "stm32-ctr-aes", 1792 .cra_priority = 200, 1793 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1794 CRYPTO_ALG_ASYNC, 1795 .cra_blocksize = 1, 1796 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1797 .cra_alignmask = 0xf, 1798 .cra_type = &crypto_ablkcipher_type, 1799 .cra_module = THIS_MODULE, 1800 .cra_init = stm32_cryp_cra_init, 1801 .cra_ablkcipher = { 1802 .min_keysize = AES_MIN_KEY_SIZE, 1803 .max_keysize = AES_MAX_KEY_SIZE, 1804 .ivsize = AES_BLOCK_SIZE, 1805 .setkey = stm32_cryp_aes_setkey, 1806 .encrypt = stm32_cryp_aes_ctr_encrypt, 1807 .decrypt = stm32_cryp_aes_ctr_decrypt, 1808 } 1809 }, 1810 { 1811 .cra_name = "ecb(des)", 1812 .cra_driver_name = "stm32-ecb-des", 1813 .cra_priority = 200, 1814 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1815 CRYPTO_ALG_ASYNC, 1816 .cra_blocksize = DES_BLOCK_SIZE, 1817 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1818 .cra_alignmask = 0xf, 1819 .cra_type = &crypto_ablkcipher_type, 1820 .cra_module = THIS_MODULE, 1821 .cra_init = stm32_cryp_cra_init, 1822 .cra_ablkcipher = { 1823 .min_keysize = DES_BLOCK_SIZE, 1824 .max_keysize = DES_BLOCK_SIZE, 1825 .setkey = stm32_cryp_des_setkey, 1826 .encrypt = stm32_cryp_des_ecb_encrypt, 1827 .decrypt = stm32_cryp_des_ecb_decrypt, 1828 } 1829 }, 1830 { 1831 .cra_name = "cbc(des)", 1832 .cra_driver_name = "stm32-cbc-des", 1833 .cra_priority = 200, 1834 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1835 CRYPTO_ALG_ASYNC, 1836 .cra_blocksize = DES_BLOCK_SIZE, 1837 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1838 .cra_alignmask = 0xf, 1839 .cra_type = &crypto_ablkcipher_type, 1840 .cra_module = THIS_MODULE, 1841 .cra_init = stm32_cryp_cra_init, 1842 .cra_ablkcipher = { 1843 .min_keysize = DES_BLOCK_SIZE, 1844 .max_keysize = DES_BLOCK_SIZE, 1845 .ivsize = DES_BLOCK_SIZE, 1846 .setkey = stm32_cryp_des_setkey, 1847 .encrypt = stm32_cryp_des_cbc_encrypt, 1848 .decrypt = stm32_cryp_des_cbc_decrypt, 1849 } 1850 }, 1851 { 1852 .cra_name = "ecb(des3_ede)", 1853 .cra_driver_name = "stm32-ecb-des3", 1854 .cra_priority = 200, 1855 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1856 CRYPTO_ALG_ASYNC, 1857 .cra_blocksize = DES_BLOCK_SIZE, 1858 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1859 .cra_alignmask = 0xf, 1860 .cra_type = &crypto_ablkcipher_type, 1861 .cra_module = THIS_MODULE, 1862 .cra_init = stm32_cryp_cra_init, 1863 .cra_ablkcipher = { 1864 .min_keysize = 3 * DES_BLOCK_SIZE, 1865 .max_keysize = 3 * DES_BLOCK_SIZE, 1866 .setkey = stm32_cryp_tdes_setkey, 1867 .encrypt = stm32_cryp_tdes_ecb_encrypt, 1868 .decrypt = stm32_cryp_tdes_ecb_decrypt, 1869 } 1870 }, 1871 { 1872 .cra_name = "cbc(des3_ede)", 1873 .cra_driver_name = "stm32-cbc-des3", 1874 .cra_priority = 200, 1875 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | 1876 CRYPTO_ALG_ASYNC, 1877 .cra_blocksize = DES_BLOCK_SIZE, 1878 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1879 .cra_alignmask = 0xf, 1880 .cra_type = &crypto_ablkcipher_type, 1881 .cra_module = THIS_MODULE, 1882 .cra_init = stm32_cryp_cra_init, 1883 .cra_ablkcipher = { 1884 .min_keysize = 3 * DES_BLOCK_SIZE, 1885 .max_keysize = 3 * DES_BLOCK_SIZE, 1886 .ivsize = DES_BLOCK_SIZE, 1887 .setkey = stm32_cryp_tdes_setkey, 1888 .encrypt = stm32_cryp_tdes_cbc_encrypt, 1889 .decrypt = stm32_cryp_tdes_cbc_decrypt, 1890 } 1891 }, 1892 }; 1893 1894 static struct aead_alg aead_algs[] = { 1895 { 1896 .setkey = stm32_cryp_aes_aead_setkey, 1897 .setauthsize = stm32_cryp_aes_gcm_setauthsize, 1898 .encrypt = stm32_cryp_aes_gcm_encrypt, 1899 .decrypt = stm32_cryp_aes_gcm_decrypt, 1900 .init = stm32_cryp_aes_aead_init, 1901 .ivsize = 12, 1902 .maxauthsize = AES_BLOCK_SIZE, 1903 1904 .base = { 1905 .cra_name = "gcm(aes)", 1906 .cra_driver_name = "stm32-gcm-aes", 1907 .cra_priority = 200, 1908 .cra_flags = CRYPTO_ALG_ASYNC, 1909 .cra_blocksize = 1, 1910 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1911 .cra_alignmask = 0xf, 1912 .cra_module = THIS_MODULE, 1913 }, 1914 }, 1915 { 1916 .setkey = stm32_cryp_aes_aead_setkey, 1917 .setauthsize = stm32_cryp_aes_ccm_setauthsize, 1918 .encrypt = stm32_cryp_aes_ccm_encrypt, 1919 .decrypt = stm32_cryp_aes_ccm_decrypt, 1920 .init = stm32_cryp_aes_aead_init, 1921 .ivsize = AES_BLOCK_SIZE, 1922 .maxauthsize = AES_BLOCK_SIZE, 1923 1924 .base = { 1925 .cra_name = "ccm(aes)", 1926 .cra_driver_name = "stm32-ccm-aes", 1927 .cra_priority = 200, 1928 .cra_flags = CRYPTO_ALG_ASYNC, 1929 .cra_blocksize = 1, 1930 .cra_ctxsize = sizeof(struct stm32_cryp_ctx), 1931 .cra_alignmask = 0xf, 1932 .cra_module = THIS_MODULE, 1933 }, 1934 }, 1935 }; 1936 1937 static const struct stm32_cryp_caps f7_data = { 1938 .swap_final = true, 1939 .padding_wa = true, 1940 }; 1941 1942 static const struct stm32_cryp_caps mp1_data = { 1943 .swap_final = false, 1944 .padding_wa = false, 1945 }; 1946 1947 static const struct of_device_id stm32_dt_ids[] = { 1948 { .compatible = "st,stm32f756-cryp", .data = &f7_data}, 1949 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data}, 1950 {}, 1951 }; 1952 MODULE_DEVICE_TABLE(of, stm32_dt_ids); 1953 1954 static int stm32_cryp_probe(struct platform_device *pdev) 1955 { 1956 struct device *dev = &pdev->dev; 1957 struct stm32_cryp *cryp; 1958 struct resource *res; 1959 struct reset_control *rst; 1960 int irq, ret; 1961 1962 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); 1963 if (!cryp) 1964 return -ENOMEM; 1965 1966 cryp->caps = of_device_get_match_data(dev); 1967 if (!cryp->caps) 1968 return -ENODEV; 1969 1970 cryp->dev = dev; 1971 1972 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1973 cryp->regs = devm_ioremap_resource(dev, res); 1974 if (IS_ERR(cryp->regs)) 1975 return PTR_ERR(cryp->regs); 1976 1977 irq = platform_get_irq(pdev, 0); 1978 if (irq < 0) { 1979 dev_err(dev, "Cannot get IRQ resource\n"); 1980 return irq; 1981 } 1982 1983 ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq, 1984 stm32_cryp_irq_thread, IRQF_ONESHOT, 1985 dev_name(dev), cryp); 1986 if (ret) { 1987 dev_err(dev, "Cannot grab IRQ\n"); 1988 return ret; 1989 } 1990 1991 cryp->clk = devm_clk_get(dev, NULL); 1992 if (IS_ERR(cryp->clk)) { 1993 dev_err(dev, "Could not get clock\n"); 1994 return PTR_ERR(cryp->clk); 1995 } 1996 1997 ret = clk_prepare_enable(cryp->clk); 1998 if (ret) { 1999 dev_err(cryp->dev, "Failed to enable clock\n"); 2000 return ret; 2001 } 2002 2003 pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY); 2004 pm_runtime_use_autosuspend(dev); 2005 2006 pm_runtime_get_noresume(dev); 2007 pm_runtime_set_active(dev); 2008 pm_runtime_enable(dev); 2009 2010 rst = devm_reset_control_get(dev, NULL); 2011 if (!IS_ERR(rst)) { 2012 reset_control_assert(rst); 2013 udelay(2); 2014 reset_control_deassert(rst); 2015 } 2016 2017 platform_set_drvdata(pdev, cryp); 2018 2019 spin_lock(&cryp_list.lock); 2020 list_add(&cryp->list, &cryp_list.dev_list); 2021 spin_unlock(&cryp_list.lock); 2022 2023 /* Initialize crypto engine */ 2024 cryp->engine = crypto_engine_alloc_init(dev, 1); 2025 if (!cryp->engine) { 2026 dev_err(dev, "Could not init crypto engine\n"); 2027 ret = -ENOMEM; 2028 goto err_engine1; 2029 } 2030 2031 ret = crypto_engine_start(cryp->engine); 2032 if (ret) { 2033 dev_err(dev, "Could not start crypto engine\n"); 2034 goto err_engine2; 2035 } 2036 2037 ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs)); 2038 if (ret) { 2039 dev_err(dev, "Could not register algs\n"); 2040 goto err_algs; 2041 } 2042 2043 ret = crypto_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2044 if (ret) 2045 goto err_aead_algs; 2046 2047 dev_info(dev, "Initialized\n"); 2048 2049 pm_runtime_put_sync(dev); 2050 2051 return 0; 2052 2053 err_aead_algs: 2054 crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs)); 2055 err_algs: 2056 err_engine2: 2057 crypto_engine_exit(cryp->engine); 2058 err_engine1: 2059 spin_lock(&cryp_list.lock); 2060 list_del(&cryp->list); 2061 spin_unlock(&cryp_list.lock); 2062 2063 pm_runtime_disable(dev); 2064 pm_runtime_put_noidle(dev); 2065 pm_runtime_disable(dev); 2066 pm_runtime_put_noidle(dev); 2067 2068 clk_disable_unprepare(cryp->clk); 2069 2070 return ret; 2071 } 2072 2073 static int stm32_cryp_remove(struct platform_device *pdev) 2074 { 2075 struct stm32_cryp *cryp = platform_get_drvdata(pdev); 2076 int ret; 2077 2078 if (!cryp) 2079 return -ENODEV; 2080 2081 ret = pm_runtime_get_sync(cryp->dev); 2082 if (ret < 0) 2083 return ret; 2084 2085 crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); 2086 crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs)); 2087 2088 crypto_engine_exit(cryp->engine); 2089 2090 spin_lock(&cryp_list.lock); 2091 list_del(&cryp->list); 2092 spin_unlock(&cryp_list.lock); 2093 2094 pm_runtime_disable(cryp->dev); 2095 pm_runtime_put_noidle(cryp->dev); 2096 2097 clk_disable_unprepare(cryp->clk); 2098 2099 return 0; 2100 } 2101 2102 #ifdef CONFIG_PM 2103 static int stm32_cryp_runtime_suspend(struct device *dev) 2104 { 2105 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2106 2107 clk_disable_unprepare(cryp->clk); 2108 2109 return 0; 2110 } 2111 2112 static int stm32_cryp_runtime_resume(struct device *dev) 2113 { 2114 struct stm32_cryp *cryp = dev_get_drvdata(dev); 2115 int ret; 2116 2117 ret = clk_prepare_enable(cryp->clk); 2118 if (ret) { 2119 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); 2120 return ret; 2121 } 2122 2123 return 0; 2124 } 2125 #endif 2126 2127 static const struct dev_pm_ops stm32_cryp_pm_ops = { 2128 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 2129 pm_runtime_force_resume) 2130 SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend, 2131 stm32_cryp_runtime_resume, NULL) 2132 }; 2133 2134 static struct platform_driver stm32_cryp_driver = { 2135 .probe = stm32_cryp_probe, 2136 .remove = stm32_cryp_remove, 2137 .driver = { 2138 .name = DRIVER_NAME, 2139 .pm = &stm32_cryp_pm_ops, 2140 .of_match_table = stm32_dt_ids, 2141 }, 2142 }; 2143 2144 module_platform_driver(stm32_cryp_driver); 2145 2146 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); 2147 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver"); 2148 MODULE_LICENSE("GPL"); 2149