1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * caam - Freescale FSL CAAM support for ahash functions of crypto API
4 *
5 * Copyright 2011 Freescale Semiconductor, Inc.
6 * Copyright 2018-2019 NXP
7 *
8 * Based on caamalg.c crypto API driver.
9 *
10 * relationship of digest job descriptor or first job descriptor after init to
11 * shared descriptors:
12 *
13 * --------------- ---------------
14 * | JobDesc #1 |-------------------->| ShareDesc |
15 * | *(packet 1) | | (hashKey) |
16 * --------------- | (operation) |
17 * ---------------
18 *
19 * relationship of subsequent job descriptors to shared descriptors:
20 *
21 * --------------- ---------------
22 * | JobDesc #2 |-------------------->| ShareDesc |
23 * | *(packet 2) | |------------->| (hashKey) |
24 * --------------- | |-------->| (operation) |
25 * . | | | (load ctx2) |
26 * . | | ---------------
27 * --------------- | |
28 * | JobDesc #3 |------| |
29 * | *(packet 3) | |
30 * --------------- |
31 * . |
32 * . |
33 * --------------- |
34 * | JobDesc #4 |------------
35 * | *(packet 4) |
36 * ---------------
37 *
38 * The SharedDesc never changes for a connection unless rekeyed, but
39 * each packet will likely be in a different place. So all we need
40 * to know to process the packet is where the input is, where the
41 * output goes, and what context we want to process with. Context is
42 * in the SharedDesc, packet references in the JobDesc.
43 *
44 * So, a job desc looks like:
45 *
46 * ---------------------
47 * | Header |
48 * | ShareDesc Pointer |
49 * | SEQ_OUT_PTR |
50 * | (output buffer) |
51 * | (output length) |
52 * | SEQ_IN_PTR |
53 * | (input buffer) |
54 * | (input length) |
55 * ---------------------
56 */
57
58 #include "compat.h"
59
60 #include "regs.h"
61 #include "intern.h"
62 #include "desc_constr.h"
63 #include "jr.h"
64 #include "error.h"
65 #include "sg_sw_sec4.h"
66 #include "key_gen.h"
67 #include "caamhash_desc.h"
68 #include <crypto/engine.h>
69
70 #define CAAM_CRA_PRIORITY 3000
71
72 /* max hash key is max split key size */
73 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
74
75 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
76 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
77
78 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
79 CAAM_MAX_HASH_KEY_SIZE)
80 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
81
82 /* caam context sizes for hashes: running digest + 8 */
83 #define HASH_MSG_LEN 8
84 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
85
86 static struct list_head hash_list;
87
88 /* ahash per-session context */
89 struct caam_hash_ctx {
90 struct crypto_engine_ctx enginectx;
91 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
92 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
93 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
94 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
95 u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned;
96 dma_addr_t sh_desc_update_dma ____cacheline_aligned;
97 dma_addr_t sh_desc_update_first_dma;
98 dma_addr_t sh_desc_fin_dma;
99 dma_addr_t sh_desc_digest_dma;
100 enum dma_data_direction dir;
101 enum dma_data_direction key_dir;
102 struct device *jrdev;
103 int ctx_len;
104 struct alginfo adata;
105 };
106
107 /* ahash state */
108 struct caam_hash_state {
109 dma_addr_t buf_dma;
110 dma_addr_t ctx_dma;
111 int ctx_dma_len;
112 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
113 int buflen;
114 int next_buflen;
115 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
116 int (*update)(struct ahash_request *req) ____cacheline_aligned;
117 int (*final)(struct ahash_request *req);
118 int (*finup)(struct ahash_request *req);
119 struct ahash_edesc *edesc;
120 void (*ahash_op_done)(struct device *jrdev, u32 *desc, u32 err,
121 void *context);
122 };
123
124 struct caam_export_state {
125 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
126 u8 caam_ctx[MAX_CTX_LEN];
127 int buflen;
128 int (*update)(struct ahash_request *req);
129 int (*final)(struct ahash_request *req);
130 int (*finup)(struct ahash_request *req);
131 };
132
is_cmac_aes(u32 algtype)133 static inline bool is_cmac_aes(u32 algtype)
134 {
135 return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) ==
136 (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC);
137 }
138 /* Common job descriptor seq in/out ptr routines */
139
140 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
map_seq_out_ptr_ctx(u32 * desc,struct device * jrdev,struct caam_hash_state * state,int ctx_len)141 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
142 struct caam_hash_state *state,
143 int ctx_len)
144 {
145 state->ctx_dma_len = ctx_len;
146 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
147 ctx_len, DMA_FROM_DEVICE);
148 if (dma_mapping_error(jrdev, state->ctx_dma)) {
149 dev_err(jrdev, "unable to map ctx\n");
150 state->ctx_dma = 0;
151 return -ENOMEM;
152 }
153
154 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
155
156 return 0;
157 }
158
159 /* Map current buffer in state (if length > 0) and put it in link table */
buf_map_to_sec4_sg(struct device * jrdev,struct sec4_sg_entry * sec4_sg,struct caam_hash_state * state)160 static inline int buf_map_to_sec4_sg(struct device *jrdev,
161 struct sec4_sg_entry *sec4_sg,
162 struct caam_hash_state *state)
163 {
164 int buflen = state->buflen;
165
166 if (!buflen)
167 return 0;
168
169 state->buf_dma = dma_map_single(jrdev, state->buf, buflen,
170 DMA_TO_DEVICE);
171 if (dma_mapping_error(jrdev, state->buf_dma)) {
172 dev_err(jrdev, "unable to map buf\n");
173 state->buf_dma = 0;
174 return -ENOMEM;
175 }
176
177 dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
178
179 return 0;
180 }
181
182 /* Map state->caam_ctx, and add it to link table */
ctx_map_to_sec4_sg(struct device * jrdev,struct caam_hash_state * state,int ctx_len,struct sec4_sg_entry * sec4_sg,u32 flag)183 static inline int ctx_map_to_sec4_sg(struct device *jrdev,
184 struct caam_hash_state *state, int ctx_len,
185 struct sec4_sg_entry *sec4_sg, u32 flag)
186 {
187 state->ctx_dma_len = ctx_len;
188 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
189 if (dma_mapping_error(jrdev, state->ctx_dma)) {
190 dev_err(jrdev, "unable to map ctx\n");
191 state->ctx_dma = 0;
192 return -ENOMEM;
193 }
194
195 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
196
197 return 0;
198 }
199
ahash_set_sh_desc(struct crypto_ahash * ahash)200 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
201 {
202 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
203 int digestsize = crypto_ahash_digestsize(ahash);
204 struct device *jrdev = ctx->jrdev;
205 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
206 u32 *desc;
207
208 ctx->adata.key_virt = ctx->key;
209
210 /* ahash_update shared descriptor */
211 desc = ctx->sh_desc_update;
212 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
213 ctx->ctx_len, true, ctrlpriv->era);
214 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
215 desc_bytes(desc), ctx->dir);
216
217 print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
218 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
219 1);
220
221 /* ahash_update_first shared descriptor */
222 desc = ctx->sh_desc_update_first;
223 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
224 ctx->ctx_len, false, ctrlpriv->era);
225 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
226 desc_bytes(desc), ctx->dir);
227 print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
228 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
229 desc_bytes(desc), 1);
230
231 /* ahash_final shared descriptor */
232 desc = ctx->sh_desc_fin;
233 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
234 ctx->ctx_len, true, ctrlpriv->era);
235 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
236 desc_bytes(desc), ctx->dir);
237
238 print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
239 DUMP_PREFIX_ADDRESS, 16, 4, desc,
240 desc_bytes(desc), 1);
241
242 /* ahash_digest shared descriptor */
243 desc = ctx->sh_desc_digest;
244 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
245 ctx->ctx_len, false, ctrlpriv->era);
246 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
247 desc_bytes(desc), ctx->dir);
248
249 print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
250 DUMP_PREFIX_ADDRESS, 16, 4, desc,
251 desc_bytes(desc), 1);
252
253 return 0;
254 }
255
axcbc_set_sh_desc(struct crypto_ahash * ahash)256 static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
257 {
258 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
259 int digestsize = crypto_ahash_digestsize(ahash);
260 struct device *jrdev = ctx->jrdev;
261 u32 *desc;
262
263 /* shared descriptor for ahash_update */
264 desc = ctx->sh_desc_update;
265 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
266 ctx->ctx_len, ctx->ctx_len);
267 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
268 desc_bytes(desc), ctx->dir);
269 print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
270 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
271 1);
272
273 /* shared descriptor for ahash_{final,finup} */
274 desc = ctx->sh_desc_fin;
275 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
276 digestsize, ctx->ctx_len);
277 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
278 desc_bytes(desc), ctx->dir);
279 print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
280 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
281 1);
282
283 /* key is immediate data for INIT and INITFINAL states */
284 ctx->adata.key_virt = ctx->key;
285
286 /* shared descriptor for first invocation of ahash_update */
287 desc = ctx->sh_desc_update_first;
288 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
289 ctx->ctx_len);
290 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
291 desc_bytes(desc), ctx->dir);
292 print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
293 " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
294 desc_bytes(desc), 1);
295
296 /* shared descriptor for ahash_digest */
297 desc = ctx->sh_desc_digest;
298 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
299 digestsize, ctx->ctx_len);
300 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
301 desc_bytes(desc), ctx->dir);
302 print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
303 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
304 1);
305 return 0;
306 }
307
acmac_set_sh_desc(struct crypto_ahash * ahash)308 static int acmac_set_sh_desc(struct crypto_ahash *ahash)
309 {
310 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
311 int digestsize = crypto_ahash_digestsize(ahash);
312 struct device *jrdev = ctx->jrdev;
313 u32 *desc;
314
315 /* shared descriptor for ahash_update */
316 desc = ctx->sh_desc_update;
317 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
318 ctx->ctx_len, ctx->ctx_len);
319 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
320 desc_bytes(desc), ctx->dir);
321 print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
322 DUMP_PREFIX_ADDRESS, 16, 4, desc,
323 desc_bytes(desc), 1);
324
325 /* shared descriptor for ahash_{final,finup} */
326 desc = ctx->sh_desc_fin;
327 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
328 digestsize, ctx->ctx_len);
329 dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
330 desc_bytes(desc), ctx->dir);
331 print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
332 DUMP_PREFIX_ADDRESS, 16, 4, desc,
333 desc_bytes(desc), 1);
334
335 /* shared descriptor for first invocation of ahash_update */
336 desc = ctx->sh_desc_update_first;
337 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
338 ctx->ctx_len);
339 dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
340 desc_bytes(desc), ctx->dir);
341 print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
342 " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
343 desc_bytes(desc), 1);
344
345 /* shared descriptor for ahash_digest */
346 desc = ctx->sh_desc_digest;
347 cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
348 digestsize, ctx->ctx_len);
349 dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
350 desc_bytes(desc), ctx->dir);
351 print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
352 DUMP_PREFIX_ADDRESS, 16, 4, desc,
353 desc_bytes(desc), 1);
354
355 return 0;
356 }
357
358 /* Digest hash size if it is too large */
hash_digest_key(struct caam_hash_ctx * ctx,u32 * keylen,u8 * key,u32 digestsize)359 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
360 u32 digestsize)
361 {
362 struct device *jrdev = ctx->jrdev;
363 u32 *desc;
364 struct split_key_result result;
365 dma_addr_t key_dma;
366 int ret;
367
368 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
369 if (!desc) {
370 dev_err(jrdev, "unable to allocate key input memory\n");
371 return -ENOMEM;
372 }
373
374 init_job_desc(desc, 0);
375
376 key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL);
377 if (dma_mapping_error(jrdev, key_dma)) {
378 dev_err(jrdev, "unable to map key memory\n");
379 kfree(desc);
380 return -ENOMEM;
381 }
382
383 /* Job descriptor to perform unkeyed hash on key_in */
384 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
385 OP_ALG_AS_INITFINAL);
386 append_seq_in_ptr(desc, key_dma, *keylen, 0);
387 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
388 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
389 append_seq_out_ptr(desc, key_dma, digestsize, 0);
390 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
391 LDST_SRCDST_BYTE_CONTEXT);
392
393 print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
394 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
395 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
396 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
397 1);
398
399 result.err = 0;
400 init_completion(&result.completion);
401
402 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
403 if (ret == -EINPROGRESS) {
404 /* in progress */
405 wait_for_completion(&result.completion);
406 ret = result.err;
407
408 print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
409 DUMP_PREFIX_ADDRESS, 16, 4, key,
410 digestsize, 1);
411 }
412 dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
413
414 *keylen = digestsize;
415
416 kfree(desc);
417
418 return ret;
419 }
420
ahash_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)421 static int ahash_setkey(struct crypto_ahash *ahash,
422 const u8 *key, unsigned int keylen)
423 {
424 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
425 struct device *jrdev = ctx->jrdev;
426 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
427 int digestsize = crypto_ahash_digestsize(ahash);
428 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
429 int ret;
430 u8 *hashed_key = NULL;
431
432 dev_dbg(jrdev, "keylen %d\n", keylen);
433
434 if (keylen > blocksize) {
435 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
436 if (!hashed_key)
437 return -ENOMEM;
438 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
439 if (ret)
440 goto bad_free_key;
441 key = hashed_key;
442 }
443
444 /*
445 * If DKP is supported, use it in the shared descriptor to generate
446 * the split key.
447 */
448 if (ctrlpriv->era >= 6) {
449 ctx->adata.key_inline = true;
450 ctx->adata.keylen = keylen;
451 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
452 OP_ALG_ALGSEL_MASK);
453
454 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
455 goto bad_free_key;
456
457 memcpy(ctx->key, key, keylen);
458
459 /*
460 * In case |user key| > |derived key|, using DKP<imm,imm>
461 * would result in invalid opcodes (last bytes of user key) in
462 * the resulting descriptor. Use DKP<ptr,imm> instead => both
463 * virtual and dma key addresses are needed.
464 */
465 if (keylen > ctx->adata.keylen_pad)
466 dma_sync_single_for_device(ctx->jrdev,
467 ctx->adata.key_dma,
468 ctx->adata.keylen_pad,
469 DMA_TO_DEVICE);
470 } else {
471 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
472 keylen, CAAM_MAX_HASH_KEY_SIZE);
473 if (ret)
474 goto bad_free_key;
475 }
476
477 kfree(hashed_key);
478 return ahash_set_sh_desc(ahash);
479 bad_free_key:
480 kfree(hashed_key);
481 return -EINVAL;
482 }
483
axcbc_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)484 static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
485 unsigned int keylen)
486 {
487 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
488 struct device *jrdev = ctx->jrdev;
489
490 if (keylen != AES_KEYSIZE_128)
491 return -EINVAL;
492
493 memcpy(ctx->key, key, keylen);
494 dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,
495 DMA_TO_DEVICE);
496 ctx->adata.keylen = keylen;
497
498 print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
499 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1);
500
501 return axcbc_set_sh_desc(ahash);
502 }
503
acmac_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)504 static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
505 unsigned int keylen)
506 {
507 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
508 int err;
509
510 err = aes_check_keylen(keylen);
511 if (err)
512 return err;
513
514 /* key is immediate data for all cmac shared descriptors */
515 ctx->adata.key_virt = key;
516 ctx->adata.keylen = keylen;
517
518 print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ",
519 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
520
521 return acmac_set_sh_desc(ahash);
522 }
523
524 /*
525 * ahash_edesc - s/w-extended ahash descriptor
526 * @sec4_sg_dma: physical mapped address of h/w link table
527 * @src_nents: number of segments in input scatterlist
528 * @sec4_sg_bytes: length of dma mapped sec4_sg space
529 * @bklog: stored to determine if the request needs backlog
530 * @hw_desc: the h/w job descriptor followed by any referenced link tables
531 * @sec4_sg: h/w link table
532 */
533 struct ahash_edesc {
534 dma_addr_t sec4_sg_dma;
535 int src_nents;
536 int sec4_sg_bytes;
537 bool bklog;
538 u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned;
539 struct sec4_sg_entry sec4_sg[];
540 };
541
ahash_unmap(struct device * dev,struct ahash_edesc * edesc,struct ahash_request * req,int dst_len)542 static inline void ahash_unmap(struct device *dev,
543 struct ahash_edesc *edesc,
544 struct ahash_request *req, int dst_len)
545 {
546 struct caam_hash_state *state = ahash_request_ctx(req);
547
548 if (edesc->src_nents)
549 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
550
551 if (edesc->sec4_sg_bytes)
552 dma_unmap_single(dev, edesc->sec4_sg_dma,
553 edesc->sec4_sg_bytes, DMA_TO_DEVICE);
554
555 if (state->buf_dma) {
556 dma_unmap_single(dev, state->buf_dma, state->buflen,
557 DMA_TO_DEVICE);
558 state->buf_dma = 0;
559 }
560 }
561
ahash_unmap_ctx(struct device * dev,struct ahash_edesc * edesc,struct ahash_request * req,int dst_len,u32 flag)562 static inline void ahash_unmap_ctx(struct device *dev,
563 struct ahash_edesc *edesc,
564 struct ahash_request *req, int dst_len, u32 flag)
565 {
566 struct caam_hash_state *state = ahash_request_ctx(req);
567
568 if (state->ctx_dma) {
569 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
570 state->ctx_dma = 0;
571 }
572 ahash_unmap(dev, edesc, req, dst_len);
573 }
574
ahash_done_cpy(struct device * jrdev,u32 * desc,u32 err,void * context,enum dma_data_direction dir)575 static inline void ahash_done_cpy(struct device *jrdev, u32 *desc, u32 err,
576 void *context, enum dma_data_direction dir)
577 {
578 struct ahash_request *req = context;
579 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
580 struct ahash_edesc *edesc;
581 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
582 int digestsize = crypto_ahash_digestsize(ahash);
583 struct caam_hash_state *state = ahash_request_ctx(req);
584 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
585 int ecode = 0;
586 bool has_bklog;
587
588 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
589
590 edesc = state->edesc;
591 has_bklog = edesc->bklog;
592
593 if (err)
594 ecode = caam_jr_strstatus(jrdev, err);
595
596 ahash_unmap_ctx(jrdev, edesc, req, digestsize, dir);
597 memcpy(req->result, state->caam_ctx, digestsize);
598 kfree(edesc);
599
600 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
601 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
602 ctx->ctx_len, 1);
603
604 /*
605 * If no backlog flag, the completion of the request is done
606 * by CAAM, not crypto engine.
607 */
608 if (!has_bklog)
609 req->base.complete(&req->base, ecode);
610 else
611 crypto_finalize_hash_request(jrp->engine, req, ecode);
612 }
613
ahash_done(struct device * jrdev,u32 * desc,u32 err,void * context)614 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
615 void *context)
616 {
617 ahash_done_cpy(jrdev, desc, err, context, DMA_FROM_DEVICE);
618 }
619
ahash_done_ctx_src(struct device * jrdev,u32 * desc,u32 err,void * context)620 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
621 void *context)
622 {
623 ahash_done_cpy(jrdev, desc, err, context, DMA_BIDIRECTIONAL);
624 }
625
ahash_done_switch(struct device * jrdev,u32 * desc,u32 err,void * context,enum dma_data_direction dir)626 static inline void ahash_done_switch(struct device *jrdev, u32 *desc, u32 err,
627 void *context, enum dma_data_direction dir)
628 {
629 struct ahash_request *req = context;
630 struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
631 struct ahash_edesc *edesc;
632 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
633 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
634 struct caam_hash_state *state = ahash_request_ctx(req);
635 int digestsize = crypto_ahash_digestsize(ahash);
636 int ecode = 0;
637 bool has_bklog;
638
639 dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
640
641 edesc = state->edesc;
642 has_bklog = edesc->bklog;
643 if (err)
644 ecode = caam_jr_strstatus(jrdev, err);
645
646 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, dir);
647 kfree(edesc);
648
649 scatterwalk_map_and_copy(state->buf, req->src,
650 req->nbytes - state->next_buflen,
651 state->next_buflen, 0);
652 state->buflen = state->next_buflen;
653
654 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
655 DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
656 state->buflen, 1);
657
658 print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
659 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
660 ctx->ctx_len, 1);
661 if (req->result)
662 print_hex_dump_debug("result@"__stringify(__LINE__)": ",
663 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
664 digestsize, 1);
665
666 /*
667 * If no backlog flag, the completion of the request is done
668 * by CAAM, not crypto engine.
669 */
670 if (!has_bklog)
671 req->base.complete(&req->base, ecode);
672 else
673 crypto_finalize_hash_request(jrp->engine, req, ecode);
674
675 }
676
ahash_done_bi(struct device * jrdev,u32 * desc,u32 err,void * context)677 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
678 void *context)
679 {
680 ahash_done_switch(jrdev, desc, err, context, DMA_BIDIRECTIONAL);
681 }
682
ahash_done_ctx_dst(struct device * jrdev,u32 * desc,u32 err,void * context)683 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
684 void *context)
685 {
686 ahash_done_switch(jrdev, desc, err, context, DMA_FROM_DEVICE);
687 }
688
689 /*
690 * Allocate an enhanced descriptor, which contains the hardware descriptor
691 * and space for hardware scatter table containing sg_num entries.
692 */
ahash_edesc_alloc(struct ahash_request * req,int sg_num,u32 * sh_desc,dma_addr_t sh_desc_dma)693 static struct ahash_edesc *ahash_edesc_alloc(struct ahash_request *req,
694 int sg_num, u32 *sh_desc,
695 dma_addr_t sh_desc_dma)
696 {
697 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
698 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
699 struct caam_hash_state *state = ahash_request_ctx(req);
700 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
701 GFP_KERNEL : GFP_ATOMIC;
702 struct ahash_edesc *edesc;
703 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
704
705 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
706 if (!edesc) {
707 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
708 return NULL;
709 }
710
711 state->edesc = edesc;
712
713 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
714 HDR_SHARE_DEFER | HDR_REVERSE);
715
716 return edesc;
717 }
718
ahash_edesc_add_src(struct caam_hash_ctx * ctx,struct ahash_edesc * edesc,struct ahash_request * req,int nents,unsigned int first_sg,unsigned int first_bytes,size_t to_hash)719 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
720 struct ahash_edesc *edesc,
721 struct ahash_request *req, int nents,
722 unsigned int first_sg,
723 unsigned int first_bytes, size_t to_hash)
724 {
725 dma_addr_t src_dma;
726 u32 options;
727
728 if (nents > 1 || first_sg) {
729 struct sec4_sg_entry *sg = edesc->sec4_sg;
730 unsigned int sgsize = sizeof(*sg) *
731 pad_sg_nents(first_sg + nents);
732
733 sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
734
735 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
736 if (dma_mapping_error(ctx->jrdev, src_dma)) {
737 dev_err(ctx->jrdev, "unable to map S/G table\n");
738 return -ENOMEM;
739 }
740
741 edesc->sec4_sg_bytes = sgsize;
742 edesc->sec4_sg_dma = src_dma;
743 options = LDST_SGF;
744 } else {
745 src_dma = sg_dma_address(req->src);
746 options = 0;
747 }
748
749 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
750 options);
751
752 return 0;
753 }
754
ahash_do_one_req(struct crypto_engine * engine,void * areq)755 static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
756 {
757 struct ahash_request *req = ahash_request_cast(areq);
758 struct caam_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
759 struct caam_hash_state *state = ahash_request_ctx(req);
760 struct device *jrdev = ctx->jrdev;
761 u32 *desc = state->edesc->hw_desc;
762 int ret;
763
764 state->edesc->bklog = true;
765
766 ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
767
768 if (ret != -EINPROGRESS) {
769 ahash_unmap(jrdev, state->edesc, req, 0);
770 kfree(state->edesc);
771 } else {
772 ret = 0;
773 }
774
775 return ret;
776 }
777
ahash_enqueue_req(struct device * jrdev,void (* cbk)(struct device * jrdev,u32 * desc,u32 err,void * context),struct ahash_request * req,int dst_len,enum dma_data_direction dir)778 static int ahash_enqueue_req(struct device *jrdev,
779 void (*cbk)(struct device *jrdev, u32 *desc,
780 u32 err, void *context),
781 struct ahash_request *req,
782 int dst_len, enum dma_data_direction dir)
783 {
784 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
785 struct caam_hash_state *state = ahash_request_ctx(req);
786 struct ahash_edesc *edesc = state->edesc;
787 u32 *desc = edesc->hw_desc;
788 int ret;
789
790 state->ahash_op_done = cbk;
791
792 /*
793 * Only the backlog request are sent to crypto-engine since the others
794 * can be handled by CAAM, if free, especially since JR has up to 1024
795 * entries (more than the 10 entries from crypto-engine).
796 */
797 if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
798 ret = crypto_transfer_hash_request_to_engine(jrpriv->engine,
799 req);
800 else
801 ret = caam_jr_enqueue(jrdev, desc, cbk, req);
802
803 if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
804 ahash_unmap_ctx(jrdev, edesc, req, dst_len, dir);
805 kfree(edesc);
806 }
807
808 return ret;
809 }
810
811 /* submit update job descriptor */
ahash_update_ctx(struct ahash_request * req)812 static int ahash_update_ctx(struct ahash_request *req)
813 {
814 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
815 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
816 struct caam_hash_state *state = ahash_request_ctx(req);
817 struct device *jrdev = ctx->jrdev;
818 u8 *buf = state->buf;
819 int *buflen = &state->buflen;
820 int *next_buflen = &state->next_buflen;
821 int blocksize = crypto_ahash_blocksize(ahash);
822 int in_len = *buflen + req->nbytes, to_hash;
823 u32 *desc;
824 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
825 struct ahash_edesc *edesc;
826 int ret = 0;
827
828 *next_buflen = in_len & (blocksize - 1);
829 to_hash = in_len - *next_buflen;
830
831 /*
832 * For XCBC and CMAC, if to_hash is multiple of block size,
833 * keep last block in internal buffer
834 */
835 if ((is_xcbc_aes(ctx->adata.algtype) ||
836 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
837 (*next_buflen == 0)) {
838 *next_buflen = blocksize;
839 to_hash -= blocksize;
840 }
841
842 if (to_hash) {
843 int pad_nents;
844 int src_len = req->nbytes - *next_buflen;
845
846 src_nents = sg_nents_for_len(req->src, src_len);
847 if (src_nents < 0) {
848 dev_err(jrdev, "Invalid number of src SG.\n");
849 return src_nents;
850 }
851
852 if (src_nents) {
853 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
854 DMA_TO_DEVICE);
855 if (!mapped_nents) {
856 dev_err(jrdev, "unable to DMA map source\n");
857 return -ENOMEM;
858 }
859 } else {
860 mapped_nents = 0;
861 }
862
863 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
864 pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
865 sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
866
867 /*
868 * allocate space for base edesc and hw desc commands,
869 * link tables
870 */
871 edesc = ahash_edesc_alloc(req, pad_nents, ctx->sh_desc_update,
872 ctx->sh_desc_update_dma);
873 if (!edesc) {
874 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
875 return -ENOMEM;
876 }
877
878 edesc->src_nents = src_nents;
879 edesc->sec4_sg_bytes = sec4_sg_bytes;
880
881 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
882 edesc->sec4_sg, DMA_BIDIRECTIONAL);
883 if (ret)
884 goto unmap_ctx;
885
886 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
887 if (ret)
888 goto unmap_ctx;
889
890 if (mapped_nents)
891 sg_to_sec4_sg_last(req->src, src_len,
892 edesc->sec4_sg + sec4_sg_src_index,
893 0);
894 else
895 sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
896 1);
897
898 desc = edesc->hw_desc;
899
900 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
901 sec4_sg_bytes,
902 DMA_TO_DEVICE);
903 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
904 dev_err(jrdev, "unable to map S/G table\n");
905 ret = -ENOMEM;
906 goto unmap_ctx;
907 }
908
909 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
910 to_hash, LDST_SGF);
911
912 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
913
914 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
915 DUMP_PREFIX_ADDRESS, 16, 4, desc,
916 desc_bytes(desc), 1);
917
918 ret = ahash_enqueue_req(jrdev, ahash_done_bi, req,
919 ctx->ctx_len, DMA_BIDIRECTIONAL);
920 } else if (*next_buflen) {
921 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
922 req->nbytes, 0);
923 *buflen = *next_buflen;
924
925 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
926 DUMP_PREFIX_ADDRESS, 16, 4, buf,
927 *buflen, 1);
928 }
929
930 return ret;
931 unmap_ctx:
932 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
933 kfree(edesc);
934 return ret;
935 }
936
ahash_final_ctx(struct ahash_request * req)937 static int ahash_final_ctx(struct ahash_request *req)
938 {
939 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
940 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
941 struct caam_hash_state *state = ahash_request_ctx(req);
942 struct device *jrdev = ctx->jrdev;
943 int buflen = state->buflen;
944 u32 *desc;
945 int sec4_sg_bytes;
946 int digestsize = crypto_ahash_digestsize(ahash);
947 struct ahash_edesc *edesc;
948 int ret;
949
950 sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
951 sizeof(struct sec4_sg_entry);
952
953 /* allocate space for base edesc and hw desc commands, link tables */
954 edesc = ahash_edesc_alloc(req, 4, ctx->sh_desc_fin,
955 ctx->sh_desc_fin_dma);
956 if (!edesc)
957 return -ENOMEM;
958
959 desc = edesc->hw_desc;
960
961 edesc->sec4_sg_bytes = sec4_sg_bytes;
962
963 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
964 edesc->sec4_sg, DMA_BIDIRECTIONAL);
965 if (ret)
966 goto unmap_ctx;
967
968 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
969 if (ret)
970 goto unmap_ctx;
971
972 sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
973
974 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
975 sec4_sg_bytes, DMA_TO_DEVICE);
976 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
977 dev_err(jrdev, "unable to map S/G table\n");
978 ret = -ENOMEM;
979 goto unmap_ctx;
980 }
981
982 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
983 LDST_SGF);
984 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
985
986 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
987 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
988 1);
989
990 return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,
991 digestsize, DMA_BIDIRECTIONAL);
992 unmap_ctx:
993 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
994 kfree(edesc);
995 return ret;
996 }
997
ahash_finup_ctx(struct ahash_request * req)998 static int ahash_finup_ctx(struct ahash_request *req)
999 {
1000 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1001 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1002 struct caam_hash_state *state = ahash_request_ctx(req);
1003 struct device *jrdev = ctx->jrdev;
1004 int buflen = state->buflen;
1005 u32 *desc;
1006 int sec4_sg_src_index;
1007 int src_nents, mapped_nents;
1008 int digestsize = crypto_ahash_digestsize(ahash);
1009 struct ahash_edesc *edesc;
1010 int ret;
1011
1012 src_nents = sg_nents_for_len(req->src, req->nbytes);
1013 if (src_nents < 0) {
1014 dev_err(jrdev, "Invalid number of src SG.\n");
1015 return src_nents;
1016 }
1017
1018 if (src_nents) {
1019 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1020 DMA_TO_DEVICE);
1021 if (!mapped_nents) {
1022 dev_err(jrdev, "unable to DMA map source\n");
1023 return -ENOMEM;
1024 }
1025 } else {
1026 mapped_nents = 0;
1027 }
1028
1029 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
1030
1031 /* allocate space for base edesc and hw desc commands, link tables */
1032 edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents,
1033 ctx->sh_desc_fin, ctx->sh_desc_fin_dma);
1034 if (!edesc) {
1035 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1036 return -ENOMEM;
1037 }
1038
1039 desc = edesc->hw_desc;
1040
1041 edesc->src_nents = src_nents;
1042
1043 ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
1044 edesc->sec4_sg, DMA_BIDIRECTIONAL);
1045 if (ret)
1046 goto unmap_ctx;
1047
1048 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
1049 if (ret)
1050 goto unmap_ctx;
1051
1052 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
1053 sec4_sg_src_index, ctx->ctx_len + buflen,
1054 req->nbytes);
1055 if (ret)
1056 goto unmap_ctx;
1057
1058 append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
1059
1060 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1061 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1062 1);
1063
1064 return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,
1065 digestsize, DMA_BIDIRECTIONAL);
1066 unmap_ctx:
1067 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
1068 kfree(edesc);
1069 return ret;
1070 }
1071
ahash_digest(struct ahash_request * req)1072 static int ahash_digest(struct ahash_request *req)
1073 {
1074 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1075 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1076 struct caam_hash_state *state = ahash_request_ctx(req);
1077 struct device *jrdev = ctx->jrdev;
1078 u32 *desc;
1079 int digestsize = crypto_ahash_digestsize(ahash);
1080 int src_nents, mapped_nents;
1081 struct ahash_edesc *edesc;
1082 int ret;
1083
1084 state->buf_dma = 0;
1085
1086 src_nents = sg_nents_for_len(req->src, req->nbytes);
1087 if (src_nents < 0) {
1088 dev_err(jrdev, "Invalid number of src SG.\n");
1089 return src_nents;
1090 }
1091
1092 if (src_nents) {
1093 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1094 DMA_TO_DEVICE);
1095 if (!mapped_nents) {
1096 dev_err(jrdev, "unable to map source for DMA\n");
1097 return -ENOMEM;
1098 }
1099 } else {
1100 mapped_nents = 0;
1101 }
1102
1103 /* allocate space for base edesc and hw desc commands, link tables */
1104 edesc = ahash_edesc_alloc(req, mapped_nents > 1 ? mapped_nents : 0,
1105 ctx->sh_desc_digest, ctx->sh_desc_digest_dma);
1106 if (!edesc) {
1107 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1108 return -ENOMEM;
1109 }
1110
1111 edesc->src_nents = src_nents;
1112
1113 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1114 req->nbytes);
1115 if (ret) {
1116 ahash_unmap(jrdev, edesc, req, digestsize);
1117 kfree(edesc);
1118 return ret;
1119 }
1120
1121 desc = edesc->hw_desc;
1122
1123 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1124 if (ret) {
1125 ahash_unmap(jrdev, edesc, req, digestsize);
1126 kfree(edesc);
1127 return -ENOMEM;
1128 }
1129
1130 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1131 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132 1);
1133
1134 return ahash_enqueue_req(jrdev, ahash_done, req, digestsize,
1135 DMA_FROM_DEVICE);
1136 }
1137
1138 /* submit ahash final if it the first job descriptor */
ahash_final_no_ctx(struct ahash_request * req)1139 static int ahash_final_no_ctx(struct ahash_request *req)
1140 {
1141 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1142 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1143 struct caam_hash_state *state = ahash_request_ctx(req);
1144 struct device *jrdev = ctx->jrdev;
1145 u8 *buf = state->buf;
1146 int buflen = state->buflen;
1147 u32 *desc;
1148 int digestsize = crypto_ahash_digestsize(ahash);
1149 struct ahash_edesc *edesc;
1150 int ret;
1151
1152 /* allocate space for base edesc and hw desc commands, link tables */
1153 edesc = ahash_edesc_alloc(req, 0, ctx->sh_desc_digest,
1154 ctx->sh_desc_digest_dma);
1155 if (!edesc)
1156 return -ENOMEM;
1157
1158 desc = edesc->hw_desc;
1159
1160 if (buflen) {
1161 state->buf_dma = dma_map_single(jrdev, buf, buflen,
1162 DMA_TO_DEVICE);
1163 if (dma_mapping_error(jrdev, state->buf_dma)) {
1164 dev_err(jrdev, "unable to map src\n");
1165 goto unmap;
1166 }
1167
1168 append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1169 }
1170
1171 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1172 if (ret)
1173 goto unmap;
1174
1175 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1176 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1177 1);
1178
1179 return ahash_enqueue_req(jrdev, ahash_done, req,
1180 digestsize, DMA_FROM_DEVICE);
1181 unmap:
1182 ahash_unmap(jrdev, edesc, req, digestsize);
1183 kfree(edesc);
1184 return -ENOMEM;
1185 }
1186
1187 /* submit ahash update if it the first job descriptor after update */
ahash_update_no_ctx(struct ahash_request * req)1188 static int ahash_update_no_ctx(struct ahash_request *req)
1189 {
1190 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1191 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1192 struct caam_hash_state *state = ahash_request_ctx(req);
1193 struct device *jrdev = ctx->jrdev;
1194 u8 *buf = state->buf;
1195 int *buflen = &state->buflen;
1196 int *next_buflen = &state->next_buflen;
1197 int blocksize = crypto_ahash_blocksize(ahash);
1198 int in_len = *buflen + req->nbytes, to_hash;
1199 int sec4_sg_bytes, src_nents, mapped_nents;
1200 struct ahash_edesc *edesc;
1201 u32 *desc;
1202 int ret = 0;
1203
1204 *next_buflen = in_len & (blocksize - 1);
1205 to_hash = in_len - *next_buflen;
1206
1207 /*
1208 * For XCBC and CMAC, if to_hash is multiple of block size,
1209 * keep last block in internal buffer
1210 */
1211 if ((is_xcbc_aes(ctx->adata.algtype) ||
1212 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
1213 (*next_buflen == 0)) {
1214 *next_buflen = blocksize;
1215 to_hash -= blocksize;
1216 }
1217
1218 if (to_hash) {
1219 int pad_nents;
1220 int src_len = req->nbytes - *next_buflen;
1221
1222 src_nents = sg_nents_for_len(req->src, src_len);
1223 if (src_nents < 0) {
1224 dev_err(jrdev, "Invalid number of src SG.\n");
1225 return src_nents;
1226 }
1227
1228 if (src_nents) {
1229 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1230 DMA_TO_DEVICE);
1231 if (!mapped_nents) {
1232 dev_err(jrdev, "unable to DMA map source\n");
1233 return -ENOMEM;
1234 }
1235 } else {
1236 mapped_nents = 0;
1237 }
1238
1239 pad_nents = pad_sg_nents(1 + mapped_nents);
1240 sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
1241
1242 /*
1243 * allocate space for base edesc and hw desc commands,
1244 * link tables
1245 */
1246 edesc = ahash_edesc_alloc(req, pad_nents,
1247 ctx->sh_desc_update_first,
1248 ctx->sh_desc_update_first_dma);
1249 if (!edesc) {
1250 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1251 return -ENOMEM;
1252 }
1253
1254 edesc->src_nents = src_nents;
1255 edesc->sec4_sg_bytes = sec4_sg_bytes;
1256
1257 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1258 if (ret)
1259 goto unmap_ctx;
1260
1261 sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
1262
1263 desc = edesc->hw_desc;
1264
1265 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1266 sec4_sg_bytes,
1267 DMA_TO_DEVICE);
1268 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1269 dev_err(jrdev, "unable to map S/G table\n");
1270 ret = -ENOMEM;
1271 goto unmap_ctx;
1272 }
1273
1274 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1275
1276 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1277 if (ret)
1278 goto unmap_ctx;
1279
1280 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1281 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1282 desc_bytes(desc), 1);
1283
1284 ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,
1285 ctx->ctx_len, DMA_TO_DEVICE);
1286 if ((ret != -EINPROGRESS) && (ret != -EBUSY))
1287 return ret;
1288 state->update = ahash_update_ctx;
1289 state->finup = ahash_finup_ctx;
1290 state->final = ahash_final_ctx;
1291 } else if (*next_buflen) {
1292 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1293 req->nbytes, 0);
1294 *buflen = *next_buflen;
1295
1296 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
1297 DUMP_PREFIX_ADDRESS, 16, 4, buf,
1298 *buflen, 1);
1299 }
1300
1301 return ret;
1302 unmap_ctx:
1303 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1304 kfree(edesc);
1305 return ret;
1306 }
1307
1308 /* submit ahash finup if it the first job descriptor after update */
ahash_finup_no_ctx(struct ahash_request * req)1309 static int ahash_finup_no_ctx(struct ahash_request *req)
1310 {
1311 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1312 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1313 struct caam_hash_state *state = ahash_request_ctx(req);
1314 struct device *jrdev = ctx->jrdev;
1315 int buflen = state->buflen;
1316 u32 *desc;
1317 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1318 int digestsize = crypto_ahash_digestsize(ahash);
1319 struct ahash_edesc *edesc;
1320 int ret;
1321
1322 src_nents = sg_nents_for_len(req->src, req->nbytes);
1323 if (src_nents < 0) {
1324 dev_err(jrdev, "Invalid number of src SG.\n");
1325 return src_nents;
1326 }
1327
1328 if (src_nents) {
1329 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1330 DMA_TO_DEVICE);
1331 if (!mapped_nents) {
1332 dev_err(jrdev, "unable to DMA map source\n");
1333 return -ENOMEM;
1334 }
1335 } else {
1336 mapped_nents = 0;
1337 }
1338
1339 sec4_sg_src_index = 2;
1340 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1341 sizeof(struct sec4_sg_entry);
1342
1343 /* allocate space for base edesc and hw desc commands, link tables */
1344 edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents,
1345 ctx->sh_desc_digest, ctx->sh_desc_digest_dma);
1346 if (!edesc) {
1347 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1348 return -ENOMEM;
1349 }
1350
1351 desc = edesc->hw_desc;
1352
1353 edesc->src_nents = src_nents;
1354 edesc->sec4_sg_bytes = sec4_sg_bytes;
1355
1356 ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
1357 if (ret)
1358 goto unmap;
1359
1360 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1361 req->nbytes);
1362 if (ret) {
1363 dev_err(jrdev, "unable to map S/G table\n");
1364 goto unmap;
1365 }
1366
1367 ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
1368 if (ret)
1369 goto unmap;
1370
1371 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1372 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1373 1);
1374
1375 return ahash_enqueue_req(jrdev, ahash_done, req,
1376 digestsize, DMA_FROM_DEVICE);
1377 unmap:
1378 ahash_unmap(jrdev, edesc, req, digestsize);
1379 kfree(edesc);
1380 return -ENOMEM;
1381
1382 }
1383
1384 /* submit first update job descriptor after init */
ahash_update_first(struct ahash_request * req)1385 static int ahash_update_first(struct ahash_request *req)
1386 {
1387 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1388 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1389 struct caam_hash_state *state = ahash_request_ctx(req);
1390 struct device *jrdev = ctx->jrdev;
1391 u8 *buf = state->buf;
1392 int *buflen = &state->buflen;
1393 int *next_buflen = &state->next_buflen;
1394 int to_hash;
1395 int blocksize = crypto_ahash_blocksize(ahash);
1396 u32 *desc;
1397 int src_nents, mapped_nents;
1398 struct ahash_edesc *edesc;
1399 int ret = 0;
1400
1401 *next_buflen = req->nbytes & (blocksize - 1);
1402 to_hash = req->nbytes - *next_buflen;
1403
1404 /*
1405 * For XCBC and CMAC, if to_hash is multiple of block size,
1406 * keep last block in internal buffer
1407 */
1408 if ((is_xcbc_aes(ctx->adata.algtype) ||
1409 is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
1410 (*next_buflen == 0)) {
1411 *next_buflen = blocksize;
1412 to_hash -= blocksize;
1413 }
1414
1415 if (to_hash) {
1416 src_nents = sg_nents_for_len(req->src,
1417 req->nbytes - *next_buflen);
1418 if (src_nents < 0) {
1419 dev_err(jrdev, "Invalid number of src SG.\n");
1420 return src_nents;
1421 }
1422
1423 if (src_nents) {
1424 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1425 DMA_TO_DEVICE);
1426 if (!mapped_nents) {
1427 dev_err(jrdev, "unable to map source for DMA\n");
1428 return -ENOMEM;
1429 }
1430 } else {
1431 mapped_nents = 0;
1432 }
1433
1434 /*
1435 * allocate space for base edesc and hw desc commands,
1436 * link tables
1437 */
1438 edesc = ahash_edesc_alloc(req, mapped_nents > 1 ?
1439 mapped_nents : 0,
1440 ctx->sh_desc_update_first,
1441 ctx->sh_desc_update_first_dma);
1442 if (!edesc) {
1443 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1444 return -ENOMEM;
1445 }
1446
1447 edesc->src_nents = src_nents;
1448
1449 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1450 to_hash);
1451 if (ret)
1452 goto unmap_ctx;
1453
1454 desc = edesc->hw_desc;
1455
1456 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1457 if (ret)
1458 goto unmap_ctx;
1459
1460 print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
1461 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1462 desc_bytes(desc), 1);
1463
1464 ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,
1465 ctx->ctx_len, DMA_TO_DEVICE);
1466 if ((ret != -EINPROGRESS) && (ret != -EBUSY))
1467 return ret;
1468 state->update = ahash_update_ctx;
1469 state->finup = ahash_finup_ctx;
1470 state->final = ahash_final_ctx;
1471 } else if (*next_buflen) {
1472 state->update = ahash_update_no_ctx;
1473 state->finup = ahash_finup_no_ctx;
1474 state->final = ahash_final_no_ctx;
1475 scatterwalk_map_and_copy(buf, req->src, 0,
1476 req->nbytes, 0);
1477 *buflen = *next_buflen;
1478
1479 print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
1480 DUMP_PREFIX_ADDRESS, 16, 4, buf,
1481 *buflen, 1);
1482 }
1483
1484 return ret;
1485 unmap_ctx:
1486 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1487 kfree(edesc);
1488 return ret;
1489 }
1490
ahash_finup_first(struct ahash_request * req)1491 static int ahash_finup_first(struct ahash_request *req)
1492 {
1493 return ahash_digest(req);
1494 }
1495
ahash_init(struct ahash_request * req)1496 static int ahash_init(struct ahash_request *req)
1497 {
1498 struct caam_hash_state *state = ahash_request_ctx(req);
1499
1500 state->update = ahash_update_first;
1501 state->finup = ahash_finup_first;
1502 state->final = ahash_final_no_ctx;
1503
1504 state->ctx_dma = 0;
1505 state->ctx_dma_len = 0;
1506 state->buf_dma = 0;
1507 state->buflen = 0;
1508 state->next_buflen = 0;
1509
1510 return 0;
1511 }
1512
ahash_update(struct ahash_request * req)1513 static int ahash_update(struct ahash_request *req)
1514 {
1515 struct caam_hash_state *state = ahash_request_ctx(req);
1516
1517 return state->update(req);
1518 }
1519
ahash_finup(struct ahash_request * req)1520 static int ahash_finup(struct ahash_request *req)
1521 {
1522 struct caam_hash_state *state = ahash_request_ctx(req);
1523
1524 return state->finup(req);
1525 }
1526
ahash_final(struct ahash_request * req)1527 static int ahash_final(struct ahash_request *req)
1528 {
1529 struct caam_hash_state *state = ahash_request_ctx(req);
1530
1531 return state->final(req);
1532 }
1533
ahash_export(struct ahash_request * req,void * out)1534 static int ahash_export(struct ahash_request *req, void *out)
1535 {
1536 struct caam_hash_state *state = ahash_request_ctx(req);
1537 struct caam_export_state *export = out;
1538 u8 *buf = state->buf;
1539 int len = state->buflen;
1540
1541 memcpy(export->buf, buf, len);
1542 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1543 export->buflen = len;
1544 export->update = state->update;
1545 export->final = state->final;
1546 export->finup = state->finup;
1547
1548 return 0;
1549 }
1550
ahash_import(struct ahash_request * req,const void * in)1551 static int ahash_import(struct ahash_request *req, const void *in)
1552 {
1553 struct caam_hash_state *state = ahash_request_ctx(req);
1554 const struct caam_export_state *export = in;
1555
1556 memset(state, 0, sizeof(*state));
1557 memcpy(state->buf, export->buf, export->buflen);
1558 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1559 state->buflen = export->buflen;
1560 state->update = export->update;
1561 state->final = export->final;
1562 state->finup = export->finup;
1563
1564 return 0;
1565 }
1566
1567 struct caam_hash_template {
1568 char name[CRYPTO_MAX_ALG_NAME];
1569 char driver_name[CRYPTO_MAX_ALG_NAME];
1570 char hmac_name[CRYPTO_MAX_ALG_NAME];
1571 char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1572 unsigned int blocksize;
1573 struct ahash_alg template_ahash;
1574 u32 alg_type;
1575 };
1576
1577 /* ahash descriptors */
1578 static struct caam_hash_template driver_hash[] = {
1579 {
1580 .name = "sha1",
1581 .driver_name = "sha1-caam",
1582 .hmac_name = "hmac(sha1)",
1583 .hmac_driver_name = "hmac-sha1-caam",
1584 .blocksize = SHA1_BLOCK_SIZE,
1585 .template_ahash = {
1586 .init = ahash_init,
1587 .update = ahash_update,
1588 .final = ahash_final,
1589 .finup = ahash_finup,
1590 .digest = ahash_digest,
1591 .export = ahash_export,
1592 .import = ahash_import,
1593 .setkey = ahash_setkey,
1594 .halg = {
1595 .digestsize = SHA1_DIGEST_SIZE,
1596 .statesize = sizeof(struct caam_export_state),
1597 },
1598 },
1599 .alg_type = OP_ALG_ALGSEL_SHA1,
1600 }, {
1601 .name = "sha224",
1602 .driver_name = "sha224-caam",
1603 .hmac_name = "hmac(sha224)",
1604 .hmac_driver_name = "hmac-sha224-caam",
1605 .blocksize = SHA224_BLOCK_SIZE,
1606 .template_ahash = {
1607 .init = ahash_init,
1608 .update = ahash_update,
1609 .final = ahash_final,
1610 .finup = ahash_finup,
1611 .digest = ahash_digest,
1612 .export = ahash_export,
1613 .import = ahash_import,
1614 .setkey = ahash_setkey,
1615 .halg = {
1616 .digestsize = SHA224_DIGEST_SIZE,
1617 .statesize = sizeof(struct caam_export_state),
1618 },
1619 },
1620 .alg_type = OP_ALG_ALGSEL_SHA224,
1621 }, {
1622 .name = "sha256",
1623 .driver_name = "sha256-caam",
1624 .hmac_name = "hmac(sha256)",
1625 .hmac_driver_name = "hmac-sha256-caam",
1626 .blocksize = SHA256_BLOCK_SIZE,
1627 .template_ahash = {
1628 .init = ahash_init,
1629 .update = ahash_update,
1630 .final = ahash_final,
1631 .finup = ahash_finup,
1632 .digest = ahash_digest,
1633 .export = ahash_export,
1634 .import = ahash_import,
1635 .setkey = ahash_setkey,
1636 .halg = {
1637 .digestsize = SHA256_DIGEST_SIZE,
1638 .statesize = sizeof(struct caam_export_state),
1639 },
1640 },
1641 .alg_type = OP_ALG_ALGSEL_SHA256,
1642 }, {
1643 .name = "sha384",
1644 .driver_name = "sha384-caam",
1645 .hmac_name = "hmac(sha384)",
1646 .hmac_driver_name = "hmac-sha384-caam",
1647 .blocksize = SHA384_BLOCK_SIZE,
1648 .template_ahash = {
1649 .init = ahash_init,
1650 .update = ahash_update,
1651 .final = ahash_final,
1652 .finup = ahash_finup,
1653 .digest = ahash_digest,
1654 .export = ahash_export,
1655 .import = ahash_import,
1656 .setkey = ahash_setkey,
1657 .halg = {
1658 .digestsize = SHA384_DIGEST_SIZE,
1659 .statesize = sizeof(struct caam_export_state),
1660 },
1661 },
1662 .alg_type = OP_ALG_ALGSEL_SHA384,
1663 }, {
1664 .name = "sha512",
1665 .driver_name = "sha512-caam",
1666 .hmac_name = "hmac(sha512)",
1667 .hmac_driver_name = "hmac-sha512-caam",
1668 .blocksize = SHA512_BLOCK_SIZE,
1669 .template_ahash = {
1670 .init = ahash_init,
1671 .update = ahash_update,
1672 .final = ahash_final,
1673 .finup = ahash_finup,
1674 .digest = ahash_digest,
1675 .export = ahash_export,
1676 .import = ahash_import,
1677 .setkey = ahash_setkey,
1678 .halg = {
1679 .digestsize = SHA512_DIGEST_SIZE,
1680 .statesize = sizeof(struct caam_export_state),
1681 },
1682 },
1683 .alg_type = OP_ALG_ALGSEL_SHA512,
1684 }, {
1685 .name = "md5",
1686 .driver_name = "md5-caam",
1687 .hmac_name = "hmac(md5)",
1688 .hmac_driver_name = "hmac-md5-caam",
1689 .blocksize = MD5_BLOCK_WORDS * 4,
1690 .template_ahash = {
1691 .init = ahash_init,
1692 .update = ahash_update,
1693 .final = ahash_final,
1694 .finup = ahash_finup,
1695 .digest = ahash_digest,
1696 .export = ahash_export,
1697 .import = ahash_import,
1698 .setkey = ahash_setkey,
1699 .halg = {
1700 .digestsize = MD5_DIGEST_SIZE,
1701 .statesize = sizeof(struct caam_export_state),
1702 },
1703 },
1704 .alg_type = OP_ALG_ALGSEL_MD5,
1705 }, {
1706 .hmac_name = "xcbc(aes)",
1707 .hmac_driver_name = "xcbc-aes-caam",
1708 .blocksize = AES_BLOCK_SIZE,
1709 .template_ahash = {
1710 .init = ahash_init,
1711 .update = ahash_update,
1712 .final = ahash_final,
1713 .finup = ahash_finup,
1714 .digest = ahash_digest,
1715 .export = ahash_export,
1716 .import = ahash_import,
1717 .setkey = axcbc_setkey,
1718 .halg = {
1719 .digestsize = AES_BLOCK_SIZE,
1720 .statesize = sizeof(struct caam_export_state),
1721 },
1722 },
1723 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC,
1724 }, {
1725 .hmac_name = "cmac(aes)",
1726 .hmac_driver_name = "cmac-aes-caam",
1727 .blocksize = AES_BLOCK_SIZE,
1728 .template_ahash = {
1729 .init = ahash_init,
1730 .update = ahash_update,
1731 .final = ahash_final,
1732 .finup = ahash_finup,
1733 .digest = ahash_digest,
1734 .export = ahash_export,
1735 .import = ahash_import,
1736 .setkey = acmac_setkey,
1737 .halg = {
1738 .digestsize = AES_BLOCK_SIZE,
1739 .statesize = sizeof(struct caam_export_state),
1740 },
1741 },
1742 .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC,
1743 },
1744 };
1745
1746 struct caam_hash_alg {
1747 struct list_head entry;
1748 int alg_type;
1749 struct ahash_alg ahash_alg;
1750 };
1751
caam_hash_cra_init(struct crypto_tfm * tfm)1752 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1753 {
1754 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1755 struct crypto_alg *base = tfm->__crt_alg;
1756 struct hash_alg_common *halg =
1757 container_of(base, struct hash_alg_common, base);
1758 struct ahash_alg *alg =
1759 container_of(halg, struct ahash_alg, halg);
1760 struct caam_hash_alg *caam_hash =
1761 container_of(alg, struct caam_hash_alg, ahash_alg);
1762 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1763 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1764 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1765 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1766 HASH_MSG_LEN + 32,
1767 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1768 HASH_MSG_LEN + 64,
1769 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1770 const size_t sh_desc_update_offset = offsetof(struct caam_hash_ctx,
1771 sh_desc_update);
1772 dma_addr_t dma_addr;
1773 struct caam_drv_private *priv;
1774
1775 /*
1776 * Get a Job ring from Job Ring driver to ensure in-order
1777 * crypto request processing per tfm
1778 */
1779 ctx->jrdev = caam_jr_alloc();
1780 if (IS_ERR(ctx->jrdev)) {
1781 pr_err("Job Ring Device allocation for transform failed\n");
1782 return PTR_ERR(ctx->jrdev);
1783 }
1784
1785 priv = dev_get_drvdata(ctx->jrdev->parent);
1786
1787 if (is_xcbc_aes(caam_hash->alg_type)) {
1788 ctx->dir = DMA_TO_DEVICE;
1789 ctx->key_dir = DMA_BIDIRECTIONAL;
1790 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
1791 ctx->ctx_len = 48;
1792 } else if (is_cmac_aes(caam_hash->alg_type)) {
1793 ctx->dir = DMA_TO_DEVICE;
1794 ctx->key_dir = DMA_NONE;
1795 ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
1796 ctx->ctx_len = 32;
1797 } else {
1798 if (priv->era >= 6) {
1799 ctx->dir = DMA_BIDIRECTIONAL;
1800 ctx->key_dir = alg->setkey ? DMA_TO_DEVICE : DMA_NONE;
1801 } else {
1802 ctx->dir = DMA_TO_DEVICE;
1803 ctx->key_dir = DMA_NONE;
1804 }
1805 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1806 ctx->ctx_len = runninglen[(ctx->adata.algtype &
1807 OP_ALG_ALGSEL_SUBMASK) >>
1808 OP_ALG_ALGSEL_SHIFT];
1809 }
1810
1811 if (ctx->key_dir != DMA_NONE) {
1812 ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
1813 ARRAY_SIZE(ctx->key),
1814 ctx->key_dir,
1815 DMA_ATTR_SKIP_CPU_SYNC);
1816 if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {
1817 dev_err(ctx->jrdev, "unable to map key\n");
1818 caam_jr_free(ctx->jrdev);
1819 return -ENOMEM;
1820 }
1821 }
1822
1823 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
1824 offsetof(struct caam_hash_ctx, key) -
1825 sh_desc_update_offset,
1826 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1827 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
1828 dev_err(ctx->jrdev, "unable to map shared descriptors\n");
1829
1830 if (ctx->key_dir != DMA_NONE)
1831 dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
1832 ARRAY_SIZE(ctx->key),
1833 ctx->key_dir,
1834 DMA_ATTR_SKIP_CPU_SYNC);
1835
1836 caam_jr_free(ctx->jrdev);
1837 return -ENOMEM;
1838 }
1839
1840 ctx->sh_desc_update_dma = dma_addr;
1841 ctx->sh_desc_update_first_dma = dma_addr +
1842 offsetof(struct caam_hash_ctx,
1843 sh_desc_update_first) -
1844 sh_desc_update_offset;
1845 ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
1846 sh_desc_fin) -
1847 sh_desc_update_offset;
1848 ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
1849 sh_desc_digest) -
1850 sh_desc_update_offset;
1851
1852 ctx->enginectx.op.do_one_request = ahash_do_one_req;
1853
1854 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1855 sizeof(struct caam_hash_state));
1856
1857 /*
1858 * For keyed hash algorithms shared descriptors
1859 * will be created later in setkey() callback
1860 */
1861 return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
1862 }
1863
caam_hash_cra_exit(struct crypto_tfm * tfm)1864 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1865 {
1866 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1867
1868 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
1869 offsetof(struct caam_hash_ctx, key) -
1870 offsetof(struct caam_hash_ctx, sh_desc_update),
1871 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
1872 if (ctx->key_dir != DMA_NONE)
1873 dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
1874 ARRAY_SIZE(ctx->key), ctx->key_dir,
1875 DMA_ATTR_SKIP_CPU_SYNC);
1876 caam_jr_free(ctx->jrdev);
1877 }
1878
caam_algapi_hash_exit(void)1879 void caam_algapi_hash_exit(void)
1880 {
1881 struct caam_hash_alg *t_alg, *n;
1882
1883 if (!hash_list.next)
1884 return;
1885
1886 list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1887 crypto_unregister_ahash(&t_alg->ahash_alg);
1888 list_del(&t_alg->entry);
1889 kfree(t_alg);
1890 }
1891 }
1892
1893 static struct caam_hash_alg *
caam_hash_alloc(struct caam_hash_template * template,bool keyed)1894 caam_hash_alloc(struct caam_hash_template *template,
1895 bool keyed)
1896 {
1897 struct caam_hash_alg *t_alg;
1898 struct ahash_alg *halg;
1899 struct crypto_alg *alg;
1900
1901 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1902 if (!t_alg) {
1903 pr_err("failed to allocate t_alg\n");
1904 return ERR_PTR(-ENOMEM);
1905 }
1906
1907 t_alg->ahash_alg = template->template_ahash;
1908 halg = &t_alg->ahash_alg;
1909 alg = &halg->halg.base;
1910
1911 if (keyed) {
1912 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1913 template->hmac_name);
1914 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1915 template->hmac_driver_name);
1916 } else {
1917 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1918 template->name);
1919 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1920 template->driver_name);
1921 t_alg->ahash_alg.setkey = NULL;
1922 }
1923 alg->cra_module = THIS_MODULE;
1924 alg->cra_init = caam_hash_cra_init;
1925 alg->cra_exit = caam_hash_cra_exit;
1926 alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1927 alg->cra_priority = CAAM_CRA_PRIORITY;
1928 alg->cra_blocksize = template->blocksize;
1929 alg->cra_alignmask = 0;
1930 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
1931
1932 t_alg->alg_type = template->alg_type;
1933
1934 return t_alg;
1935 }
1936
caam_algapi_hash_init(struct device * ctrldev)1937 int caam_algapi_hash_init(struct device *ctrldev)
1938 {
1939 int i = 0, err = 0;
1940 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
1941 unsigned int md_limit = SHA512_DIGEST_SIZE;
1942 u32 md_inst, md_vid;
1943
1944 /*
1945 * Register crypto algorithms the device supports. First, identify
1946 * presence and attributes of MD block.
1947 */
1948 if (priv->era < 10) {
1949 md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) &
1950 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
1951 md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
1952 CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
1953 } else {
1954 u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
1955
1956 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
1957 md_inst = mdha & CHA_VER_NUM_MASK;
1958 }
1959
1960 /*
1961 * Skip registration of any hashing algorithms if MD block
1962 * is not present.
1963 */
1964 if (!md_inst)
1965 return 0;
1966
1967 /* Limit digest size based on LP256 */
1968 if (md_vid == CHA_VER_VID_MD_LP256)
1969 md_limit = SHA256_DIGEST_SIZE;
1970
1971 INIT_LIST_HEAD(&hash_list);
1972
1973 /* register crypto algorithms the device supports */
1974 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1975 struct caam_hash_alg *t_alg;
1976 struct caam_hash_template *alg = driver_hash + i;
1977
1978 /* If MD size is not supported by device, skip registration */
1979 if (is_mdha(alg->alg_type) &&
1980 alg->template_ahash.halg.digestsize > md_limit)
1981 continue;
1982
1983 /* register hmac version */
1984 t_alg = caam_hash_alloc(alg, true);
1985 if (IS_ERR(t_alg)) {
1986 err = PTR_ERR(t_alg);
1987 pr_warn("%s alg allocation failed\n",
1988 alg->hmac_driver_name);
1989 continue;
1990 }
1991
1992 err = crypto_register_ahash(&t_alg->ahash_alg);
1993 if (err) {
1994 pr_warn("%s alg registration failed: %d\n",
1995 t_alg->ahash_alg.halg.base.cra_driver_name,
1996 err);
1997 kfree(t_alg);
1998 } else
1999 list_add_tail(&t_alg->entry, &hash_list);
2000
2001 if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES)
2002 continue;
2003
2004 /* register unkeyed version */
2005 t_alg = caam_hash_alloc(alg, false);
2006 if (IS_ERR(t_alg)) {
2007 err = PTR_ERR(t_alg);
2008 pr_warn("%s alg allocation failed\n", alg->driver_name);
2009 continue;
2010 }
2011
2012 err = crypto_register_ahash(&t_alg->ahash_alg);
2013 if (err) {
2014 pr_warn("%s alg registration failed: %d\n",
2015 t_alg->ahash_alg.halg.base.cra_driver_name,
2016 err);
2017 kfree(t_alg);
2018 } else
2019 list_add_tail(&t_alg->entry, &hash_list);
2020 }
2021
2022 return err;
2023 }
2024