1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cryptographic API.
4 * Support for Nomadik hardware crypto engine.
5
6 * Copyright (C) ST-Ericsson SA 2010
7 * Author: Shujuan Chen <shujuan.chen@stericsson.com> for ST-Ericsson
8 * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson
9 * Author: Berne Hebark <berne.herbark@stericsson.com> for ST-Ericsson.
10 * Author: Niklas Hernaeus <niklas.hernaeus@stericsson.com> for ST-Ericsson.
11 * Author: Andreas Westin <andreas.westin@stericsson.com> for ST-Ericsson.
12 */
13
14 #define pr_fmt(fmt) "hashX hashX: " fmt
15
16 #include <linux/clk.h>
17 #include <linux/device.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/init.h>
21 #include <linux/io.h>
22 #include <linux/klist.h>
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/mod_devicetable.h>
26 #include <linux/platform_device.h>
27 #include <linux/crypto.h>
28
29 #include <linux/regulator/consumer.h>
30 #include <linux/dmaengine.h>
31 #include <linux/bitops.h>
32
33 #include <crypto/internal/hash.h>
34 #include <crypto/sha1.h>
35 #include <crypto/sha2.h>
36 #include <crypto/scatterwalk.h>
37 #include <crypto/algapi.h>
38
39 #include <linux/platform_data/crypto-ux500.h>
40
41 #include "hash_alg.h"
42
43 static int hash_mode;
44 module_param(hash_mode, int, 0);
45 MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1");
46
47 /* HMAC-SHA1, no key */
48 static const u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = {
49 0xfb, 0xdb, 0x1d, 0x1b, 0x18, 0xaa, 0x6c, 0x08,
50 0x32, 0x4b, 0x7d, 0x64, 0xb7, 0x1f, 0xb7, 0x63,
51 0x70, 0x69, 0x0e, 0x1d
52 };
53
54 /* HMAC-SHA256, no key */
55 static const u8 zero_message_hmac_sha256[SHA256_DIGEST_SIZE] = {
56 0xb6, 0x13, 0x67, 0x9a, 0x08, 0x14, 0xd9, 0xec,
57 0x77, 0x2f, 0x95, 0xd7, 0x78, 0xc3, 0x5f, 0xc5,
58 0xff, 0x16, 0x97, 0xc4, 0x93, 0x71, 0x56, 0x53,
59 0xc6, 0xc7, 0x12, 0x14, 0x42, 0x92, 0xc5, 0xad
60 };
61
62 /**
63 * struct hash_driver_data - data specific to the driver.
64 *
65 * @device_list: A list of registered devices to choose from.
66 * @device_allocation: A semaphore initialized with number of devices.
67 */
68 struct hash_driver_data {
69 struct klist device_list;
70 struct semaphore device_allocation;
71 };
72
73 static struct hash_driver_data driver_data;
74
75 /* Declaration of functions */
76 /**
77 * hash_messagepad - Pads a message and write the nblw bits.
78 * @device_data: Structure for the hash device.
79 * @message: Last word of a message
80 * @index_bytes: The number of bytes in the last message
81 *
82 * This function manages the final part of the digest calculation, when less
83 * than 512 bits (64 bytes) remain in message. This means index_bytes < 64.
84 *
85 */
86 static void hash_messagepad(struct hash_device_data *device_data,
87 const u32 *message, u8 index_bytes);
88
89 /**
90 * release_hash_device - Releases a previously allocated hash device.
91 * @device_data: Structure for the hash device.
92 *
93 */
release_hash_device(struct hash_device_data * device_data)94 static void release_hash_device(struct hash_device_data *device_data)
95 {
96 spin_lock(&device_data->ctx_lock);
97 device_data->current_ctx->device = NULL;
98 device_data->current_ctx = NULL;
99 spin_unlock(&device_data->ctx_lock);
100
101 /*
102 * The down_interruptible part for this semaphore is called in
103 * cryp_get_device_data.
104 */
105 up(&driver_data.device_allocation);
106 }
107
hash_dma_setup_channel(struct hash_device_data * device_data,struct device * dev)108 static void hash_dma_setup_channel(struct hash_device_data *device_data,
109 struct device *dev)
110 {
111 struct hash_platform_data *platform_data = dev->platform_data;
112 struct dma_slave_config conf = {
113 .direction = DMA_MEM_TO_DEV,
114 .dst_addr = device_data->phybase + HASH_DMA_FIFO,
115 .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
116 .dst_maxburst = 16,
117 };
118
119 dma_cap_zero(device_data->dma.mask);
120 dma_cap_set(DMA_SLAVE, device_data->dma.mask);
121
122 device_data->dma.cfg_mem2hash = platform_data->mem_to_engine;
123 device_data->dma.chan_mem2hash =
124 dma_request_channel(device_data->dma.mask,
125 platform_data->dma_filter,
126 device_data->dma.cfg_mem2hash);
127
128 dmaengine_slave_config(device_data->dma.chan_mem2hash, &conf);
129
130 init_completion(&device_data->dma.complete);
131 }
132
hash_dma_callback(void * data)133 static void hash_dma_callback(void *data)
134 {
135 struct hash_ctx *ctx = data;
136
137 complete(&ctx->device->dma.complete);
138 }
139
hash_set_dma_transfer(struct hash_ctx * ctx,struct scatterlist * sg,int len,enum dma_data_direction direction)140 static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg,
141 int len, enum dma_data_direction direction)
142 {
143 struct dma_async_tx_descriptor *desc = NULL;
144 struct dma_chan *channel = NULL;
145
146 if (direction != DMA_TO_DEVICE) {
147 dev_err(ctx->device->dev, "%s: Invalid DMA direction\n",
148 __func__);
149 return -EFAULT;
150 }
151
152 sg->length = ALIGN(sg->length, HASH_DMA_ALIGN_SIZE);
153
154 channel = ctx->device->dma.chan_mem2hash;
155 ctx->device->dma.sg = sg;
156 ctx->device->dma.sg_len = dma_map_sg(channel->device->dev,
157 ctx->device->dma.sg, ctx->device->dma.nents,
158 direction);
159
160 if (!ctx->device->dma.sg_len) {
161 dev_err(ctx->device->dev, "%s: Could not map the sg list (TO_DEVICE)\n",
162 __func__);
163 return -EFAULT;
164 }
165
166 dev_dbg(ctx->device->dev, "%s: Setting up DMA for buffer (TO_DEVICE)\n",
167 __func__);
168 desc = dmaengine_prep_slave_sg(channel,
169 ctx->device->dma.sg, ctx->device->dma.sg_len,
170 DMA_MEM_TO_DEV, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
171 if (!desc) {
172 dev_err(ctx->device->dev,
173 "%s: dmaengine_prep_slave_sg() failed!\n", __func__);
174 return -EFAULT;
175 }
176
177 desc->callback = hash_dma_callback;
178 desc->callback_param = ctx;
179
180 dmaengine_submit(desc);
181 dma_async_issue_pending(channel);
182
183 return 0;
184 }
185
hash_dma_done(struct hash_ctx * ctx)186 static void hash_dma_done(struct hash_ctx *ctx)
187 {
188 struct dma_chan *chan;
189
190 chan = ctx->device->dma.chan_mem2hash;
191 dmaengine_terminate_all(chan);
192 dma_unmap_sg(chan->device->dev, ctx->device->dma.sg,
193 ctx->device->dma.nents, DMA_TO_DEVICE);
194 }
195
hash_dma_write(struct hash_ctx * ctx,struct scatterlist * sg,int len)196 static int hash_dma_write(struct hash_ctx *ctx,
197 struct scatterlist *sg, int len)
198 {
199 int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE);
200 if (error) {
201 dev_dbg(ctx->device->dev,
202 "%s: hash_set_dma_transfer() failed\n", __func__);
203 return error;
204 }
205
206 return len;
207 }
208
209 /**
210 * get_empty_message_digest - Returns a pre-calculated digest for
211 * the empty message.
212 * @device_data: Structure for the hash device.
213 * @zero_hash: Buffer to return the empty message digest.
214 * @zero_hash_size: Hash size of the empty message digest.
215 * @zero_digest: True if zero_digest returned.
216 */
get_empty_message_digest(struct hash_device_data * device_data,u8 * zero_hash,u32 * zero_hash_size,bool * zero_digest)217 static int get_empty_message_digest(
218 struct hash_device_data *device_data,
219 u8 *zero_hash, u32 *zero_hash_size, bool *zero_digest)
220 {
221 int ret = 0;
222 struct hash_ctx *ctx = device_data->current_ctx;
223 *zero_digest = false;
224
225 /**
226 * Caller responsible for ctx != NULL.
227 */
228
229 if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) {
230 if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
231 memcpy(zero_hash, &sha1_zero_message_hash[0],
232 SHA1_DIGEST_SIZE);
233 *zero_hash_size = SHA1_DIGEST_SIZE;
234 *zero_digest = true;
235 } else if (HASH_ALGO_SHA256 ==
236 ctx->config.algorithm) {
237 memcpy(zero_hash, &sha256_zero_message_hash[0],
238 SHA256_DIGEST_SIZE);
239 *zero_hash_size = SHA256_DIGEST_SIZE;
240 *zero_digest = true;
241 } else {
242 dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
243 __func__);
244 ret = -EINVAL;
245 goto out;
246 }
247 } else if (HASH_OPER_MODE_HMAC == ctx->config.oper_mode) {
248 if (!ctx->keylen) {
249 if (HASH_ALGO_SHA1 == ctx->config.algorithm) {
250 memcpy(zero_hash, &zero_message_hmac_sha1[0],
251 SHA1_DIGEST_SIZE);
252 *zero_hash_size = SHA1_DIGEST_SIZE;
253 *zero_digest = true;
254 } else if (HASH_ALGO_SHA256 == ctx->config.algorithm) {
255 memcpy(zero_hash, &zero_message_hmac_sha256[0],
256 SHA256_DIGEST_SIZE);
257 *zero_hash_size = SHA256_DIGEST_SIZE;
258 *zero_digest = true;
259 } else {
260 dev_err(device_data->dev, "%s: Incorrect algorithm!\n",
261 __func__);
262 ret = -EINVAL;
263 goto out;
264 }
265 } else {
266 dev_dbg(device_data->dev,
267 "%s: Continue hash calculation, since hmac key available\n",
268 __func__);
269 }
270 }
271 out:
272
273 return ret;
274 }
275
276 /**
277 * hash_disable_power - Request to disable power and clock.
278 * @device_data: Structure for the hash device.
279 * @save_device_state: If true, saves the current hw state.
280 *
281 * This function request for disabling power (regulator) and clock,
282 * and could also save current hw state.
283 */
hash_disable_power(struct hash_device_data * device_data,bool save_device_state)284 static int hash_disable_power(struct hash_device_data *device_data,
285 bool save_device_state)
286 {
287 int ret = 0;
288 struct device *dev = device_data->dev;
289
290 spin_lock(&device_data->power_state_lock);
291 if (!device_data->power_state)
292 goto out;
293
294 if (save_device_state) {
295 hash_save_state(device_data,
296 &device_data->state);
297 device_data->restore_dev_state = true;
298 }
299
300 clk_disable(device_data->clk);
301 ret = regulator_disable(device_data->regulator);
302 if (ret)
303 dev_err(dev, "%s: regulator_disable() failed!\n", __func__);
304
305 device_data->power_state = false;
306
307 out:
308 spin_unlock(&device_data->power_state_lock);
309
310 return ret;
311 }
312
313 /**
314 * hash_enable_power - Request to enable power and clock.
315 * @device_data: Structure for the hash device.
316 * @restore_device_state: If true, restores a previous saved hw state.
317 *
318 * This function request for enabling power (regulator) and clock,
319 * and could also restore a previously saved hw state.
320 */
hash_enable_power(struct hash_device_data * device_data,bool restore_device_state)321 static int hash_enable_power(struct hash_device_data *device_data,
322 bool restore_device_state)
323 {
324 int ret = 0;
325 struct device *dev = device_data->dev;
326
327 spin_lock(&device_data->power_state_lock);
328 if (!device_data->power_state) {
329 ret = regulator_enable(device_data->regulator);
330 if (ret) {
331 dev_err(dev, "%s: regulator_enable() failed!\n",
332 __func__);
333 goto out;
334 }
335 ret = clk_enable(device_data->clk);
336 if (ret) {
337 dev_err(dev, "%s: clk_enable() failed!\n", __func__);
338 ret = regulator_disable(
339 device_data->regulator);
340 goto out;
341 }
342 device_data->power_state = true;
343 }
344
345 if (device_data->restore_dev_state) {
346 if (restore_device_state) {
347 device_data->restore_dev_state = false;
348 hash_resume_state(device_data, &device_data->state);
349 }
350 }
351 out:
352 spin_unlock(&device_data->power_state_lock);
353
354 return ret;
355 }
356
357 /**
358 * hash_get_device_data - Checks for an available hash device and return it.
359 * @ctx: Structure for the hash context.
360 * @device_data: Structure for the hash device.
361 *
362 * This function check for an available hash device and return it to
363 * the caller.
364 * Note! Caller need to release the device, calling up().
365 */
hash_get_device_data(struct hash_ctx * ctx,struct hash_device_data ** device_data)366 static int hash_get_device_data(struct hash_ctx *ctx,
367 struct hash_device_data **device_data)
368 {
369 int ret;
370 struct klist_iter device_iterator;
371 struct klist_node *device_node;
372 struct hash_device_data *local_device_data = NULL;
373
374 /* Wait until a device is available */
375 ret = down_interruptible(&driver_data.device_allocation);
376 if (ret)
377 return ret; /* Interrupted */
378
379 /* Select a device */
380 klist_iter_init(&driver_data.device_list, &device_iterator);
381 device_node = klist_next(&device_iterator);
382 while (device_node) {
383 local_device_data = container_of(device_node,
384 struct hash_device_data, list_node);
385 spin_lock(&local_device_data->ctx_lock);
386 /* current_ctx allocates a device, NULL = unallocated */
387 if (local_device_data->current_ctx) {
388 device_node = klist_next(&device_iterator);
389 } else {
390 local_device_data->current_ctx = ctx;
391 ctx->device = local_device_data;
392 spin_unlock(&local_device_data->ctx_lock);
393 break;
394 }
395 spin_unlock(&local_device_data->ctx_lock);
396 }
397 klist_iter_exit(&device_iterator);
398
399 if (!device_node) {
400 /**
401 * No free device found.
402 * Since we allocated a device with down_interruptible, this
403 * should not be able to happen.
404 * Number of available devices, which are contained in
405 * device_allocation, is therefore decremented by not doing
406 * an up(device_allocation).
407 */
408 return -EBUSY;
409 }
410
411 *device_data = local_device_data;
412
413 return 0;
414 }
415
416 /**
417 * hash_hw_write_key - Writes the key to the hardware registries.
418 *
419 * @device_data: Structure for the hash device.
420 * @key: Key to be written.
421 * @keylen: The lengt of the key.
422 *
423 * Note! This function DOES NOT write to the NBLW registry, even though
424 * specified in the the hw design spec. Either due to incorrect info in the
425 * spec or due to a bug in the hw.
426 */
hash_hw_write_key(struct hash_device_data * device_data,const u8 * key,unsigned int keylen)427 static void hash_hw_write_key(struct hash_device_data *device_data,
428 const u8 *key, unsigned int keylen)
429 {
430 u32 word = 0;
431 int nwords = 1;
432
433 HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);
434
435 while (keylen >= 4) {
436 u32 *key_word = (u32 *)key;
437
438 HASH_SET_DIN(key_word, nwords);
439 keylen -= 4;
440 key += 4;
441 }
442
443 /* Take care of the remaining bytes in the last word */
444 if (keylen) {
445 word = 0;
446 while (keylen) {
447 word |= (key[keylen - 1] << (8 * (keylen - 1)));
448 keylen--;
449 }
450
451 HASH_SET_DIN(&word, nwords);
452 }
453
454 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
455 cpu_relax();
456
457 HASH_SET_DCAL;
458
459 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
460 cpu_relax();
461 }
462
463 /**
464 * init_hash_hw - Initialise the hash hardware for a new calculation.
465 * @device_data: Structure for the hash device.
466 * @ctx: The hash context.
467 *
468 * This function will enable the bits needed to clear and start a new
469 * calculation.
470 */
init_hash_hw(struct hash_device_data * device_data,struct hash_ctx * ctx)471 static int init_hash_hw(struct hash_device_data *device_data,
472 struct hash_ctx *ctx)
473 {
474 int ret = 0;
475
476 ret = hash_setconfiguration(device_data, &ctx->config);
477 if (ret) {
478 dev_err(device_data->dev, "%s: hash_setconfiguration() failed!\n",
479 __func__);
480 return ret;
481 }
482
483 hash_begin(device_data, ctx);
484
485 if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC)
486 hash_hw_write_key(device_data, ctx->key, ctx->keylen);
487
488 return ret;
489 }
490
491 /**
492 * hash_get_nents - Return number of entries (nents) in scatterlist (sg).
493 *
494 * @sg: Scatterlist.
495 * @size: Size in bytes.
496 * @aligned: True if sg data aligned to work in DMA mode.
497 *
498 */
hash_get_nents(struct scatterlist * sg,int size,bool * aligned)499 static int hash_get_nents(struct scatterlist *sg, int size, bool *aligned)
500 {
501 int nents = 0;
502 bool aligned_data = true;
503
504 while (size > 0 && sg) {
505 nents++;
506 size -= sg->length;
507
508 /* hash_set_dma_transfer will align last nent */
509 if ((aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE)) ||
510 (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) && size > 0))
511 aligned_data = false;
512
513 sg = sg_next(sg);
514 }
515
516 if (aligned)
517 *aligned = aligned_data;
518
519 if (size != 0)
520 return -EFAULT;
521
522 return nents;
523 }
524
525 /**
526 * hash_dma_valid_data - checks for dma valid sg data.
527 * @sg: Scatterlist.
528 * @datasize: Datasize in bytes.
529 *
530 * NOTE! This function checks for dma valid sg data, since dma
531 * only accept datasizes of even wordsize.
532 */
hash_dma_valid_data(struct scatterlist * sg,int datasize)533 static bool hash_dma_valid_data(struct scatterlist *sg, int datasize)
534 {
535 bool aligned;
536
537 /* Need to include at least one nent, else error */
538 if (hash_get_nents(sg, datasize, &aligned) < 1)
539 return false;
540
541 return aligned;
542 }
543
544 /**
545 * ux500_hash_init - Common hash init function for SHA1/SHA2 (SHA256).
546 * @req: The hash request for the job.
547 *
548 * Initialize structures.
549 */
ux500_hash_init(struct ahash_request * req)550 static int ux500_hash_init(struct ahash_request *req)
551 {
552 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
553 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
554 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
555
556 if (!ctx->key)
557 ctx->keylen = 0;
558
559 memset(&req_ctx->state, 0, sizeof(struct hash_state));
560 req_ctx->updated = 0;
561 if (hash_mode == HASH_MODE_DMA) {
562 if (req->nbytes < HASH_DMA_ALIGN_SIZE) {
563 req_ctx->dma_mode = false; /* Don't use DMA */
564
565 pr_debug("%s: DMA mode, but direct to CPU mode for data size < %d\n",
566 __func__, HASH_DMA_ALIGN_SIZE);
567 } else {
568 if (req->nbytes >= HASH_DMA_PERFORMANCE_MIN_SIZE &&
569 hash_dma_valid_data(req->src, req->nbytes)) {
570 req_ctx->dma_mode = true;
571 } else {
572 req_ctx->dma_mode = false;
573 pr_debug("%s: DMA mode, but use CPU mode for datalength < %d or non-aligned data, except in last nent\n",
574 __func__,
575 HASH_DMA_PERFORMANCE_MIN_SIZE);
576 }
577 }
578 }
579 return 0;
580 }
581
582 /**
583 * hash_processblock - This function processes a single block of 512 bits (64
584 * bytes), word aligned, starting at message.
585 * @device_data: Structure for the hash device.
586 * @message: Block (512 bits) of message to be written to
587 * the HASH hardware.
588 * @length: Message length
589 *
590 */
hash_processblock(struct hash_device_data * device_data,const u32 * message,int length)591 static void hash_processblock(struct hash_device_data *device_data,
592 const u32 *message, int length)
593 {
594 int len = length / HASH_BYTES_PER_WORD;
595 /*
596 * NBLW bits. Reset the number of bits in last word (NBLW).
597 */
598 HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);
599
600 /*
601 * Write message data to the HASH_DIN register.
602 */
603 HASH_SET_DIN(message, len);
604 }
605
606 /**
607 * hash_messagepad - Pads a message and write the nblw bits.
608 * @device_data: Structure for the hash device.
609 * @message: Last word of a message.
610 * @index_bytes: The number of bytes in the last message.
611 *
612 * This function manages the final part of the digest calculation, when less
613 * than 512 bits (64 bytes) remain in message. This means index_bytes < 64.
614 *
615 */
hash_messagepad(struct hash_device_data * device_data,const u32 * message,u8 index_bytes)616 static void hash_messagepad(struct hash_device_data *device_data,
617 const u32 *message, u8 index_bytes)
618 {
619 int nwords = 1;
620
621 /*
622 * Clear hash str register, only clear NBLW
623 * since DCAL will be reset by hardware.
624 */
625 HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);
626
627 /* Main loop */
628 while (index_bytes >= 4) {
629 HASH_SET_DIN(message, nwords);
630 index_bytes -= 4;
631 message++;
632 }
633
634 if (index_bytes)
635 HASH_SET_DIN(message, nwords);
636
637 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
638 cpu_relax();
639
640 /* num_of_bytes == 0 => NBLW <- 0 (32 bits valid in DATAIN) */
641 HASH_SET_NBLW(index_bytes * 8);
642 dev_dbg(device_data->dev, "%s: DIN=0x%08x NBLW=%lu\n",
643 __func__, readl_relaxed(&device_data->base->din),
644 readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
645 HASH_SET_DCAL;
646 dev_dbg(device_data->dev, "%s: after dcal -> DIN=0x%08x NBLW=%lu\n",
647 __func__, readl_relaxed(&device_data->base->din),
648 readl_relaxed(&device_data->base->str) & HASH_STR_NBLW_MASK);
649
650 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
651 cpu_relax();
652 }
653
654 /**
655 * hash_incrementlength - Increments the length of the current message.
656 * @ctx: Hash context
657 * @incr: Length of message processed already
658 *
659 * Overflow cannot occur, because conditions for overflow are checked in
660 * hash_hw_update.
661 */
hash_incrementlength(struct hash_req_ctx * ctx,u32 incr)662 static void hash_incrementlength(struct hash_req_ctx *ctx, u32 incr)
663 {
664 ctx->state.length.low_word += incr;
665
666 /* Check for wrap-around */
667 if (ctx->state.length.low_word < incr)
668 ctx->state.length.high_word++;
669 }
670
671 /**
672 * hash_setconfiguration - Sets the required configuration for the hash
673 * hardware.
674 * @device_data: Structure for the hash device.
675 * @config: Pointer to a configuration structure.
676 */
hash_setconfiguration(struct hash_device_data * device_data,struct hash_config * config)677 int hash_setconfiguration(struct hash_device_data *device_data,
678 struct hash_config *config)
679 {
680 int ret = 0;
681
682 if (config->algorithm != HASH_ALGO_SHA1 &&
683 config->algorithm != HASH_ALGO_SHA256)
684 return -EPERM;
685
686 /*
687 * DATAFORM bits. Set the DATAFORM bits to 0b11, which means the data
688 * to be written to HASH_DIN is considered as 32 bits.
689 */
690 HASH_SET_DATA_FORMAT(config->data_format);
691
692 /*
693 * ALGO bit. Set to 0b1 for SHA-1 and 0b0 for SHA-256
694 */
695 switch (config->algorithm) {
696 case HASH_ALGO_SHA1:
697 HASH_SET_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK);
698 break;
699
700 case HASH_ALGO_SHA256:
701 HASH_CLEAR_BITS(&device_data->base->cr, HASH_CR_ALGO_MASK);
702 break;
703
704 default:
705 dev_err(device_data->dev, "%s: Incorrect algorithm\n",
706 __func__);
707 return -EPERM;
708 }
709
710 /*
711 * MODE bit. This bit selects between HASH or HMAC mode for the
712 * selected algorithm. 0b0 = HASH and 0b1 = HMAC.
713 */
714 if (HASH_OPER_MODE_HASH == config->oper_mode)
715 HASH_CLEAR_BITS(&device_data->base->cr,
716 HASH_CR_MODE_MASK);
717 else if (HASH_OPER_MODE_HMAC == config->oper_mode) {
718 HASH_SET_BITS(&device_data->base->cr, HASH_CR_MODE_MASK);
719 if (device_data->current_ctx->keylen > HASH_BLOCK_SIZE) {
720 /* Truncate key to blocksize */
721 dev_dbg(device_data->dev, "%s: LKEY set\n", __func__);
722 HASH_SET_BITS(&device_data->base->cr,
723 HASH_CR_LKEY_MASK);
724 } else {
725 dev_dbg(device_data->dev, "%s: LKEY cleared\n",
726 __func__);
727 HASH_CLEAR_BITS(&device_data->base->cr,
728 HASH_CR_LKEY_MASK);
729 }
730 } else { /* Wrong hash mode */
731 ret = -EPERM;
732 dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
733 __func__);
734 }
735 return ret;
736 }
737
738 /**
739 * hash_begin - This routine resets some globals and initializes the hash
740 * hardware.
741 * @device_data: Structure for the hash device.
742 * @ctx: Hash context.
743 */
hash_begin(struct hash_device_data * device_data,struct hash_ctx * ctx)744 void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx)
745 {
746 /* HW and SW initializations */
747 /* Note: there is no need to initialize buffer and digest members */
748
749 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
750 cpu_relax();
751
752 /*
753 * INIT bit. Set this bit to 0b1 to reset the HASH processor core and
754 * prepare the initialize the HASH accelerator to compute the message
755 * digest of a new message.
756 */
757 HASH_INITIALIZE;
758
759 /*
760 * NBLW bits. Reset the number of bits in last word (NBLW).
761 */
762 HASH_CLEAR_BITS(&device_data->base->str, HASH_STR_NBLW_MASK);
763 }
764
hash_process_data(struct hash_device_data * device_data,struct hash_ctx * ctx,struct hash_req_ctx * req_ctx,int msg_length,u8 * data_buffer,u8 * buffer,u8 * index)765 static int hash_process_data(struct hash_device_data *device_data,
766 struct hash_ctx *ctx, struct hash_req_ctx *req_ctx,
767 int msg_length, u8 *data_buffer, u8 *buffer,
768 u8 *index)
769 {
770 int ret = 0;
771 u32 count;
772
773 do {
774 if ((*index + msg_length) < HASH_BLOCK_SIZE) {
775 for (count = 0; count < msg_length; count++) {
776 buffer[*index + count] =
777 *(data_buffer + count);
778 }
779 *index += msg_length;
780 msg_length = 0;
781 } else {
782 if (req_ctx->updated) {
783 ret = hash_resume_state(device_data,
784 &device_data->state);
785 memmove(req_ctx->state.buffer,
786 device_data->state.buffer,
787 HASH_BLOCK_SIZE);
788 if (ret) {
789 dev_err(device_data->dev,
790 "%s: hash_resume_state() failed!\n",
791 __func__);
792 goto out;
793 }
794 } else {
795 ret = init_hash_hw(device_data, ctx);
796 if (ret) {
797 dev_err(device_data->dev,
798 "%s: init_hash_hw() failed!\n",
799 __func__);
800 goto out;
801 }
802 req_ctx->updated = 1;
803 }
804 /*
805 * If 'data_buffer' is four byte aligned and
806 * local buffer does not have any data, we can
807 * write data directly from 'data_buffer' to
808 * HW peripheral, otherwise we first copy data
809 * to a local buffer
810 */
811 if (IS_ALIGNED((unsigned long)data_buffer, 4) &&
812 (0 == *index))
813 hash_processblock(device_data,
814 (const u32 *)data_buffer,
815 HASH_BLOCK_SIZE);
816 else {
817 for (count = 0;
818 count < (u32)(HASH_BLOCK_SIZE - *index);
819 count++) {
820 buffer[*index + count] =
821 *(data_buffer + count);
822 }
823 hash_processblock(device_data,
824 (const u32 *)buffer,
825 HASH_BLOCK_SIZE);
826 }
827 hash_incrementlength(req_ctx, HASH_BLOCK_SIZE);
828 data_buffer += (HASH_BLOCK_SIZE - *index);
829
830 msg_length -= (HASH_BLOCK_SIZE - *index);
831 *index = 0;
832
833 ret = hash_save_state(device_data,
834 &device_data->state);
835
836 memmove(device_data->state.buffer,
837 req_ctx->state.buffer,
838 HASH_BLOCK_SIZE);
839 if (ret) {
840 dev_err(device_data->dev, "%s: hash_save_state() failed!\n",
841 __func__);
842 goto out;
843 }
844 }
845 } while (msg_length != 0);
846 out:
847
848 return ret;
849 }
850
851 /**
852 * hash_dma_final - The hash dma final function for SHA1/SHA256.
853 * @req: The hash request for the job.
854 */
hash_dma_final(struct ahash_request * req)855 static int hash_dma_final(struct ahash_request *req)
856 {
857 int ret = 0;
858 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
859 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
860 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
861 struct hash_device_data *device_data;
862 u8 digest[SHA256_DIGEST_SIZE];
863 int bytes_written = 0;
864
865 ret = hash_get_device_data(ctx, &device_data);
866 if (ret)
867 return ret;
868
869 dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__,
870 (unsigned long)ctx);
871
872 if (req_ctx->updated) {
873 ret = hash_resume_state(device_data, &device_data->state);
874
875 if (ret) {
876 dev_err(device_data->dev, "%s: hash_resume_state() failed!\n",
877 __func__);
878 goto out;
879 }
880 }
881
882 if (!req_ctx->updated) {
883 ret = hash_setconfiguration(device_data, &ctx->config);
884 if (ret) {
885 dev_err(device_data->dev,
886 "%s: hash_setconfiguration() failed!\n",
887 __func__);
888 goto out;
889 }
890
891 /* Enable DMA input */
892 if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode) {
893 HASH_CLEAR_BITS(&device_data->base->cr,
894 HASH_CR_DMAE_MASK);
895 } else {
896 HASH_SET_BITS(&device_data->base->cr,
897 HASH_CR_DMAE_MASK);
898 HASH_SET_BITS(&device_data->base->cr,
899 HASH_CR_PRIVN_MASK);
900 }
901
902 HASH_INITIALIZE;
903
904 if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC)
905 hash_hw_write_key(device_data, ctx->key, ctx->keylen);
906
907 /* Number of bits in last word = (nbytes * 8) % 32 */
908 HASH_SET_NBLW((req->nbytes * 8) % 32);
909 req_ctx->updated = 1;
910 }
911
912 /* Store the nents in the dma struct. */
913 ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL);
914 if (!ctx->device->dma.nents) {
915 dev_err(device_data->dev, "%s: ctx->device->dma.nents = 0\n",
916 __func__);
917 ret = ctx->device->dma.nents;
918 goto out;
919 }
920
921 bytes_written = hash_dma_write(ctx, req->src, req->nbytes);
922 if (bytes_written != req->nbytes) {
923 dev_err(device_data->dev, "%s: hash_dma_write() failed!\n",
924 __func__);
925 ret = bytes_written;
926 goto out;
927 }
928
929 wait_for_completion(&ctx->device->dma.complete);
930 hash_dma_done(ctx);
931
932 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
933 cpu_relax();
934
935 if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) {
936 unsigned int keylen = ctx->keylen;
937 u8 *key = ctx->key;
938
939 dev_dbg(device_data->dev, "%s: keylen: %d\n",
940 __func__, ctx->keylen);
941 hash_hw_write_key(device_data, key, keylen);
942 }
943
944 hash_get_digest(device_data, digest, ctx->config.algorithm);
945 memcpy(req->result, digest, ctx->digestsize);
946
947 out:
948 release_hash_device(device_data);
949
950 /**
951 * Allocated in setkey, and only used in HMAC.
952 */
953 kfree(ctx->key);
954
955 return ret;
956 }
957
958 /**
959 * hash_hw_final - The final hash calculation function
960 * @req: The hash request for the job.
961 */
hash_hw_final(struct ahash_request * req)962 static int hash_hw_final(struct ahash_request *req)
963 {
964 int ret = 0;
965 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
966 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
967 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
968 struct hash_device_data *device_data;
969 u8 digest[SHA256_DIGEST_SIZE];
970
971 ret = hash_get_device_data(ctx, &device_data);
972 if (ret)
973 return ret;
974
975 dev_dbg(device_data->dev, "%s: (ctx=0x%lx)!\n", __func__,
976 (unsigned long)ctx);
977
978 if (req_ctx->updated) {
979 ret = hash_resume_state(device_data, &device_data->state);
980
981 if (ret) {
982 dev_err(device_data->dev,
983 "%s: hash_resume_state() failed!\n", __func__);
984 goto out;
985 }
986 } else if (req->nbytes == 0 && ctx->keylen == 0) {
987 u8 zero_hash[SHA256_DIGEST_SIZE];
988 u32 zero_hash_size = 0;
989 bool zero_digest = false;
990 /**
991 * Use a pre-calculated empty message digest
992 * (workaround since hw return zeroes, hw bug!?)
993 */
994 ret = get_empty_message_digest(device_data, &zero_hash[0],
995 &zero_hash_size, &zero_digest);
996 if (!ret && likely(zero_hash_size == ctx->digestsize) &&
997 zero_digest) {
998 memcpy(req->result, &zero_hash[0], ctx->digestsize);
999 goto out;
1000 } else if (!ret && !zero_digest) {
1001 dev_dbg(device_data->dev,
1002 "%s: HMAC zero msg with key, continue...\n",
1003 __func__);
1004 } else {
1005 dev_err(device_data->dev,
1006 "%s: ret=%d, or wrong digest size? %s\n",
1007 __func__, ret,
1008 zero_hash_size == ctx->digestsize ?
1009 "true" : "false");
1010 /* Return error */
1011 goto out;
1012 }
1013 } else if (req->nbytes == 0 && ctx->keylen > 0) {
1014 dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n",
1015 __func__);
1016 goto out;
1017 }
1018
1019 if (!req_ctx->updated) {
1020 ret = init_hash_hw(device_data, ctx);
1021 if (ret) {
1022 dev_err(device_data->dev,
1023 "%s: init_hash_hw() failed!\n", __func__);
1024 goto out;
1025 }
1026 }
1027
1028 if (req_ctx->state.index) {
1029 hash_messagepad(device_data, req_ctx->state.buffer,
1030 req_ctx->state.index);
1031 } else {
1032 HASH_SET_DCAL;
1033 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
1034 cpu_relax();
1035 }
1036
1037 if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) {
1038 unsigned int keylen = ctx->keylen;
1039 u8 *key = ctx->key;
1040
1041 dev_dbg(device_data->dev, "%s: keylen: %d\n",
1042 __func__, ctx->keylen);
1043 hash_hw_write_key(device_data, key, keylen);
1044 }
1045
1046 hash_get_digest(device_data, digest, ctx->config.algorithm);
1047 memcpy(req->result, digest, ctx->digestsize);
1048
1049 out:
1050 release_hash_device(device_data);
1051
1052 /**
1053 * Allocated in setkey, and only used in HMAC.
1054 */
1055 kfree(ctx->key);
1056
1057 return ret;
1058 }
1059
1060 /**
1061 * hash_hw_update - Updates current HASH computation hashing another part of
1062 * the message.
1063 * @req: Byte array containing the message to be hashed (caller
1064 * allocated).
1065 */
hash_hw_update(struct ahash_request * req)1066 int hash_hw_update(struct ahash_request *req)
1067 {
1068 int ret = 0;
1069 u8 index = 0;
1070 u8 *buffer;
1071 struct hash_device_data *device_data;
1072 u8 *data_buffer;
1073 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1074 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1075 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
1076 struct crypto_hash_walk walk;
1077 int msg_length;
1078
1079 index = req_ctx->state.index;
1080 buffer = (u8 *)req_ctx->state.buffer;
1081
1082 ret = hash_get_device_data(ctx, &device_data);
1083 if (ret)
1084 return ret;
1085
1086 msg_length = crypto_hash_walk_first(req, &walk);
1087
1088 /* Empty message ("") is correct indata */
1089 if (msg_length == 0) {
1090 ret = 0;
1091 goto release_dev;
1092 }
1093
1094 /* Check if ctx->state.length + msg_length
1095 overflows */
1096 if (msg_length > (req_ctx->state.length.low_word + msg_length) &&
1097 HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) {
1098 pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__);
1099 ret = crypto_hash_walk_done(&walk, -EPERM);
1100 goto release_dev;
1101 }
1102
1103 /* Main loop */
1104 while (0 != msg_length) {
1105 data_buffer = walk.data;
1106 ret = hash_process_data(device_data, ctx, req_ctx, msg_length,
1107 data_buffer, buffer, &index);
1108
1109 if (ret) {
1110 dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n",
1111 __func__);
1112 crypto_hash_walk_done(&walk, ret);
1113 goto release_dev;
1114 }
1115
1116 msg_length = crypto_hash_walk_done(&walk, 0);
1117 }
1118
1119 req_ctx->state.index = index;
1120 dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n",
1121 __func__, req_ctx->state.index, req_ctx->state.bit_index);
1122
1123 release_dev:
1124 release_hash_device(device_data);
1125
1126 return ret;
1127 }
1128
1129 /**
1130 * hash_resume_state - Function that resumes the state of an calculation.
1131 * @device_data: Pointer to the device structure.
1132 * @device_state: The state to be restored in the hash hardware
1133 */
hash_resume_state(struct hash_device_data * device_data,const struct hash_state * device_state)1134 int hash_resume_state(struct hash_device_data *device_data,
1135 const struct hash_state *device_state)
1136 {
1137 u32 temp_cr;
1138 s32 count;
1139 int hash_mode = HASH_OPER_MODE_HASH;
1140
1141 if (NULL == device_state) {
1142 dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
1143 __func__);
1144 return -EPERM;
1145 }
1146
1147 /* Check correctness of index and length members */
1148 if (device_state->index > HASH_BLOCK_SIZE ||
1149 (device_state->length.low_word % HASH_BLOCK_SIZE) != 0) {
1150 dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
1151 __func__);
1152 return -EPERM;
1153 }
1154
1155 /*
1156 * INIT bit. Set this bit to 0b1 to reset the HASH processor core and
1157 * prepare the initialize the HASH accelerator to compute the message
1158 * digest of a new message.
1159 */
1160 HASH_INITIALIZE;
1161
1162 temp_cr = device_state->temp_cr;
1163 writel_relaxed(temp_cr & HASH_CR_RESUME_MASK, &device_data->base->cr);
1164
1165 if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK)
1166 hash_mode = HASH_OPER_MODE_HMAC;
1167 else
1168 hash_mode = HASH_OPER_MODE_HASH;
1169
1170 for (count = 0; count < HASH_CSR_COUNT; count++) {
1171 if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH))
1172 break;
1173
1174 writel_relaxed(device_state->csr[count],
1175 &device_data->base->csrx[count]);
1176 }
1177
1178 writel_relaxed(device_state->csfull, &device_data->base->csfull);
1179 writel_relaxed(device_state->csdatain, &device_data->base->csdatain);
1180
1181 writel_relaxed(device_state->str_reg, &device_data->base->str);
1182 writel_relaxed(temp_cr, &device_data->base->cr);
1183
1184 return 0;
1185 }
1186
1187 /**
1188 * hash_save_state - Function that saves the state of hardware.
1189 * @device_data: Pointer to the device structure.
1190 * @device_state: The strucure where the hardware state should be saved.
1191 */
hash_save_state(struct hash_device_data * device_data,struct hash_state * device_state)1192 int hash_save_state(struct hash_device_data *device_data,
1193 struct hash_state *device_state)
1194 {
1195 u32 temp_cr;
1196 u32 count;
1197 int hash_mode = HASH_OPER_MODE_HASH;
1198
1199 if (NULL == device_state) {
1200 dev_err(device_data->dev, "%s: HASH_INVALID_PARAMETER!\n",
1201 __func__);
1202 return -ENOTSUPP;
1203 }
1204
1205 /* Write dummy value to force digest intermediate calculation. This
1206 * actually makes sure that there isn't any ongoing calculation in the
1207 * hardware.
1208 */
1209 while (readl(&device_data->base->str) & HASH_STR_DCAL_MASK)
1210 cpu_relax();
1211
1212 temp_cr = readl_relaxed(&device_data->base->cr);
1213
1214 device_state->str_reg = readl_relaxed(&device_data->base->str);
1215
1216 device_state->din_reg = readl_relaxed(&device_data->base->din);
1217
1218 if (readl(&device_data->base->cr) & HASH_CR_MODE_MASK)
1219 hash_mode = HASH_OPER_MODE_HMAC;
1220 else
1221 hash_mode = HASH_OPER_MODE_HASH;
1222
1223 for (count = 0; count < HASH_CSR_COUNT; count++) {
1224 if ((count >= 36) && (hash_mode == HASH_OPER_MODE_HASH))
1225 break;
1226
1227 device_state->csr[count] =
1228 readl_relaxed(&device_data->base->csrx[count]);
1229 }
1230
1231 device_state->csfull = readl_relaxed(&device_data->base->csfull);
1232 device_state->csdatain = readl_relaxed(&device_data->base->csdatain);
1233
1234 device_state->temp_cr = temp_cr;
1235
1236 return 0;
1237 }
1238
1239 /**
1240 * hash_check_hw - This routine checks for peripheral Ids and PCell Ids.
1241 * @device_data:
1242 *
1243 */
hash_check_hw(struct hash_device_data * device_data)1244 int hash_check_hw(struct hash_device_data *device_data)
1245 {
1246 /* Checking Peripheral Ids */
1247 if (HASH_P_ID0 == readl_relaxed(&device_data->base->periphid0) &&
1248 HASH_P_ID1 == readl_relaxed(&device_data->base->periphid1) &&
1249 HASH_P_ID2 == readl_relaxed(&device_data->base->periphid2) &&
1250 HASH_P_ID3 == readl_relaxed(&device_data->base->periphid3) &&
1251 HASH_CELL_ID0 == readl_relaxed(&device_data->base->cellid0) &&
1252 HASH_CELL_ID1 == readl_relaxed(&device_data->base->cellid1) &&
1253 HASH_CELL_ID2 == readl_relaxed(&device_data->base->cellid2) &&
1254 HASH_CELL_ID3 == readl_relaxed(&device_data->base->cellid3)) {
1255 return 0;
1256 }
1257
1258 dev_err(device_data->dev, "%s: HASH_UNSUPPORTED_HW!\n", __func__);
1259 return -ENOTSUPP;
1260 }
1261
1262 /**
1263 * hash_get_digest - Gets the digest.
1264 * @device_data: Pointer to the device structure.
1265 * @digest: User allocated byte array for the calculated digest.
1266 * @algorithm: The algorithm in use.
1267 */
hash_get_digest(struct hash_device_data * device_data,u8 * digest,int algorithm)1268 void hash_get_digest(struct hash_device_data *device_data,
1269 u8 *digest, int algorithm)
1270 {
1271 u32 temp_hx_val, count;
1272 int loop_ctr;
1273
1274 if (algorithm != HASH_ALGO_SHA1 && algorithm != HASH_ALGO_SHA256) {
1275 dev_err(device_data->dev, "%s: Incorrect algorithm %d\n",
1276 __func__, algorithm);
1277 return;
1278 }
1279
1280 if (algorithm == HASH_ALGO_SHA1)
1281 loop_ctr = SHA1_DIGEST_SIZE / sizeof(u32);
1282 else
1283 loop_ctr = SHA256_DIGEST_SIZE / sizeof(u32);
1284
1285 dev_dbg(device_data->dev, "%s: digest array:(0x%lx)\n",
1286 __func__, (unsigned long)digest);
1287
1288 /* Copy result into digest array */
1289 for (count = 0; count < loop_ctr; count++) {
1290 temp_hx_val = readl_relaxed(&device_data->base->hx[count]);
1291 digest[count * 4] = (u8) ((temp_hx_val >> 24) & 0xFF);
1292 digest[count * 4 + 1] = (u8) ((temp_hx_val >> 16) & 0xFF);
1293 digest[count * 4 + 2] = (u8) ((temp_hx_val >> 8) & 0xFF);
1294 digest[count * 4 + 3] = (u8) ((temp_hx_val >> 0) & 0xFF);
1295 }
1296 }
1297
1298 /**
1299 * ahash_update - The hash update function for SHA1/SHA2 (SHA256).
1300 * @req: The hash request for the job.
1301 */
ahash_update(struct ahash_request * req)1302 static int ahash_update(struct ahash_request *req)
1303 {
1304 int ret = 0;
1305 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
1306
1307 if (hash_mode != HASH_MODE_DMA || !req_ctx->dma_mode)
1308 ret = hash_hw_update(req);
1309 /* Skip update for DMA, all data will be passed to DMA in final */
1310
1311 if (ret) {
1312 pr_err("%s: hash_hw_update() failed!\n", __func__);
1313 }
1314
1315 return ret;
1316 }
1317
1318 /**
1319 * ahash_final - The hash final function for SHA1/SHA2 (SHA256).
1320 * @req: The hash request for the job.
1321 */
ahash_final(struct ahash_request * req)1322 static int ahash_final(struct ahash_request *req)
1323 {
1324 int ret = 0;
1325 struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
1326
1327 pr_debug("%s: data size: %d\n", __func__, req->nbytes);
1328
1329 if ((hash_mode == HASH_MODE_DMA) && req_ctx->dma_mode)
1330 ret = hash_dma_final(req);
1331 else
1332 ret = hash_hw_final(req);
1333
1334 if (ret) {
1335 pr_err("%s: hash_hw/dma_final() failed\n", __func__);
1336 }
1337
1338 return ret;
1339 }
1340
hash_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen,int alg)1341 static int hash_setkey(struct crypto_ahash *tfm,
1342 const u8 *key, unsigned int keylen, int alg)
1343 {
1344 int ret = 0;
1345 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1346
1347 /**
1348 * Freed in final.
1349 */
1350 ctx->key = kmemdup(key, keylen, GFP_KERNEL);
1351 if (!ctx->key) {
1352 pr_err("%s: Failed to allocate ctx->key for %d\n",
1353 __func__, alg);
1354 return -ENOMEM;
1355 }
1356 ctx->keylen = keylen;
1357
1358 return ret;
1359 }
1360
ahash_sha1_init(struct ahash_request * req)1361 static int ahash_sha1_init(struct ahash_request *req)
1362 {
1363 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1364 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1365
1366 ctx->config.data_format = HASH_DATA_8_BITS;
1367 ctx->config.algorithm = HASH_ALGO_SHA1;
1368 ctx->config.oper_mode = HASH_OPER_MODE_HASH;
1369 ctx->digestsize = SHA1_DIGEST_SIZE;
1370
1371 return ux500_hash_init(req);
1372 }
1373
ahash_sha256_init(struct ahash_request * req)1374 static int ahash_sha256_init(struct ahash_request *req)
1375 {
1376 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1377 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1378
1379 ctx->config.data_format = HASH_DATA_8_BITS;
1380 ctx->config.algorithm = HASH_ALGO_SHA256;
1381 ctx->config.oper_mode = HASH_OPER_MODE_HASH;
1382 ctx->digestsize = SHA256_DIGEST_SIZE;
1383
1384 return ux500_hash_init(req);
1385 }
1386
ahash_sha1_digest(struct ahash_request * req)1387 static int ahash_sha1_digest(struct ahash_request *req)
1388 {
1389 int ret2, ret1;
1390
1391 ret1 = ahash_sha1_init(req);
1392 if (ret1)
1393 goto out;
1394
1395 ret1 = ahash_update(req);
1396 ret2 = ahash_final(req);
1397
1398 out:
1399 return ret1 ? ret1 : ret2;
1400 }
1401
ahash_sha256_digest(struct ahash_request * req)1402 static int ahash_sha256_digest(struct ahash_request *req)
1403 {
1404 int ret2, ret1;
1405
1406 ret1 = ahash_sha256_init(req);
1407 if (ret1)
1408 goto out;
1409
1410 ret1 = ahash_update(req);
1411 ret2 = ahash_final(req);
1412
1413 out:
1414 return ret1 ? ret1 : ret2;
1415 }
1416
ahash_noimport(struct ahash_request * req,const void * in)1417 static int ahash_noimport(struct ahash_request *req, const void *in)
1418 {
1419 return -ENOSYS;
1420 }
1421
ahash_noexport(struct ahash_request * req,void * out)1422 static int ahash_noexport(struct ahash_request *req, void *out)
1423 {
1424 return -ENOSYS;
1425 }
1426
hmac_sha1_init(struct ahash_request * req)1427 static int hmac_sha1_init(struct ahash_request *req)
1428 {
1429 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1430 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1431
1432 ctx->config.data_format = HASH_DATA_8_BITS;
1433 ctx->config.algorithm = HASH_ALGO_SHA1;
1434 ctx->config.oper_mode = HASH_OPER_MODE_HMAC;
1435 ctx->digestsize = SHA1_DIGEST_SIZE;
1436
1437 return ux500_hash_init(req);
1438 }
1439
hmac_sha256_init(struct ahash_request * req)1440 static int hmac_sha256_init(struct ahash_request *req)
1441 {
1442 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1443 struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
1444
1445 ctx->config.data_format = HASH_DATA_8_BITS;
1446 ctx->config.algorithm = HASH_ALGO_SHA256;
1447 ctx->config.oper_mode = HASH_OPER_MODE_HMAC;
1448 ctx->digestsize = SHA256_DIGEST_SIZE;
1449
1450 return ux500_hash_init(req);
1451 }
1452
hmac_sha1_digest(struct ahash_request * req)1453 static int hmac_sha1_digest(struct ahash_request *req)
1454 {
1455 int ret2, ret1;
1456
1457 ret1 = hmac_sha1_init(req);
1458 if (ret1)
1459 goto out;
1460
1461 ret1 = ahash_update(req);
1462 ret2 = ahash_final(req);
1463
1464 out:
1465 return ret1 ? ret1 : ret2;
1466 }
1467
hmac_sha256_digest(struct ahash_request * req)1468 static int hmac_sha256_digest(struct ahash_request *req)
1469 {
1470 int ret2, ret1;
1471
1472 ret1 = hmac_sha256_init(req);
1473 if (ret1)
1474 goto out;
1475
1476 ret1 = ahash_update(req);
1477 ret2 = ahash_final(req);
1478
1479 out:
1480 return ret1 ? ret1 : ret2;
1481 }
1482
hmac_sha1_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1483 static int hmac_sha1_setkey(struct crypto_ahash *tfm,
1484 const u8 *key, unsigned int keylen)
1485 {
1486 return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1);
1487 }
1488
hmac_sha256_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)1489 static int hmac_sha256_setkey(struct crypto_ahash *tfm,
1490 const u8 *key, unsigned int keylen)
1491 {
1492 return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256);
1493 }
1494
1495 struct hash_algo_template {
1496 struct hash_config conf;
1497 struct ahash_alg hash;
1498 };
1499
hash_cra_init(struct crypto_tfm * tfm)1500 static int hash_cra_init(struct crypto_tfm *tfm)
1501 {
1502 struct hash_ctx *ctx = crypto_tfm_ctx(tfm);
1503 struct crypto_alg *alg = tfm->__crt_alg;
1504 struct hash_algo_template *hash_alg;
1505
1506 hash_alg = container_of(__crypto_ahash_alg(alg),
1507 struct hash_algo_template,
1508 hash);
1509
1510 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1511 sizeof(struct hash_req_ctx));
1512
1513 ctx->config.data_format = HASH_DATA_8_BITS;
1514 ctx->config.algorithm = hash_alg->conf.algorithm;
1515 ctx->config.oper_mode = hash_alg->conf.oper_mode;
1516
1517 ctx->digestsize = hash_alg->hash.halg.digestsize;
1518
1519 return 0;
1520 }
1521
1522 static struct hash_algo_template hash_algs[] = {
1523 {
1524 .conf.algorithm = HASH_ALGO_SHA1,
1525 .conf.oper_mode = HASH_OPER_MODE_HASH,
1526 .hash = {
1527 .init = ux500_hash_init,
1528 .update = ahash_update,
1529 .final = ahash_final,
1530 .digest = ahash_sha1_digest,
1531 .export = ahash_noexport,
1532 .import = ahash_noimport,
1533 .halg.digestsize = SHA1_DIGEST_SIZE,
1534 .halg.statesize = sizeof(struct hash_ctx),
1535 .halg.base = {
1536 .cra_name = "sha1",
1537 .cra_driver_name = "sha1-ux500",
1538 .cra_flags = CRYPTO_ALG_ASYNC,
1539 .cra_blocksize = SHA1_BLOCK_SIZE,
1540 .cra_ctxsize = sizeof(struct hash_ctx),
1541 .cra_init = hash_cra_init,
1542 .cra_module = THIS_MODULE,
1543 }
1544 }
1545 },
1546 {
1547 .conf.algorithm = HASH_ALGO_SHA256,
1548 .conf.oper_mode = HASH_OPER_MODE_HASH,
1549 .hash = {
1550 .init = ux500_hash_init,
1551 .update = ahash_update,
1552 .final = ahash_final,
1553 .digest = ahash_sha256_digest,
1554 .export = ahash_noexport,
1555 .import = ahash_noimport,
1556 .halg.digestsize = SHA256_DIGEST_SIZE,
1557 .halg.statesize = sizeof(struct hash_ctx),
1558 .halg.base = {
1559 .cra_name = "sha256",
1560 .cra_driver_name = "sha256-ux500",
1561 .cra_flags = CRYPTO_ALG_ASYNC,
1562 .cra_blocksize = SHA256_BLOCK_SIZE,
1563 .cra_ctxsize = sizeof(struct hash_ctx),
1564 .cra_init = hash_cra_init,
1565 .cra_module = THIS_MODULE,
1566 }
1567 }
1568 },
1569 {
1570 .conf.algorithm = HASH_ALGO_SHA1,
1571 .conf.oper_mode = HASH_OPER_MODE_HMAC,
1572 .hash = {
1573 .init = ux500_hash_init,
1574 .update = ahash_update,
1575 .final = ahash_final,
1576 .digest = hmac_sha1_digest,
1577 .setkey = hmac_sha1_setkey,
1578 .export = ahash_noexport,
1579 .import = ahash_noimport,
1580 .halg.digestsize = SHA1_DIGEST_SIZE,
1581 .halg.statesize = sizeof(struct hash_ctx),
1582 .halg.base = {
1583 .cra_name = "hmac(sha1)",
1584 .cra_driver_name = "hmac-sha1-ux500",
1585 .cra_flags = CRYPTO_ALG_ASYNC,
1586 .cra_blocksize = SHA1_BLOCK_SIZE,
1587 .cra_ctxsize = sizeof(struct hash_ctx),
1588 .cra_init = hash_cra_init,
1589 .cra_module = THIS_MODULE,
1590 }
1591 }
1592 },
1593 {
1594 .conf.algorithm = HASH_ALGO_SHA256,
1595 .conf.oper_mode = HASH_OPER_MODE_HMAC,
1596 .hash = {
1597 .init = ux500_hash_init,
1598 .update = ahash_update,
1599 .final = ahash_final,
1600 .digest = hmac_sha256_digest,
1601 .setkey = hmac_sha256_setkey,
1602 .export = ahash_noexport,
1603 .import = ahash_noimport,
1604 .halg.digestsize = SHA256_DIGEST_SIZE,
1605 .halg.statesize = sizeof(struct hash_ctx),
1606 .halg.base = {
1607 .cra_name = "hmac(sha256)",
1608 .cra_driver_name = "hmac-sha256-ux500",
1609 .cra_flags = CRYPTO_ALG_ASYNC,
1610 .cra_blocksize = SHA256_BLOCK_SIZE,
1611 .cra_ctxsize = sizeof(struct hash_ctx),
1612 .cra_init = hash_cra_init,
1613 .cra_module = THIS_MODULE,
1614 }
1615 }
1616 }
1617 };
1618
ahash_algs_register_all(struct hash_device_data * device_data)1619 static int ahash_algs_register_all(struct hash_device_data *device_data)
1620 {
1621 int ret;
1622 int i;
1623 int count;
1624
1625 for (i = 0; i < ARRAY_SIZE(hash_algs); i++) {
1626 ret = crypto_register_ahash(&hash_algs[i].hash);
1627 if (ret) {
1628 count = i;
1629 dev_err(device_data->dev, "%s: alg registration failed\n",
1630 hash_algs[i].hash.halg.base.cra_driver_name);
1631 goto unreg;
1632 }
1633 }
1634 return 0;
1635 unreg:
1636 for (i = 0; i < count; i++)
1637 crypto_unregister_ahash(&hash_algs[i].hash);
1638 return ret;
1639 }
1640
ahash_algs_unregister_all(struct hash_device_data * device_data)1641 static void ahash_algs_unregister_all(struct hash_device_data *device_data)
1642 {
1643 int i;
1644
1645 for (i = 0; i < ARRAY_SIZE(hash_algs); i++)
1646 crypto_unregister_ahash(&hash_algs[i].hash);
1647 }
1648
1649 /**
1650 * ux500_hash_probe - Function that probes the hash hardware.
1651 * @pdev: The platform device.
1652 */
ux500_hash_probe(struct platform_device * pdev)1653 static int ux500_hash_probe(struct platform_device *pdev)
1654 {
1655 int ret = 0;
1656 struct resource *res = NULL;
1657 struct hash_device_data *device_data;
1658 struct device *dev = &pdev->dev;
1659
1660 device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC);
1661 if (!device_data) {
1662 ret = -ENOMEM;
1663 goto out;
1664 }
1665
1666 device_data->dev = dev;
1667 device_data->current_ctx = NULL;
1668
1669 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1670 if (!res) {
1671 dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__);
1672 ret = -ENODEV;
1673 goto out;
1674 }
1675
1676 device_data->phybase = res->start;
1677 device_data->base = devm_ioremap_resource(dev, res);
1678 if (IS_ERR(device_data->base)) {
1679 ret = PTR_ERR(device_data->base);
1680 goto out;
1681 }
1682 spin_lock_init(&device_data->ctx_lock);
1683 spin_lock_init(&device_data->power_state_lock);
1684
1685 /* Enable power for HASH1 hardware block */
1686 device_data->regulator = regulator_get(dev, "v-ape");
1687 if (IS_ERR(device_data->regulator)) {
1688 dev_err(dev, "%s: regulator_get() failed!\n", __func__);
1689 ret = PTR_ERR(device_data->regulator);
1690 device_data->regulator = NULL;
1691 goto out;
1692 }
1693
1694 /* Enable the clock for HASH1 hardware block */
1695 device_data->clk = devm_clk_get(dev, NULL);
1696 if (IS_ERR(device_data->clk)) {
1697 dev_err(dev, "%s: clk_get() failed!\n", __func__);
1698 ret = PTR_ERR(device_data->clk);
1699 goto out_regulator;
1700 }
1701
1702 ret = clk_prepare(device_data->clk);
1703 if (ret) {
1704 dev_err(dev, "%s: clk_prepare() failed!\n", __func__);
1705 goto out_regulator;
1706 }
1707
1708 /* Enable device power (and clock) */
1709 ret = hash_enable_power(device_data, false);
1710 if (ret) {
1711 dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
1712 goto out_clk_unprepare;
1713 }
1714
1715 ret = hash_check_hw(device_data);
1716 if (ret) {
1717 dev_err(dev, "%s: hash_check_hw() failed!\n", __func__);
1718 goto out_power;
1719 }
1720
1721 if (hash_mode == HASH_MODE_DMA)
1722 hash_dma_setup_channel(device_data, dev);
1723
1724 platform_set_drvdata(pdev, device_data);
1725
1726 /* Put the new device into the device list... */
1727 klist_add_tail(&device_data->list_node, &driver_data.device_list);
1728 /* ... and signal that a new device is available. */
1729 up(&driver_data.device_allocation);
1730
1731 ret = ahash_algs_register_all(device_data);
1732 if (ret) {
1733 dev_err(dev, "%s: ahash_algs_register_all() failed!\n",
1734 __func__);
1735 goto out_power;
1736 }
1737
1738 dev_info(dev, "successfully registered\n");
1739 return 0;
1740
1741 out_power:
1742 hash_disable_power(device_data, false);
1743
1744 out_clk_unprepare:
1745 clk_unprepare(device_data->clk);
1746
1747 out_regulator:
1748 regulator_put(device_data->regulator);
1749
1750 out:
1751 return ret;
1752 }
1753
1754 /**
1755 * ux500_hash_remove - Function that removes the hash device from the platform.
1756 * @pdev: The platform device.
1757 */
ux500_hash_remove(struct platform_device * pdev)1758 static int ux500_hash_remove(struct platform_device *pdev)
1759 {
1760 struct hash_device_data *device_data;
1761 struct device *dev = &pdev->dev;
1762
1763 device_data = platform_get_drvdata(pdev);
1764 if (!device_data) {
1765 dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
1766 return -ENOMEM;
1767 }
1768
1769 /* Try to decrease the number of available devices. */
1770 if (down_trylock(&driver_data.device_allocation))
1771 return -EBUSY;
1772
1773 /* Check that the device is free */
1774 spin_lock(&device_data->ctx_lock);
1775 /* current_ctx allocates a device, NULL = unallocated */
1776 if (device_data->current_ctx) {
1777 /* The device is busy */
1778 spin_unlock(&device_data->ctx_lock);
1779 /* Return the device to the pool. */
1780 up(&driver_data.device_allocation);
1781 return -EBUSY;
1782 }
1783
1784 spin_unlock(&device_data->ctx_lock);
1785
1786 /* Remove the device from the list */
1787 if (klist_node_attached(&device_data->list_node))
1788 klist_remove(&device_data->list_node);
1789
1790 /* If this was the last device, remove the services */
1791 if (list_empty(&driver_data.device_list.k_list))
1792 ahash_algs_unregister_all(device_data);
1793
1794 if (hash_disable_power(device_data, false))
1795 dev_err(dev, "%s: hash_disable_power() failed\n",
1796 __func__);
1797
1798 clk_unprepare(device_data->clk);
1799 regulator_put(device_data->regulator);
1800
1801 return 0;
1802 }
1803
1804 /**
1805 * ux500_hash_shutdown - Function that shutdown the hash device.
1806 * @pdev: The platform device
1807 */
ux500_hash_shutdown(struct platform_device * pdev)1808 static void ux500_hash_shutdown(struct platform_device *pdev)
1809 {
1810 struct hash_device_data *device_data;
1811
1812 device_data = platform_get_drvdata(pdev);
1813 if (!device_data) {
1814 dev_err(&pdev->dev, "%s: platform_get_drvdata() failed!\n",
1815 __func__);
1816 return;
1817 }
1818
1819 /* Check that the device is free */
1820 spin_lock(&device_data->ctx_lock);
1821 /* current_ctx allocates a device, NULL = unallocated */
1822 if (!device_data->current_ctx) {
1823 if (down_trylock(&driver_data.device_allocation))
1824 dev_dbg(&pdev->dev, "%s: Cryp still in use! Shutting down anyway...\n",
1825 __func__);
1826 /**
1827 * (Allocate the device)
1828 * Need to set this to non-null (dummy) value,
1829 * to avoid usage if context switching.
1830 */
1831 device_data->current_ctx++;
1832 }
1833 spin_unlock(&device_data->ctx_lock);
1834
1835 /* Remove the device from the list */
1836 if (klist_node_attached(&device_data->list_node))
1837 klist_remove(&device_data->list_node);
1838
1839 /* If this was the last device, remove the services */
1840 if (list_empty(&driver_data.device_list.k_list))
1841 ahash_algs_unregister_all(device_data);
1842
1843 if (hash_disable_power(device_data, false))
1844 dev_err(&pdev->dev, "%s: hash_disable_power() failed\n",
1845 __func__);
1846 }
1847
1848 #ifdef CONFIG_PM_SLEEP
1849 /**
1850 * ux500_hash_suspend - Function that suspends the hash device.
1851 * @dev: Device to suspend.
1852 */
ux500_hash_suspend(struct device * dev)1853 static int ux500_hash_suspend(struct device *dev)
1854 {
1855 int ret;
1856 struct hash_device_data *device_data;
1857 struct hash_ctx *temp_ctx = NULL;
1858
1859 device_data = dev_get_drvdata(dev);
1860 if (!device_data) {
1861 dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
1862 return -ENOMEM;
1863 }
1864
1865 spin_lock(&device_data->ctx_lock);
1866 if (!device_data->current_ctx)
1867 device_data->current_ctx++;
1868 spin_unlock(&device_data->ctx_lock);
1869
1870 if (device_data->current_ctx == ++temp_ctx) {
1871 if (down_interruptible(&driver_data.device_allocation))
1872 dev_dbg(dev, "%s: down_interruptible() failed\n",
1873 __func__);
1874 ret = hash_disable_power(device_data, false);
1875
1876 } else {
1877 ret = hash_disable_power(device_data, true);
1878 }
1879
1880 if (ret)
1881 dev_err(dev, "%s: hash_disable_power()\n", __func__);
1882
1883 return ret;
1884 }
1885
1886 /**
1887 * ux500_hash_resume - Function that resume the hash device.
1888 * @dev: Device to resume.
1889 */
ux500_hash_resume(struct device * dev)1890 static int ux500_hash_resume(struct device *dev)
1891 {
1892 int ret = 0;
1893 struct hash_device_data *device_data;
1894 struct hash_ctx *temp_ctx = NULL;
1895
1896 device_data = dev_get_drvdata(dev);
1897 if (!device_data) {
1898 dev_err(dev, "%s: platform_get_drvdata() failed!\n", __func__);
1899 return -ENOMEM;
1900 }
1901
1902 spin_lock(&device_data->ctx_lock);
1903 if (device_data->current_ctx == ++temp_ctx)
1904 device_data->current_ctx = NULL;
1905 spin_unlock(&device_data->ctx_lock);
1906
1907 if (!device_data->current_ctx)
1908 up(&driver_data.device_allocation);
1909 else
1910 ret = hash_enable_power(device_data, true);
1911
1912 if (ret)
1913 dev_err(dev, "%s: hash_enable_power() failed!\n", __func__);
1914
1915 return ret;
1916 }
1917 #endif
1918
1919 static SIMPLE_DEV_PM_OPS(ux500_hash_pm, ux500_hash_suspend, ux500_hash_resume);
1920
1921 static const struct of_device_id ux500_hash_match[] = {
1922 { .compatible = "stericsson,ux500-hash" },
1923 { },
1924 };
1925 MODULE_DEVICE_TABLE(of, ux500_hash_match);
1926
1927 static struct platform_driver hash_driver = {
1928 .probe = ux500_hash_probe,
1929 .remove = ux500_hash_remove,
1930 .shutdown = ux500_hash_shutdown,
1931 .driver = {
1932 .name = "hash1",
1933 .of_match_table = ux500_hash_match,
1934 .pm = &ux500_hash_pm,
1935 }
1936 };
1937
1938 /**
1939 * ux500_hash_mod_init - The kernel module init function.
1940 */
ux500_hash_mod_init(void)1941 static int __init ux500_hash_mod_init(void)
1942 {
1943 klist_init(&driver_data.device_list, NULL, NULL);
1944 /* Initialize the semaphore to 0 devices (locked state) */
1945 sema_init(&driver_data.device_allocation, 0);
1946
1947 return platform_driver_register(&hash_driver);
1948 }
1949
1950 /**
1951 * ux500_hash_mod_fini - The kernel module exit function.
1952 */
ux500_hash_mod_fini(void)1953 static void __exit ux500_hash_mod_fini(void)
1954 {
1955 platform_driver_unregister(&hash_driver);
1956 }
1957
1958 module_init(ux500_hash_mod_init);
1959 module_exit(ux500_hash_mod_fini);
1960
1961 MODULE_DESCRIPTION("Driver for ST-Ericsson UX500 HASH engine.");
1962 MODULE_LICENSE("GPL");
1963
1964 MODULE_ALIAS_CRYPTO("sha1-all");
1965 MODULE_ALIAS_CRYPTO("sha256-all");
1966 MODULE_ALIAS_CRYPTO("hmac-sha1-all");
1967 MODULE_ALIAS_CRYPTO("hmac-sha256-all");
1968