156f6d5aeSCorentin Labbe // SPDX-License-Identifier: GPL-2.0
256f6d5aeSCorentin Labbe /*
356f6d5aeSCorentin Labbe * sun8i-ce-hash.c - hardware cryptographic offloader for
456f6d5aeSCorentin Labbe * Allwinner H3/A64/H5/H2+/H6/R40 SoC
556f6d5aeSCorentin Labbe *
656f6d5aeSCorentin Labbe * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
756f6d5aeSCorentin Labbe *
856f6d5aeSCorentin Labbe * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512.
956f6d5aeSCorentin Labbe *
1039db3f15SJonathan Corbet * You could find the datasheet in Documentation/arch/arm/sunxi.rst
1156f6d5aeSCorentin Labbe */
1207e34cd3SHerbert Xu
1356f6d5aeSCorentin Labbe #include <crypto/internal/hash.h>
1407e34cd3SHerbert Xu #include <crypto/md5.h>
15a24d22b2SEric Biggers #include <crypto/sha1.h>
16a24d22b2SEric Biggers #include <crypto/sha2.h>
1707e34cd3SHerbert Xu #include <linux/bottom_half.h>
1807e34cd3SHerbert Xu #include <linux/dma-mapping.h>
1907e34cd3SHerbert Xu #include <linux/kernel.h>
2007e34cd3SHerbert Xu #include <linux/pm_runtime.h>
2107e34cd3SHerbert Xu #include <linux/scatterlist.h>
2207e34cd3SHerbert Xu #include <linux/slab.h>
2307e34cd3SHerbert Xu #include <linux/string.h>
2456f6d5aeSCorentin Labbe #include "sun8i-ce.h"
2556f6d5aeSCorentin Labbe
sun8i_ce_hash_init_tfm(struct crypto_ahash * tfm)2607e34cd3SHerbert Xu int sun8i_ce_hash_init_tfm(struct crypto_ahash *tfm)
2756f6d5aeSCorentin Labbe {
2807e34cd3SHerbert Xu struct sun8i_ce_hash_tfm_ctx *op = crypto_ahash_ctx(tfm);
2907e34cd3SHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
3056f6d5aeSCorentin Labbe struct sun8i_ce_alg_template *algt;
3156f6d5aeSCorentin Labbe int err;
3256f6d5aeSCorentin Labbe
3307e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);
3456f6d5aeSCorentin Labbe op->ce = algt->ce;
3556f6d5aeSCorentin Labbe
3656f6d5aeSCorentin Labbe /* FALLBACK */
3707e34cd3SHerbert Xu op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0,
3856f6d5aeSCorentin Labbe CRYPTO_ALG_NEED_FALLBACK);
3956f6d5aeSCorentin Labbe if (IS_ERR(op->fallback_tfm)) {
4056f6d5aeSCorentin Labbe dev_err(algt->ce->dev, "Fallback driver could no be loaded\n");
4156f6d5aeSCorentin Labbe return PTR_ERR(op->fallback_tfm);
4256f6d5aeSCorentin Labbe }
4356f6d5aeSCorentin Labbe
4407e34cd3SHerbert Xu crypto_ahash_set_statesize(tfm,
4507e34cd3SHerbert Xu crypto_ahash_statesize(op->fallback_tfm));
4656f6d5aeSCorentin Labbe
4707e34cd3SHerbert Xu crypto_ahash_set_reqsize(tfm,
4856f6d5aeSCorentin Labbe sizeof(struct sun8i_ce_hash_reqctx) +
4956f6d5aeSCorentin Labbe crypto_ahash_reqsize(op->fallback_tfm));
5056f6d5aeSCorentin Labbe
5107e34cd3SHerbert Xu memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm),
52aff388f7SCorentin Labbe CRYPTO_MAX_ALG_NAME);
53aff388f7SCorentin Labbe
5456f6d5aeSCorentin Labbe err = pm_runtime_get_sync(op->ce->dev);
5556f6d5aeSCorentin Labbe if (err < 0)
5656f6d5aeSCorentin Labbe goto error_pm;
5756f6d5aeSCorentin Labbe return 0;
5856f6d5aeSCorentin Labbe error_pm:
5956f6d5aeSCorentin Labbe pm_runtime_put_noidle(op->ce->dev);
6056f6d5aeSCorentin Labbe crypto_free_ahash(op->fallback_tfm);
6156f6d5aeSCorentin Labbe return err;
6256f6d5aeSCorentin Labbe }
6356f6d5aeSCorentin Labbe
sun8i_ce_hash_exit_tfm(struct crypto_ahash * tfm)6407e34cd3SHerbert Xu void sun8i_ce_hash_exit_tfm(struct crypto_ahash *tfm)
6556f6d5aeSCorentin Labbe {
6607e34cd3SHerbert Xu struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
6756f6d5aeSCorentin Labbe
6856f6d5aeSCorentin Labbe crypto_free_ahash(tfmctx->fallback_tfm);
6956f6d5aeSCorentin Labbe pm_runtime_put_sync_suspend(tfmctx->ce->dev);
7056f6d5aeSCorentin Labbe }
7156f6d5aeSCorentin Labbe
sun8i_ce_hash_init(struct ahash_request * areq)7256f6d5aeSCorentin Labbe int sun8i_ce_hash_init(struct ahash_request *areq)
7356f6d5aeSCorentin Labbe {
7456f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
7556f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
7656f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
7756f6d5aeSCorentin Labbe
7856f6d5aeSCorentin Labbe memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx));
7956f6d5aeSCorentin Labbe
8056f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
8156f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
8256f6d5aeSCorentin Labbe
8356f6d5aeSCorentin Labbe return crypto_ahash_init(&rctx->fallback_req);
8456f6d5aeSCorentin Labbe }
8556f6d5aeSCorentin Labbe
sun8i_ce_hash_export(struct ahash_request * areq,void * out)8656f6d5aeSCorentin Labbe int sun8i_ce_hash_export(struct ahash_request *areq, void *out)
8756f6d5aeSCorentin Labbe {
8856f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
8956f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
9056f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
9156f6d5aeSCorentin Labbe
9256f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
9356f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
9456f6d5aeSCorentin Labbe
9556f6d5aeSCorentin Labbe return crypto_ahash_export(&rctx->fallback_req, out);
9656f6d5aeSCorentin Labbe }
9756f6d5aeSCorentin Labbe
sun8i_ce_hash_import(struct ahash_request * areq,const void * in)9856f6d5aeSCorentin Labbe int sun8i_ce_hash_import(struct ahash_request *areq, const void *in)
9956f6d5aeSCorentin Labbe {
10056f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
10156f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
10256f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
10356f6d5aeSCorentin Labbe
10456f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
10556f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
10656f6d5aeSCorentin Labbe
10756f6d5aeSCorentin Labbe return crypto_ahash_import(&rctx->fallback_req, in);
10856f6d5aeSCorentin Labbe }
10956f6d5aeSCorentin Labbe
sun8i_ce_hash_final(struct ahash_request * areq)11056f6d5aeSCorentin Labbe int sun8i_ce_hash_final(struct ahash_request *areq)
11156f6d5aeSCorentin Labbe {
11256f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
11356f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
11456f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
11556f6d5aeSCorentin Labbe
11656f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
11756f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
11856f6d5aeSCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
11956f6d5aeSCorentin Labbe rctx->fallback_req.result = areq->result;
12056f6d5aeSCorentin Labbe
12107e34cd3SHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
12207e34cd3SHerbert Xu struct sun8i_ce_alg_template *algt __maybe_unused;
12307e34cd3SHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
12407e34cd3SHerbert Xu
12507e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template,
12607e34cd3SHerbert Xu alg.hash.base);
12756f6d5aeSCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
12856f6d5aeSCorentin Labbe algt->stat_fb++;
12956f6d5aeSCorentin Labbe #endif
13007e34cd3SHerbert Xu }
13156f6d5aeSCorentin Labbe
13256f6d5aeSCorentin Labbe return crypto_ahash_final(&rctx->fallback_req);
13356f6d5aeSCorentin Labbe }
13456f6d5aeSCorentin Labbe
sun8i_ce_hash_update(struct ahash_request * areq)13556f6d5aeSCorentin Labbe int sun8i_ce_hash_update(struct ahash_request *areq)
13656f6d5aeSCorentin Labbe {
13756f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
13856f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
13956f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
14056f6d5aeSCorentin Labbe
14156f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
14256f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
14356f6d5aeSCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
14456f6d5aeSCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
14556f6d5aeSCorentin Labbe rctx->fallback_req.src = areq->src;
14656f6d5aeSCorentin Labbe
14756f6d5aeSCorentin Labbe return crypto_ahash_update(&rctx->fallback_req);
14856f6d5aeSCorentin Labbe }
14956f6d5aeSCorentin Labbe
sun8i_ce_hash_finup(struct ahash_request * areq)15056f6d5aeSCorentin Labbe int sun8i_ce_hash_finup(struct ahash_request *areq)
15156f6d5aeSCorentin Labbe {
15256f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
15356f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
15456f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
15556f6d5aeSCorentin Labbe
15656f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
15756f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
15856f6d5aeSCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
15956f6d5aeSCorentin Labbe
16056f6d5aeSCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
16156f6d5aeSCorentin Labbe rctx->fallback_req.src = areq->src;
16256f6d5aeSCorentin Labbe rctx->fallback_req.result = areq->result;
16307e34cd3SHerbert Xu
16407e34cd3SHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
16507e34cd3SHerbert Xu struct sun8i_ce_alg_template *algt __maybe_unused;
16607e34cd3SHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
16707e34cd3SHerbert Xu
16807e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template,
16907e34cd3SHerbert Xu alg.hash.base);
17056f6d5aeSCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
17156f6d5aeSCorentin Labbe algt->stat_fb++;
17256f6d5aeSCorentin Labbe #endif
17307e34cd3SHerbert Xu }
17456f6d5aeSCorentin Labbe
17556f6d5aeSCorentin Labbe return crypto_ahash_finup(&rctx->fallback_req);
17656f6d5aeSCorentin Labbe }
17756f6d5aeSCorentin Labbe
sun8i_ce_hash_digest_fb(struct ahash_request * areq)17856f6d5aeSCorentin Labbe static int sun8i_ce_hash_digest_fb(struct ahash_request *areq)
17956f6d5aeSCorentin Labbe {
18056f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
18156f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
18256f6d5aeSCorentin Labbe struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
18356f6d5aeSCorentin Labbe
18456f6d5aeSCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
18556f6d5aeSCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
18656f6d5aeSCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
18756f6d5aeSCorentin Labbe
18856f6d5aeSCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
18956f6d5aeSCorentin Labbe rctx->fallback_req.src = areq->src;
19056f6d5aeSCorentin Labbe rctx->fallback_req.result = areq->result;
19107e34cd3SHerbert Xu
19207e34cd3SHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
19307e34cd3SHerbert Xu struct sun8i_ce_alg_template *algt __maybe_unused;
19407e34cd3SHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
19507e34cd3SHerbert Xu
19607e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template,
19707e34cd3SHerbert Xu alg.hash.base);
19856f6d5aeSCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
19956f6d5aeSCorentin Labbe algt->stat_fb++;
20056f6d5aeSCorentin Labbe #endif
20107e34cd3SHerbert Xu }
20256f6d5aeSCorentin Labbe
20356f6d5aeSCorentin Labbe return crypto_ahash_digest(&rctx->fallback_req);
20456f6d5aeSCorentin Labbe }
20556f6d5aeSCorentin Labbe
sun8i_ce_hash_need_fallback(struct ahash_request * areq)20656f6d5aeSCorentin Labbe static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq)
20756f6d5aeSCorentin Labbe {
208aff388f7SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
209aff388f7SCorentin Labbe struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
210aff388f7SCorentin Labbe struct sun8i_ce_alg_template *algt;
21156f6d5aeSCorentin Labbe struct scatterlist *sg;
21256f6d5aeSCorentin Labbe
21307e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);
214aff388f7SCorentin Labbe
215aff388f7SCorentin Labbe if (areq->nbytes == 0) {
216aff388f7SCorentin Labbe algt->stat_fb_len0++;
21756f6d5aeSCorentin Labbe return true;
218aff388f7SCorentin Labbe }
21956f6d5aeSCorentin Labbe /* we need to reserve one SG for padding one */
220aff388f7SCorentin Labbe if (sg_nents_for_len(areq->src, areq->nbytes) > MAX_SG - 1) {
221aff388f7SCorentin Labbe algt->stat_fb_maxsg++;
22256f6d5aeSCorentin Labbe return true;
223aff388f7SCorentin Labbe }
22456f6d5aeSCorentin Labbe sg = areq->src;
22556f6d5aeSCorentin Labbe while (sg) {
226aff388f7SCorentin Labbe if (sg->length % 4) {
227aff388f7SCorentin Labbe algt->stat_fb_srclen++;
22856f6d5aeSCorentin Labbe return true;
229aff388f7SCorentin Labbe }
230aff388f7SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
231aff388f7SCorentin Labbe algt->stat_fb_srcali++;
232aff388f7SCorentin Labbe return true;
233aff388f7SCorentin Labbe }
23456f6d5aeSCorentin Labbe sg = sg_next(sg);
23556f6d5aeSCorentin Labbe }
23656f6d5aeSCorentin Labbe return false;
23756f6d5aeSCorentin Labbe }
23856f6d5aeSCorentin Labbe
sun8i_ce_hash_digest(struct ahash_request * areq)23956f6d5aeSCorentin Labbe int sun8i_ce_hash_digest(struct ahash_request *areq)
24056f6d5aeSCorentin Labbe {
24156f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
24256f6d5aeSCorentin Labbe struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
24356f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
24456f6d5aeSCorentin Labbe struct sun8i_ce_alg_template *algt;
24556f6d5aeSCorentin Labbe struct sun8i_ce_dev *ce;
24656f6d5aeSCorentin Labbe struct crypto_engine *engine;
24756f6d5aeSCorentin Labbe struct scatterlist *sg;
24856f6d5aeSCorentin Labbe int nr_sgs, e, i;
24956f6d5aeSCorentin Labbe
25056f6d5aeSCorentin Labbe if (sun8i_ce_hash_need_fallback(areq))
25156f6d5aeSCorentin Labbe return sun8i_ce_hash_digest_fb(areq);
25256f6d5aeSCorentin Labbe
2536b8309faSCorentin Labbe nr_sgs = sg_nents_for_len(areq->src, areq->nbytes);
25456f6d5aeSCorentin Labbe if (nr_sgs > MAX_SG - 1)
25556f6d5aeSCorentin Labbe return sun8i_ce_hash_digest_fb(areq);
25656f6d5aeSCorentin Labbe
25756f6d5aeSCorentin Labbe for_each_sg(areq->src, sg, nr_sgs, i) {
25856f6d5aeSCorentin Labbe if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32)))
25956f6d5aeSCorentin Labbe return sun8i_ce_hash_digest_fb(areq);
26056f6d5aeSCorentin Labbe }
26156f6d5aeSCorentin Labbe
26207e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);
26356f6d5aeSCorentin Labbe ce = algt->ce;
26456f6d5aeSCorentin Labbe
26556f6d5aeSCorentin Labbe e = sun8i_ce_get_engine_number(ce);
26656f6d5aeSCorentin Labbe rctx->flow = e;
26756f6d5aeSCorentin Labbe engine = ce->chanlist[e].engine;
26856f6d5aeSCorentin Labbe
26956f6d5aeSCorentin Labbe return crypto_transfer_hash_request_to_engine(engine, areq);
27056f6d5aeSCorentin Labbe }
27156f6d5aeSCorentin Labbe
hash_pad(__le32 * buf,unsigned int bufsize,u64 padi,u64 byte_count,bool le,int bs)2728a1714adSCorentin Labbe static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs)
2738a1714adSCorentin Labbe {
2748a1714adSCorentin Labbe u64 fill, min_fill, j, k;
2758a1714adSCorentin Labbe __be64 *bebits;
2768a1714adSCorentin Labbe __le64 *lebits;
2778a1714adSCorentin Labbe
2788a1714adSCorentin Labbe j = padi;
2798a1714adSCorentin Labbe buf[j++] = cpu_to_le32(0x80);
2808a1714adSCorentin Labbe
2818a1714adSCorentin Labbe if (bs == 64) {
2828a1714adSCorentin Labbe fill = 64 - (byte_count % 64);
2838a1714adSCorentin Labbe min_fill = 2 * sizeof(u32) + sizeof(u32);
2848a1714adSCorentin Labbe } else {
2858a1714adSCorentin Labbe fill = 128 - (byte_count % 128);
2868a1714adSCorentin Labbe min_fill = 4 * sizeof(u32) + sizeof(u32);
2878a1714adSCorentin Labbe }
2888a1714adSCorentin Labbe
2898a1714adSCorentin Labbe if (fill < min_fill)
2908a1714adSCorentin Labbe fill += bs;
2918a1714adSCorentin Labbe
2928a1714adSCorentin Labbe k = j;
2938a1714adSCorentin Labbe j += (fill - min_fill) / sizeof(u32);
2948a1714adSCorentin Labbe if (j * 4 > bufsize) {
2958a1714adSCorentin Labbe pr_err("%s OVERFLOW %llu\n", __func__, j);
2968a1714adSCorentin Labbe return 0;
2978a1714adSCorentin Labbe }
2988a1714adSCorentin Labbe for (; k < j; k++)
2998a1714adSCorentin Labbe buf[k] = 0;
3008a1714adSCorentin Labbe
3018a1714adSCorentin Labbe if (le) {
3028a1714adSCorentin Labbe /* MD5 */
3038a1714adSCorentin Labbe lebits = (__le64 *)&buf[j];
3048a1714adSCorentin Labbe *lebits = cpu_to_le64(byte_count << 3);
3058a1714adSCorentin Labbe j += 2;
3068a1714adSCorentin Labbe } else {
3078a1714adSCorentin Labbe if (bs == 64) {
3088a1714adSCorentin Labbe /* sha1 sha224 sha256 */
3098a1714adSCorentin Labbe bebits = (__be64 *)&buf[j];
3108a1714adSCorentin Labbe *bebits = cpu_to_be64(byte_count << 3);
3118a1714adSCorentin Labbe j += 2;
3128a1714adSCorentin Labbe } else {
3138a1714adSCorentin Labbe /* sha384 sha512*/
3148a1714adSCorentin Labbe bebits = (__be64 *)&buf[j];
3158a1714adSCorentin Labbe *bebits = cpu_to_be64(byte_count >> 61);
3168a1714adSCorentin Labbe j += 2;
3178a1714adSCorentin Labbe bebits = (__be64 *)&buf[j];
3188a1714adSCorentin Labbe *bebits = cpu_to_be64(byte_count << 3);
3198a1714adSCorentin Labbe j += 2;
3208a1714adSCorentin Labbe }
3218a1714adSCorentin Labbe }
3228a1714adSCorentin Labbe if (j * 4 > bufsize) {
3238a1714adSCorentin Labbe pr_err("%s OVERFLOW %llu\n", __func__, j);
3248a1714adSCorentin Labbe return 0;
3258a1714adSCorentin Labbe }
3268a1714adSCorentin Labbe
3278a1714adSCorentin Labbe return j;
3288a1714adSCorentin Labbe }
3298a1714adSCorentin Labbe
sun8i_ce_hash_run(struct crypto_engine * engine,void * breq)33056f6d5aeSCorentin Labbe int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq)
33156f6d5aeSCorentin Labbe {
33256f6d5aeSCorentin Labbe struct ahash_request *areq = container_of(breq, struct ahash_request, base);
33356f6d5aeSCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
33456f6d5aeSCorentin Labbe struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
33556f6d5aeSCorentin Labbe struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
33656f6d5aeSCorentin Labbe struct sun8i_ce_alg_template *algt;
33756f6d5aeSCorentin Labbe struct sun8i_ce_dev *ce;
33856f6d5aeSCorentin Labbe struct sun8i_ce_flow *chan;
33956f6d5aeSCorentin Labbe struct ce_task *cet;
34056f6d5aeSCorentin Labbe struct scatterlist *sg;
34156f6d5aeSCorentin Labbe int nr_sgs, flow, err;
34256f6d5aeSCorentin Labbe unsigned int len;
34356f6d5aeSCorentin Labbe u32 common;
34456f6d5aeSCorentin Labbe u64 byte_count;
34556f6d5aeSCorentin Labbe __le32 *bf;
346732b7640SCorentin Labbe void *buf = NULL;
34756f6d5aeSCorentin Labbe int j, i, todo;
348732b7640SCorentin Labbe void *result = NULL;
34956f6d5aeSCorentin Labbe u64 bs;
35056f6d5aeSCorentin Labbe int digestsize;
35156f6d5aeSCorentin Labbe dma_addr_t addr_res, addr_pad;
3526b8309faSCorentin Labbe int ns = sg_nents_for_len(areq->src, areq->nbytes);
35356f6d5aeSCorentin Labbe
35407e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);
35556f6d5aeSCorentin Labbe ce = algt->ce;
35656f6d5aeSCorentin Labbe
35707e34cd3SHerbert Xu bs = algt->alg.hash.base.halg.base.cra_blocksize;
35807e34cd3SHerbert Xu digestsize = algt->alg.hash.base.halg.digestsize;
35956f6d5aeSCorentin Labbe if (digestsize == SHA224_DIGEST_SIZE)
36056f6d5aeSCorentin Labbe digestsize = SHA256_DIGEST_SIZE;
36156f6d5aeSCorentin Labbe if (digestsize == SHA384_DIGEST_SIZE)
36256f6d5aeSCorentin Labbe digestsize = SHA512_DIGEST_SIZE;
36356f6d5aeSCorentin Labbe
36456f6d5aeSCorentin Labbe /* the padding could be up to two block. */
365*1dc0c138SErick Archer buf = kcalloc(2, bs, GFP_KERNEL | GFP_DMA);
366732b7640SCorentin Labbe if (!buf) {
367732b7640SCorentin Labbe err = -ENOMEM;
368732b7640SCorentin Labbe goto theend;
369732b7640SCorentin Labbe }
37056f6d5aeSCorentin Labbe bf = (__le32 *)buf;
37156f6d5aeSCorentin Labbe
37256f6d5aeSCorentin Labbe result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA);
373732b7640SCorentin Labbe if (!result) {
374732b7640SCorentin Labbe err = -ENOMEM;
375732b7640SCorentin Labbe goto theend;
376732b7640SCorentin Labbe }
37756f6d5aeSCorentin Labbe
37856f6d5aeSCorentin Labbe flow = rctx->flow;
37956f6d5aeSCorentin Labbe chan = &ce->chanlist[flow];
38056f6d5aeSCorentin Labbe
38156f6d5aeSCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
38256f6d5aeSCorentin Labbe algt->stat_req++;
38356f6d5aeSCorentin Labbe #endif
38456f6d5aeSCorentin Labbe dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes);
38556f6d5aeSCorentin Labbe
38656f6d5aeSCorentin Labbe cet = chan->tl;
38756f6d5aeSCorentin Labbe memset(cet, 0, sizeof(struct ce_task));
38856f6d5aeSCorentin Labbe
38956f6d5aeSCorentin Labbe cet->t_id = cpu_to_le32(flow);
39056f6d5aeSCorentin Labbe common = ce->variant->alg_hash[algt->ce_algo_id];
39156f6d5aeSCorentin Labbe common |= CE_COMM_INT;
39256f6d5aeSCorentin Labbe cet->t_common_ctl = cpu_to_le32(common);
39356f6d5aeSCorentin Labbe
39456f6d5aeSCorentin Labbe cet->t_sym_ctl = 0;
39556f6d5aeSCorentin Labbe cet->t_asym_ctl = 0;
39656f6d5aeSCorentin Labbe
3976b8309faSCorentin Labbe nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
39856f6d5aeSCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
39956f6d5aeSCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
40056f6d5aeSCorentin Labbe err = -EINVAL;
40156f6d5aeSCorentin Labbe goto theend;
40256f6d5aeSCorentin Labbe }
40356f6d5aeSCorentin Labbe
40456f6d5aeSCorentin Labbe len = areq->nbytes;
40556f6d5aeSCorentin Labbe for_each_sg(areq->src, sg, nr_sgs, i) {
40656f6d5aeSCorentin Labbe cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg));
40756f6d5aeSCorentin Labbe todo = min(len, sg_dma_len(sg));
40856f6d5aeSCorentin Labbe cet->t_src[i].len = cpu_to_le32(todo / 4);
40956f6d5aeSCorentin Labbe len -= todo;
41056f6d5aeSCorentin Labbe }
41156f6d5aeSCorentin Labbe if (len > 0) {
41256f6d5aeSCorentin Labbe dev_err(ce->dev, "remaining len %d\n", len);
41356f6d5aeSCorentin Labbe err = -EINVAL;
41456f6d5aeSCorentin Labbe goto theend;
41556f6d5aeSCorentin Labbe }
41656f6d5aeSCorentin Labbe addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE);
41756f6d5aeSCorentin Labbe cet->t_dst[0].addr = cpu_to_le32(addr_res);
41856f6d5aeSCorentin Labbe cet->t_dst[0].len = cpu_to_le32(digestsize / 4);
41956f6d5aeSCorentin Labbe if (dma_mapping_error(ce->dev, addr_res)) {
42056f6d5aeSCorentin Labbe dev_err(ce->dev, "DMA map dest\n");
42156f6d5aeSCorentin Labbe err = -EINVAL;
42256f6d5aeSCorentin Labbe goto theend;
42356f6d5aeSCorentin Labbe }
42456f6d5aeSCorentin Labbe
42556f6d5aeSCorentin Labbe byte_count = areq->nbytes;
42656f6d5aeSCorentin Labbe j = 0;
42756f6d5aeSCorentin Labbe
42856f6d5aeSCorentin Labbe switch (algt->ce_algo_id) {
42956f6d5aeSCorentin Labbe case CE_ID_HASH_MD5:
4308a1714adSCorentin Labbe j = hash_pad(bf, 2 * bs, j, byte_count, true, bs);
43156f6d5aeSCorentin Labbe break;
43256f6d5aeSCorentin Labbe case CE_ID_HASH_SHA1:
43356f6d5aeSCorentin Labbe case CE_ID_HASH_SHA224:
43456f6d5aeSCorentin Labbe case CE_ID_HASH_SHA256:
4358a1714adSCorentin Labbe j = hash_pad(bf, 2 * bs, j, byte_count, false, bs);
43656f6d5aeSCorentin Labbe break;
43756f6d5aeSCorentin Labbe case CE_ID_HASH_SHA384:
43856f6d5aeSCorentin Labbe case CE_ID_HASH_SHA512:
4398a1714adSCorentin Labbe j = hash_pad(bf, 2 * bs, j, byte_count, false, bs);
44056f6d5aeSCorentin Labbe break;
44156f6d5aeSCorentin Labbe }
4428a1714adSCorentin Labbe if (!j) {
4438a1714adSCorentin Labbe err = -EINVAL;
4448a1714adSCorentin Labbe goto theend;
4458a1714adSCorentin Labbe }
44656f6d5aeSCorentin Labbe
44756f6d5aeSCorentin Labbe addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE);
44856f6d5aeSCorentin Labbe cet->t_src[i].addr = cpu_to_le32(addr_pad);
44956f6d5aeSCorentin Labbe cet->t_src[i].len = cpu_to_le32(j);
45056f6d5aeSCorentin Labbe if (dma_mapping_error(ce->dev, addr_pad)) {
45156f6d5aeSCorentin Labbe dev_err(ce->dev, "DMA error on padding SG\n");
45256f6d5aeSCorentin Labbe err = -EINVAL;
45356f6d5aeSCorentin Labbe goto theend;
45456f6d5aeSCorentin Labbe }
45556f6d5aeSCorentin Labbe
45656f6d5aeSCorentin Labbe if (ce->variant->hash_t_dlen_in_bits)
45756f6d5aeSCorentin Labbe cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8);
45856f6d5aeSCorentin Labbe else
45956f6d5aeSCorentin Labbe cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j);
46056f6d5aeSCorentin Labbe
46156f6d5aeSCorentin Labbe chan->timeout = areq->nbytes;
46256f6d5aeSCorentin Labbe
46307e34cd3SHerbert Xu err = sun8i_ce_run_task(ce, flow, crypto_ahash_alg_name(tfm));
46456f6d5aeSCorentin Labbe
46556f6d5aeSCorentin Labbe dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE);
4666b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
46756f6d5aeSCorentin Labbe dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE);
46856f6d5aeSCorentin Labbe
46956f6d5aeSCorentin Labbe
47007e34cd3SHerbert Xu memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize);
47156f6d5aeSCorentin Labbe theend:
472732b7640SCorentin Labbe kfree(buf);
473732b7640SCorentin Labbe kfree(result);
474f75a749bSCorentin Labbe local_bh_disable();
47556f6d5aeSCorentin Labbe crypto_finalize_hash_request(engine, breq, err);
476f75a749bSCorentin Labbe local_bh_enable();
47756f6d5aeSCorentin Labbe return 0;
47856f6d5aeSCorentin Labbe }
479