1d9b45418SCorentin Labbe // SPDX-License-Identifier: GPL-2.0
2d9b45418SCorentin Labbe /*
3d9b45418SCorentin Labbe * sun8i-ss-hash.c - hardware cryptographic offloader for
4d9b45418SCorentin Labbe * Allwinner A80/A83T SoC
5d9b45418SCorentin Labbe *
6d9b45418SCorentin Labbe * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
7d9b45418SCorentin Labbe *
8d9b45418SCorentin Labbe * This file add support for MD5 and SHA1/SHA224/SHA256.
9d9b45418SCorentin Labbe *
1039db3f15SJonathan Corbet * You could find the datasheet in Documentation/arch/arm/sunxi.rst
11d9b45418SCorentin Labbe */
124c19e8fbSHerbert Xu
13801b7d57SCorentin Labbe #include <crypto/hmac.h>
144c19e8fbSHerbert Xu #include <crypto/internal/hash.h>
154c19e8fbSHerbert Xu #include <crypto/md5.h>
16c35e523aSCorentin Labbe #include <crypto/scatterwalk.h>
17a24d22b2SEric Biggers #include <crypto/sha1.h>
18a24d22b2SEric Biggers #include <crypto/sha2.h>
194c19e8fbSHerbert Xu #include <linux/bottom_half.h>
204c19e8fbSHerbert Xu #include <linux/dma-mapping.h>
214c19e8fbSHerbert Xu #include <linux/err.h>
224c19e8fbSHerbert Xu #include <linux/kernel.h>
234c19e8fbSHerbert Xu #include <linux/pm_runtime.h>
244c19e8fbSHerbert Xu #include <linux/scatterlist.h>
254c19e8fbSHerbert Xu #include <linux/slab.h>
264c19e8fbSHerbert Xu #include <linux/string.h>
27d9b45418SCorentin Labbe #include "sun8i-ss.h"
28d9b45418SCorentin Labbe
sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx * tfmctx,const u8 * key,unsigned int keylen)29801b7d57SCorentin Labbe static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key,
30801b7d57SCorentin Labbe unsigned int keylen)
31801b7d57SCorentin Labbe {
32801b7d57SCorentin Labbe struct crypto_shash *xtfm;
33*84d02173SEric Biggers int ret;
34801b7d57SCorentin Labbe
35801b7d57SCorentin Labbe xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK);
367e8df1fcSPeng Wu if (IS_ERR(xtfm))
377e8df1fcSPeng Wu return PTR_ERR(xtfm);
38801b7d57SCorentin Labbe
39*84d02173SEric Biggers ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key);
40801b7d57SCorentin Labbe if (ret)
41*84d02173SEric Biggers dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret);
42*84d02173SEric Biggers
43801b7d57SCorentin Labbe crypto_free_shash(xtfm);
44801b7d57SCorentin Labbe return ret;
45801b7d57SCorentin Labbe }
46801b7d57SCorentin Labbe
sun8i_ss_hmac_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)47801b7d57SCorentin Labbe int sun8i_ss_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
48801b7d57SCorentin Labbe unsigned int keylen)
49801b7d57SCorentin Labbe {
50801b7d57SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(ahash);
51801b7d57SCorentin Labbe int digestsize, i;
52801b7d57SCorentin Labbe int bs = crypto_ahash_blocksize(ahash);
53801b7d57SCorentin Labbe int ret;
54801b7d57SCorentin Labbe
554c19e8fbSHerbert Xu digestsize = crypto_ahash_digestsize(ahash);
56801b7d57SCorentin Labbe
57801b7d57SCorentin Labbe if (keylen > bs) {
58801b7d57SCorentin Labbe ret = sun8i_ss_hashkey(tfmctx, key, keylen);
59801b7d57SCorentin Labbe if (ret)
60801b7d57SCorentin Labbe return ret;
61801b7d57SCorentin Labbe tfmctx->keylen = digestsize;
62801b7d57SCorentin Labbe } else {
63801b7d57SCorentin Labbe tfmctx->keylen = keylen;
64801b7d57SCorentin Labbe memcpy(tfmctx->key, key, keylen);
65801b7d57SCorentin Labbe }
66801b7d57SCorentin Labbe
6739a76cf1SHerbert Xu tfmctx->ipad = kzalloc(bs, GFP_KERNEL);
68801b7d57SCorentin Labbe if (!tfmctx->ipad)
69801b7d57SCorentin Labbe return -ENOMEM;
7039a76cf1SHerbert Xu tfmctx->opad = kzalloc(bs, GFP_KERNEL);
71801b7d57SCorentin Labbe if (!tfmctx->opad) {
72801b7d57SCorentin Labbe ret = -ENOMEM;
73801b7d57SCorentin Labbe goto err_opad;
74801b7d57SCorentin Labbe }
75801b7d57SCorentin Labbe
76801b7d57SCorentin Labbe memset(tfmctx->key + tfmctx->keylen, 0, bs - tfmctx->keylen);
77801b7d57SCorentin Labbe memcpy(tfmctx->ipad, tfmctx->key, tfmctx->keylen);
78801b7d57SCorentin Labbe memcpy(tfmctx->opad, tfmctx->key, tfmctx->keylen);
79801b7d57SCorentin Labbe for (i = 0; i < bs; i++) {
80801b7d57SCorentin Labbe tfmctx->ipad[i] ^= HMAC_IPAD_VALUE;
81801b7d57SCorentin Labbe tfmctx->opad[i] ^= HMAC_OPAD_VALUE;
82801b7d57SCorentin Labbe }
83801b7d57SCorentin Labbe
84801b7d57SCorentin Labbe ret = crypto_ahash_setkey(tfmctx->fallback_tfm, key, keylen);
85801b7d57SCorentin Labbe if (!ret)
86801b7d57SCorentin Labbe return 0;
87801b7d57SCorentin Labbe
88801b7d57SCorentin Labbe memzero_explicit(tfmctx->key, keylen);
89801b7d57SCorentin Labbe kfree_sensitive(tfmctx->opad);
90801b7d57SCorentin Labbe err_opad:
91801b7d57SCorentin Labbe kfree_sensitive(tfmctx->ipad);
92801b7d57SCorentin Labbe return ret;
93801b7d57SCorentin Labbe }
94801b7d57SCorentin Labbe
sun8i_ss_hash_init_tfm(struct crypto_ahash * tfm)954c19e8fbSHerbert Xu int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm)
96d9b45418SCorentin Labbe {
974c19e8fbSHerbert Xu struct sun8i_ss_hash_tfm_ctx *op = crypto_ahash_ctx(tfm);
984c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
99d9b45418SCorentin Labbe struct sun8i_ss_alg_template *algt;
100d9b45418SCorentin Labbe int err;
101d9b45418SCorentin Labbe
1024c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base);
103d9b45418SCorentin Labbe op->ss = algt->ss;
104d9b45418SCorentin Labbe
105d9b45418SCorentin Labbe /* FALLBACK */
1064c19e8fbSHerbert Xu op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0,
107d9b45418SCorentin Labbe CRYPTO_ALG_NEED_FALLBACK);
108d9b45418SCorentin Labbe if (IS_ERR(op->fallback_tfm)) {
109d9b45418SCorentin Labbe dev_err(algt->ss->dev, "Fallback driver could no be loaded\n");
110d9b45418SCorentin Labbe return PTR_ERR(op->fallback_tfm);
111d9b45418SCorentin Labbe }
112d9b45418SCorentin Labbe
1134c19e8fbSHerbert Xu crypto_ahash_set_statesize(tfm,
1144c19e8fbSHerbert Xu crypto_ahash_statesize(op->fallback_tfm));
115d9b45418SCorentin Labbe
1164c19e8fbSHerbert Xu crypto_ahash_set_reqsize(tfm,
117d9b45418SCorentin Labbe sizeof(struct sun8i_ss_hash_reqctx) +
118d9b45418SCorentin Labbe crypto_ahash_reqsize(op->fallback_tfm));
119d9b45418SCorentin Labbe
1204c19e8fbSHerbert Xu memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm),
1214c19e8fbSHerbert Xu CRYPTO_MAX_ALG_NAME);
122f95f61d0SCorentin Labbe
123d9b45418SCorentin Labbe err = pm_runtime_get_sync(op->ss->dev);
124d9b45418SCorentin Labbe if (err < 0)
125d9b45418SCorentin Labbe goto error_pm;
126d9b45418SCorentin Labbe return 0;
127d9b45418SCorentin Labbe error_pm:
128d9b45418SCorentin Labbe pm_runtime_put_noidle(op->ss->dev);
129d9b45418SCorentin Labbe crypto_free_ahash(op->fallback_tfm);
130d9b45418SCorentin Labbe return err;
131d9b45418SCorentin Labbe }
132d9b45418SCorentin Labbe
sun8i_ss_hash_exit_tfm(struct crypto_ahash * tfm)1334c19e8fbSHerbert Xu void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm)
134d9b45418SCorentin Labbe {
1354c19e8fbSHerbert Xu struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
136d9b45418SCorentin Labbe
137801b7d57SCorentin Labbe kfree_sensitive(tfmctx->ipad);
138801b7d57SCorentin Labbe kfree_sensitive(tfmctx->opad);
139801b7d57SCorentin Labbe
140d9b45418SCorentin Labbe crypto_free_ahash(tfmctx->fallback_tfm);
141d9b45418SCorentin Labbe pm_runtime_put_sync_suspend(tfmctx->ss->dev);
142d9b45418SCorentin Labbe }
143d9b45418SCorentin Labbe
sun8i_ss_hash_init(struct ahash_request * areq)144d9b45418SCorentin Labbe int sun8i_ss_hash_init(struct ahash_request *areq)
145d9b45418SCorentin Labbe {
146d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
147d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
148d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
149d9b45418SCorentin Labbe
150d9b45418SCorentin Labbe memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx));
151d9b45418SCorentin Labbe
152d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
153d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
154d9b45418SCorentin Labbe
155d9b45418SCorentin Labbe return crypto_ahash_init(&rctx->fallback_req);
156d9b45418SCorentin Labbe }
157d9b45418SCorentin Labbe
sun8i_ss_hash_export(struct ahash_request * areq,void * out)158d9b45418SCorentin Labbe int sun8i_ss_hash_export(struct ahash_request *areq, void *out)
159d9b45418SCorentin Labbe {
160d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
161d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
162d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
163d9b45418SCorentin Labbe
164d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
165d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
166d9b45418SCorentin Labbe
167d9b45418SCorentin Labbe return crypto_ahash_export(&rctx->fallback_req, out);
168d9b45418SCorentin Labbe }
169d9b45418SCorentin Labbe
sun8i_ss_hash_import(struct ahash_request * areq,const void * in)170d9b45418SCorentin Labbe int sun8i_ss_hash_import(struct ahash_request *areq, const void *in)
171d9b45418SCorentin Labbe {
172d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
173d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
174d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
175d9b45418SCorentin Labbe
176d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
177d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
178d9b45418SCorentin Labbe
179d9b45418SCorentin Labbe return crypto_ahash_import(&rctx->fallback_req, in);
180d9b45418SCorentin Labbe }
181d9b45418SCorentin Labbe
sun8i_ss_hash_final(struct ahash_request * areq)182d9b45418SCorentin Labbe int sun8i_ss_hash_final(struct ahash_request *areq)
183d9b45418SCorentin Labbe {
184d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
185d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
186d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
187d9b45418SCorentin Labbe
188d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
189d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
190d9b45418SCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
191d9b45418SCorentin Labbe rctx->fallback_req.result = areq->result;
192d9b45418SCorentin Labbe
1934c19e8fbSHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) {
1944c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
1954c19e8fbSHerbert Xu struct sun8i_ss_alg_template *algt __maybe_unused;
1964c19e8fbSHerbert Xu
1974c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template,
1984c19e8fbSHerbert Xu alg.hash.base);
1994c19e8fbSHerbert Xu
200d9b45418SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
201d9b45418SCorentin Labbe algt->stat_fb++;
202d9b45418SCorentin Labbe #endif
2034c19e8fbSHerbert Xu }
204d9b45418SCorentin Labbe
205d9b45418SCorentin Labbe return crypto_ahash_final(&rctx->fallback_req);
206d9b45418SCorentin Labbe }
207d9b45418SCorentin Labbe
sun8i_ss_hash_update(struct ahash_request * areq)208d9b45418SCorentin Labbe int sun8i_ss_hash_update(struct ahash_request *areq)
209d9b45418SCorentin Labbe {
210d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
211d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
212d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
213d9b45418SCorentin Labbe
214d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
215d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
216d9b45418SCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
217d9b45418SCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
218d9b45418SCorentin Labbe rctx->fallback_req.src = areq->src;
219d9b45418SCorentin Labbe
220d9b45418SCorentin Labbe return crypto_ahash_update(&rctx->fallback_req);
221d9b45418SCorentin Labbe }
222d9b45418SCorentin Labbe
sun8i_ss_hash_finup(struct ahash_request * areq)223d9b45418SCorentin Labbe int sun8i_ss_hash_finup(struct ahash_request *areq)
224d9b45418SCorentin Labbe {
225d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
226d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
227d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
228d9b45418SCorentin Labbe
229d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
230d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
231d9b45418SCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
232d9b45418SCorentin Labbe
233d9b45418SCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
234d9b45418SCorentin Labbe rctx->fallback_req.src = areq->src;
235d9b45418SCorentin Labbe rctx->fallback_req.result = areq->result;
2364c19e8fbSHerbert Xu
2374c19e8fbSHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) {
2384c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
2394c19e8fbSHerbert Xu struct sun8i_ss_alg_template *algt __maybe_unused;
2404c19e8fbSHerbert Xu
2414c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template,
2424c19e8fbSHerbert Xu alg.hash.base);
2434c19e8fbSHerbert Xu
244d9b45418SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
245d9b45418SCorentin Labbe algt->stat_fb++;
246d9b45418SCorentin Labbe #endif
2474c19e8fbSHerbert Xu }
248d9b45418SCorentin Labbe
249d9b45418SCorentin Labbe return crypto_ahash_finup(&rctx->fallback_req);
250d9b45418SCorentin Labbe }
251d9b45418SCorentin Labbe
sun8i_ss_hash_digest_fb(struct ahash_request * areq)252d9b45418SCorentin Labbe static int sun8i_ss_hash_digest_fb(struct ahash_request *areq)
253d9b45418SCorentin Labbe {
254d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
255d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
256d9b45418SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
257d9b45418SCorentin Labbe
258d9b45418SCorentin Labbe ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
259d9b45418SCorentin Labbe rctx->fallback_req.base.flags = areq->base.flags &
260d9b45418SCorentin Labbe CRYPTO_TFM_REQ_MAY_SLEEP;
261d9b45418SCorentin Labbe
262d9b45418SCorentin Labbe rctx->fallback_req.nbytes = areq->nbytes;
263d9b45418SCorentin Labbe rctx->fallback_req.src = areq->src;
264d9b45418SCorentin Labbe rctx->fallback_req.result = areq->result;
2654c19e8fbSHerbert Xu
2664c19e8fbSHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) {
2674c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
2684c19e8fbSHerbert Xu struct sun8i_ss_alg_template *algt __maybe_unused;
2694c19e8fbSHerbert Xu
2704c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template,
2714c19e8fbSHerbert Xu alg.hash.base);
2724c19e8fbSHerbert Xu
273d9b45418SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
274d9b45418SCorentin Labbe algt->stat_fb++;
275d9b45418SCorentin Labbe #endif
2764c19e8fbSHerbert Xu }
277d9b45418SCorentin Labbe
278d9b45418SCorentin Labbe return crypto_ahash_digest(&rctx->fallback_req);
279d9b45418SCorentin Labbe }
280d9b45418SCorentin Labbe
sun8i_ss_run_hash_task(struct sun8i_ss_dev * ss,struct sun8i_ss_hash_reqctx * rctx,const char * name)281d9b45418SCorentin Labbe static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss,
282d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx,
283d9b45418SCorentin Labbe const char *name)
284d9b45418SCorentin Labbe {
285d9b45418SCorentin Labbe int flow = rctx->flow;
286d9b45418SCorentin Labbe u32 v = SS_START;
287d9b45418SCorentin Labbe int i;
288d9b45418SCorentin Labbe
289d9b45418SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
290d9b45418SCorentin Labbe ss->flows[flow].stat_req++;
291d9b45418SCorentin Labbe #endif
292d9b45418SCorentin Labbe
293d9b45418SCorentin Labbe /* choose between stream0/stream1 */
294d9b45418SCorentin Labbe if (flow)
295d9b45418SCorentin Labbe v |= SS_FLOW1;
296d9b45418SCorentin Labbe else
297d9b45418SCorentin Labbe v |= SS_FLOW0;
298d9b45418SCorentin Labbe
299d9b45418SCorentin Labbe v |= rctx->method;
300d9b45418SCorentin Labbe
301d9b45418SCorentin Labbe for (i = 0; i < MAX_SG; i++) {
302d9b45418SCorentin Labbe if (!rctx->t_dst[i].addr)
303d9b45418SCorentin Labbe break;
304d9b45418SCorentin Labbe
305d9b45418SCorentin Labbe mutex_lock(&ss->mlock);
306d9b45418SCorentin Labbe if (i > 0) {
307d9b45418SCorentin Labbe v |= BIT(17);
308d9b45418SCorentin Labbe writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG);
309d9b45418SCorentin Labbe writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG);
310d9b45418SCorentin Labbe }
311d9b45418SCorentin Labbe
312d9b45418SCorentin Labbe dev_dbg(ss->dev,
313d9b45418SCorentin Labbe "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n",
314d9b45418SCorentin Labbe i, flow, name, v,
315d9b45418SCorentin Labbe rctx->t_src[i].len, rctx->t_dst[i].len,
316d9b45418SCorentin Labbe rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr);
317d9b45418SCorentin Labbe
318d9b45418SCorentin Labbe writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG);
319d9b45418SCorentin Labbe writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG);
320d9b45418SCorentin Labbe writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG);
321d9b45418SCorentin Labbe writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG);
322d9b45418SCorentin Labbe
323d9b45418SCorentin Labbe reinit_completion(&ss->flows[flow].complete);
324d9b45418SCorentin Labbe ss->flows[flow].status = 0;
325d9b45418SCorentin Labbe wmb();
326d9b45418SCorentin Labbe
327d9b45418SCorentin Labbe writel(v, ss->base + SS_CTL_REG);
328d9b45418SCorentin Labbe mutex_unlock(&ss->mlock);
329d9b45418SCorentin Labbe wait_for_completion_interruptible_timeout(&ss->flows[flow].complete,
330d9b45418SCorentin Labbe msecs_to_jiffies(2000));
331d9b45418SCorentin Labbe if (ss->flows[flow].status == 0) {
332d9b45418SCorentin Labbe dev_err(ss->dev, "DMA timeout for %s\n", name);
333d9b45418SCorentin Labbe return -EFAULT;
334d9b45418SCorentin Labbe }
335d9b45418SCorentin Labbe }
336d9b45418SCorentin Labbe
337d9b45418SCorentin Labbe return 0;
338d9b45418SCorentin Labbe }
339d9b45418SCorentin Labbe
sun8i_ss_hash_need_fallback(struct ahash_request * areq)340d9b45418SCorentin Labbe static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq)
341d9b45418SCorentin Labbe {
342f95f61d0SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3434c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
344f95f61d0SCorentin Labbe struct sun8i_ss_alg_template *algt;
345d9b45418SCorentin Labbe struct scatterlist *sg;
346d9b45418SCorentin Labbe
3474c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base);
348f95f61d0SCorentin Labbe
349f95f61d0SCorentin Labbe if (areq->nbytes == 0) {
350f95f61d0SCorentin Labbe algt->stat_fb_len++;
351d9b45418SCorentin Labbe return true;
352f95f61d0SCorentin Labbe }
353f95f61d0SCorentin Labbe
354f95f61d0SCorentin Labbe if (areq->nbytes >= MAX_PAD_SIZE - 64) {
355f95f61d0SCorentin Labbe algt->stat_fb_len++;
356c35e523aSCorentin Labbe return true;
357f95f61d0SCorentin Labbe }
358c35e523aSCorentin Labbe
359d9b45418SCorentin Labbe /* we need to reserve one SG for the padding one */
360f95f61d0SCorentin Labbe if (sg_nents(areq->src) > MAX_SG - 1) {
361f95f61d0SCorentin Labbe algt->stat_fb_sgnum++;
362d9b45418SCorentin Labbe return true;
363f95f61d0SCorentin Labbe }
364f95f61d0SCorentin Labbe
365d9b45418SCorentin Labbe sg = areq->src;
366d9b45418SCorentin Labbe while (sg) {
367d9b45418SCorentin Labbe /* SS can operate hash only on full block size
368d9b45418SCorentin Labbe * since SS support only MD5,sha1,sha224 and sha256, blocksize
369d9b45418SCorentin Labbe * is always 64
370d9b45418SCorentin Labbe */
371c35e523aSCorentin Labbe /* Only the last block could be bounced to the pad buffer */
372f95f61d0SCorentin Labbe if (sg->length % 64 && sg_next(sg)) {
373f95f61d0SCorentin Labbe algt->stat_fb_sglen++;
374c35e523aSCorentin Labbe return true;
375f95f61d0SCorentin Labbe }
376f95f61d0SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
377f95f61d0SCorentin Labbe algt->stat_fb_align++;
378c35e523aSCorentin Labbe return true;
379f95f61d0SCorentin Labbe }
380f95f61d0SCorentin Labbe if (sg->length % 4) {
381f95f61d0SCorentin Labbe algt->stat_fb_sglen++;
382d9b45418SCorentin Labbe return true;
383f95f61d0SCorentin Labbe }
384d9b45418SCorentin Labbe sg = sg_next(sg);
385d9b45418SCorentin Labbe }
386d9b45418SCorentin Labbe return false;
387d9b45418SCorentin Labbe }
388d9b45418SCorentin Labbe
sun8i_ss_hash_digest(struct ahash_request * areq)389d9b45418SCorentin Labbe int sun8i_ss_hash_digest(struct ahash_request *areq)
390d9b45418SCorentin Labbe {
391d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
392d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
3934c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
394d9b45418SCorentin Labbe struct sun8i_ss_alg_template *algt;
395d9b45418SCorentin Labbe struct sun8i_ss_dev *ss;
396d9b45418SCorentin Labbe struct crypto_engine *engine;
39746e2fcbcSCorentin Labbe int e;
398d9b45418SCorentin Labbe
399d9b45418SCorentin Labbe if (sun8i_ss_hash_need_fallback(areq))
400d9b45418SCorentin Labbe return sun8i_ss_hash_digest_fb(areq);
401d9b45418SCorentin Labbe
4024c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base);
403d9b45418SCorentin Labbe ss = algt->ss;
404d9b45418SCorentin Labbe
405d9b45418SCorentin Labbe e = sun8i_ss_get_engine_number(ss);
406d9b45418SCorentin Labbe rctx->flow = e;
407d9b45418SCorentin Labbe engine = ss->flows[e].engine;
408d9b45418SCorentin Labbe
409d9b45418SCorentin Labbe return crypto_transfer_hash_request_to_engine(engine, areq);
410d9b45418SCorentin Labbe }
411d9b45418SCorentin Labbe
hash_pad(__le32 * buf,unsigned int bufsize,u64 padi,u64 byte_count,bool le,int bs)412e76ee4dbSCorentin Labbe static u64 hash_pad(__le32 *buf, unsigned int bufsize, u64 padi, u64 byte_count, bool le, int bs)
413e76ee4dbSCorentin Labbe {
414e76ee4dbSCorentin Labbe u64 fill, min_fill, j, k;
415e76ee4dbSCorentin Labbe __be64 *bebits;
416e76ee4dbSCorentin Labbe __le64 *lebits;
417e76ee4dbSCorentin Labbe
418e76ee4dbSCorentin Labbe j = padi;
419e76ee4dbSCorentin Labbe buf[j++] = cpu_to_le32(0x80);
420e76ee4dbSCorentin Labbe
421e76ee4dbSCorentin Labbe if (bs == 64) {
422e76ee4dbSCorentin Labbe fill = 64 - (byte_count % 64);
423e76ee4dbSCorentin Labbe min_fill = 2 * sizeof(u32) + sizeof(u32);
424e76ee4dbSCorentin Labbe } else {
425e76ee4dbSCorentin Labbe fill = 128 - (byte_count % 128);
426e76ee4dbSCorentin Labbe min_fill = 4 * sizeof(u32) + sizeof(u32);
427e76ee4dbSCorentin Labbe }
428e76ee4dbSCorentin Labbe
429e76ee4dbSCorentin Labbe if (fill < min_fill)
430e76ee4dbSCorentin Labbe fill += bs;
431e76ee4dbSCorentin Labbe
432e76ee4dbSCorentin Labbe k = j;
433e76ee4dbSCorentin Labbe j += (fill - min_fill) / sizeof(u32);
434e76ee4dbSCorentin Labbe if (j * 4 > bufsize) {
435e76ee4dbSCorentin Labbe pr_err("%s OVERFLOW %llu\n", __func__, j);
436e76ee4dbSCorentin Labbe return 0;
437e76ee4dbSCorentin Labbe }
438e76ee4dbSCorentin Labbe for (; k < j; k++)
439e76ee4dbSCorentin Labbe buf[k] = 0;
440e76ee4dbSCorentin Labbe
441e76ee4dbSCorentin Labbe if (le) {
442e76ee4dbSCorentin Labbe /* MD5 */
443e76ee4dbSCorentin Labbe lebits = (__le64 *)&buf[j];
444e76ee4dbSCorentin Labbe *lebits = cpu_to_le64(byte_count << 3);
445e76ee4dbSCorentin Labbe j += 2;
446e76ee4dbSCorentin Labbe } else {
447e76ee4dbSCorentin Labbe if (bs == 64) {
448e76ee4dbSCorentin Labbe /* sha1 sha224 sha256 */
449e76ee4dbSCorentin Labbe bebits = (__be64 *)&buf[j];
450e76ee4dbSCorentin Labbe *bebits = cpu_to_be64(byte_count << 3);
451e76ee4dbSCorentin Labbe j += 2;
452e76ee4dbSCorentin Labbe } else {
453e76ee4dbSCorentin Labbe /* sha384 sha512*/
454e76ee4dbSCorentin Labbe bebits = (__be64 *)&buf[j];
455e76ee4dbSCorentin Labbe *bebits = cpu_to_be64(byte_count >> 61);
456e76ee4dbSCorentin Labbe j += 2;
457e76ee4dbSCorentin Labbe bebits = (__be64 *)&buf[j];
458e76ee4dbSCorentin Labbe *bebits = cpu_to_be64(byte_count << 3);
459e76ee4dbSCorentin Labbe j += 2;
460e76ee4dbSCorentin Labbe }
461e76ee4dbSCorentin Labbe }
462e76ee4dbSCorentin Labbe if (j * 4 > bufsize) {
463e76ee4dbSCorentin Labbe pr_err("%s OVERFLOW %llu\n", __func__, j);
464e76ee4dbSCorentin Labbe return 0;
465e76ee4dbSCorentin Labbe }
466e76ee4dbSCorentin Labbe
467e76ee4dbSCorentin Labbe return j;
468e76ee4dbSCorentin Labbe }
469e76ee4dbSCorentin Labbe
470d9b45418SCorentin Labbe /* sun8i_ss_hash_run - run an ahash request
471d9b45418SCorentin Labbe * Send the data of the request to the SS along with an extra SG with padding
472d9b45418SCorentin Labbe */
sun8i_ss_hash_run(struct crypto_engine * engine,void * breq)473d9b45418SCorentin Labbe int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq)
474d9b45418SCorentin Labbe {
475d9b45418SCorentin Labbe struct ahash_request *areq = container_of(breq, struct ahash_request, base);
476d9b45418SCorentin Labbe struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
477801b7d57SCorentin Labbe struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
478d9b45418SCorentin Labbe struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
4794c19e8fbSHerbert Xu struct ahash_alg *alg = crypto_ahash_alg(tfm);
480d9b45418SCorentin Labbe struct sun8i_ss_alg_template *algt;
481d9b45418SCorentin Labbe struct sun8i_ss_dev *ss;
482d9b45418SCorentin Labbe struct scatterlist *sg;
483801b7d57SCorentin Labbe int bs = crypto_ahash_blocksize(tfm);
484d9b45418SCorentin Labbe int nr_sgs, err, digestsize;
485d9b45418SCorentin Labbe unsigned int len;
486e76ee4dbSCorentin Labbe u64 byte_count;
487d9b45418SCorentin Labbe void *pad, *result;
488db0c62bcSCorentin Labbe int j, i, k, todo;
489801b7d57SCorentin Labbe dma_addr_t addr_res, addr_pad, addr_xpad;
490d9b45418SCorentin Labbe __le32 *bf;
491801b7d57SCorentin Labbe /* HMAC step:
492801b7d57SCorentin Labbe * 0: normal hashing
493801b7d57SCorentin Labbe * 1: IPAD
494801b7d57SCorentin Labbe * 2: OPAD
495801b7d57SCorentin Labbe */
496801b7d57SCorentin Labbe int hmac = 0;
497d9b45418SCorentin Labbe
4984c19e8fbSHerbert Xu algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base);
499d9b45418SCorentin Labbe ss = algt->ss;
500d9b45418SCorentin Labbe
5014c19e8fbSHerbert Xu digestsize = crypto_ahash_digestsize(tfm);
502d9b45418SCorentin Labbe if (digestsize == SHA224_DIGEST_SIZE)
503d9b45418SCorentin Labbe digestsize = SHA256_DIGEST_SIZE;
504d9b45418SCorentin Labbe
5058eec4563SCorentin Labbe result = ss->flows[rctx->flow].result;
5068eec4563SCorentin Labbe pad = ss->flows[rctx->flow].pad;
507d9b45418SCorentin Labbe bf = (__le32 *)pad;
508d9b45418SCorentin Labbe
509d9b45418SCorentin Labbe for (i = 0; i < MAX_SG; i++) {
510d9b45418SCorentin Labbe rctx->t_dst[i].addr = 0;
511d9b45418SCorentin Labbe rctx->t_dst[i].len = 0;
512d9b45418SCorentin Labbe }
513d9b45418SCorentin Labbe
514d9b45418SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
515d9b45418SCorentin Labbe algt->stat_req++;
516d9b45418SCorentin Labbe #endif
517d9b45418SCorentin Labbe
518d9b45418SCorentin Labbe rctx->method = ss->variant->alg_hash[algt->ss_algo_id];
519d9b45418SCorentin Labbe
520d9b45418SCorentin Labbe nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
521d9b45418SCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
522d9b45418SCorentin Labbe dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs);
523d9b45418SCorentin Labbe err = -EINVAL;
524d9b45418SCorentin Labbe goto theend;
525d9b45418SCorentin Labbe }
526d9b45418SCorentin Labbe
527d9b45418SCorentin Labbe addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE);
528d9b45418SCorentin Labbe if (dma_mapping_error(ss->dev, addr_res)) {
529d9b45418SCorentin Labbe dev_err(ss->dev, "DMA map dest\n");
530d9b45418SCorentin Labbe err = -EINVAL;
531801b7d57SCorentin Labbe goto err_dma_result;
532d9b45418SCorentin Labbe }
533d9b45418SCorentin Labbe
534c35e523aSCorentin Labbe j = 0;
535d9b45418SCorentin Labbe len = areq->nbytes;
536c149e476SCorentin Labbe sg = areq->src;
537c149e476SCorentin Labbe i = 0;
538c149e476SCorentin Labbe while (len > 0 && sg) {
539c149e476SCorentin Labbe if (sg_dma_len(sg) == 0) {
540c149e476SCorentin Labbe sg = sg_next(sg);
541c149e476SCorentin Labbe continue;
542c149e476SCorentin Labbe }
543d9b45418SCorentin Labbe todo = min(len, sg_dma_len(sg));
544c35e523aSCorentin Labbe /* only the last SG could be with a size not modulo64 */
545c35e523aSCorentin Labbe if (todo % 64 == 0) {
546c35e523aSCorentin Labbe rctx->t_src[i].addr = sg_dma_address(sg);
547d9b45418SCorentin Labbe rctx->t_src[i].len = todo / 4;
548d9b45418SCorentin Labbe rctx->t_dst[i].addr = addr_res;
549d9b45418SCorentin Labbe rctx->t_dst[i].len = digestsize / 4;
550c35e523aSCorentin Labbe len -= todo;
551c35e523aSCorentin Labbe } else {
552c35e523aSCorentin Labbe scatterwalk_map_and_copy(bf, sg, 0, todo, 0);
553c35e523aSCorentin Labbe j += todo / 4;
554c35e523aSCorentin Labbe len -= todo;
555c35e523aSCorentin Labbe }
556c149e476SCorentin Labbe sg = sg_next(sg);
557c149e476SCorentin Labbe i++;
558d9b45418SCorentin Labbe }
559d9b45418SCorentin Labbe if (len > 0) {
560d9b45418SCorentin Labbe dev_err(ss->dev, "remaining len %d\n", len);
561d9b45418SCorentin Labbe err = -EINVAL;
562d9b45418SCorentin Labbe goto theend;
563d9b45418SCorentin Labbe }
564d9b45418SCorentin Labbe
565c35e523aSCorentin Labbe if (j > 0)
566c35e523aSCorentin Labbe i--;
567c35e523aSCorentin Labbe
568801b7d57SCorentin Labbe retry:
569d9b45418SCorentin Labbe byte_count = areq->nbytes;
570801b7d57SCorentin Labbe if (tfmctx->keylen && hmac == 0) {
571801b7d57SCorentin Labbe hmac = 1;
572801b7d57SCorentin Labbe /* shift all SG one slot up, to free slot 0 for IPAD */
573801b7d57SCorentin Labbe for (k = 6; k >= 0; k--) {
574801b7d57SCorentin Labbe rctx->t_src[k + 1].addr = rctx->t_src[k].addr;
575801b7d57SCorentin Labbe rctx->t_src[k + 1].len = rctx->t_src[k].len;
576801b7d57SCorentin Labbe rctx->t_dst[k + 1].addr = rctx->t_dst[k].addr;
577801b7d57SCorentin Labbe rctx->t_dst[k + 1].len = rctx->t_dst[k].len;
578801b7d57SCorentin Labbe }
579801b7d57SCorentin Labbe addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE);
5806cb3f9b2SDan Carpenter err = dma_mapping_error(ss->dev, addr_xpad);
5816cb3f9b2SDan Carpenter if (err) {
582801b7d57SCorentin Labbe dev_err(ss->dev, "Fail to create DMA mapping of ipad\n");
583801b7d57SCorentin Labbe goto err_dma_xpad;
584801b7d57SCorentin Labbe }
585801b7d57SCorentin Labbe rctx->t_src[0].addr = addr_xpad;
586801b7d57SCorentin Labbe rctx->t_src[0].len = bs / 4;
587801b7d57SCorentin Labbe rctx->t_dst[0].addr = addr_res;
588801b7d57SCorentin Labbe rctx->t_dst[0].len = digestsize / 4;
589801b7d57SCorentin Labbe i++;
590801b7d57SCorentin Labbe byte_count = areq->nbytes + bs;
591801b7d57SCorentin Labbe }
592801b7d57SCorentin Labbe if (tfmctx->keylen && hmac == 2) {
593801b7d57SCorentin Labbe for (i = 0; i < MAX_SG; i++) {
594801b7d57SCorentin Labbe rctx->t_src[i].addr = 0;
595801b7d57SCorentin Labbe rctx->t_src[i].len = 0;
596801b7d57SCorentin Labbe rctx->t_dst[i].addr = 0;
597801b7d57SCorentin Labbe rctx->t_dst[i].len = 0;
598801b7d57SCorentin Labbe }
599801b7d57SCorentin Labbe
600801b7d57SCorentin Labbe addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE);
601801b7d57SCorentin Labbe if (dma_mapping_error(ss->dev, addr_res)) {
602801b7d57SCorentin Labbe dev_err(ss->dev, "Fail to create DMA mapping of result\n");
603801b7d57SCorentin Labbe err = -EINVAL;
604801b7d57SCorentin Labbe goto err_dma_result;
605801b7d57SCorentin Labbe }
606801b7d57SCorentin Labbe addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE);
6076cb3f9b2SDan Carpenter err = dma_mapping_error(ss->dev, addr_xpad);
6086cb3f9b2SDan Carpenter if (err) {
609801b7d57SCorentin Labbe dev_err(ss->dev, "Fail to create DMA mapping of opad\n");
610801b7d57SCorentin Labbe goto err_dma_xpad;
611801b7d57SCorentin Labbe }
612801b7d57SCorentin Labbe rctx->t_src[0].addr = addr_xpad;
613801b7d57SCorentin Labbe rctx->t_src[0].len = bs / 4;
614801b7d57SCorentin Labbe
615801b7d57SCorentin Labbe memcpy(bf, result, digestsize);
616801b7d57SCorentin Labbe j = digestsize / 4;
617801b7d57SCorentin Labbe i = 1;
618801b7d57SCorentin Labbe byte_count = digestsize + bs;
619801b7d57SCorentin Labbe
620801b7d57SCorentin Labbe rctx->t_dst[0].addr = addr_res;
621801b7d57SCorentin Labbe rctx->t_dst[0].len = digestsize / 4;
622801b7d57SCorentin Labbe }
623801b7d57SCorentin Labbe
624d9b45418SCorentin Labbe switch (algt->ss_algo_id) {
625d9b45418SCorentin Labbe case SS_ID_HASH_MD5:
626e76ee4dbSCorentin Labbe j = hash_pad(bf, 4096, j, byte_count, true, bs);
627d9b45418SCorentin Labbe break;
628d9b45418SCorentin Labbe case SS_ID_HASH_SHA1:
629d9b45418SCorentin Labbe case SS_ID_HASH_SHA224:
630d9b45418SCorentin Labbe case SS_ID_HASH_SHA256:
631e76ee4dbSCorentin Labbe j = hash_pad(bf, 4096, j, byte_count, false, bs);
632d9b45418SCorentin Labbe break;
633d9b45418SCorentin Labbe }
634e76ee4dbSCorentin Labbe if (!j) {
635e76ee4dbSCorentin Labbe err = -EINVAL;
636e76ee4dbSCorentin Labbe goto theend;
637e76ee4dbSCorentin Labbe }
638d9b45418SCorentin Labbe
639d9b45418SCorentin Labbe addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE);
640d9b45418SCorentin Labbe if (dma_mapping_error(ss->dev, addr_pad)) {
641d9b45418SCorentin Labbe dev_err(ss->dev, "DMA error on padding SG\n");
642d9b45418SCorentin Labbe err = -EINVAL;
643801b7d57SCorentin Labbe goto err_dma_pad;
644d9b45418SCorentin Labbe }
645d86e3f37SCorentin Labbe rctx->t_src[i].addr = addr_pad;
646d86e3f37SCorentin Labbe rctx->t_src[i].len = j;
647d86e3f37SCorentin Labbe rctx->t_dst[i].addr = addr_res;
648d86e3f37SCorentin Labbe rctx->t_dst[i].len = digestsize / 4;
649d9b45418SCorentin Labbe
650d9b45418SCorentin Labbe err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm));
651d9b45418SCorentin Labbe
652801b7d57SCorentin Labbe /*
653801b7d57SCorentin Labbe * mini helper for checking dma map/unmap
654801b7d57SCorentin Labbe * flow start for hmac = 0 (and HMAC = 1)
655801b7d57SCorentin Labbe * HMAC = 0
656801b7d57SCorentin Labbe * MAP src
657801b7d57SCorentin Labbe * MAP res
658801b7d57SCorentin Labbe *
659801b7d57SCorentin Labbe * retry:
660801b7d57SCorentin Labbe * if hmac then hmac = 1
661801b7d57SCorentin Labbe * MAP xpad (ipad)
662801b7d57SCorentin Labbe * if hmac == 2
663801b7d57SCorentin Labbe * MAP res
664801b7d57SCorentin Labbe * MAP xpad (opad)
665801b7d57SCorentin Labbe * MAP pad
666801b7d57SCorentin Labbe * ACTION!
667801b7d57SCorentin Labbe * UNMAP pad
668801b7d57SCorentin Labbe * if hmac
669801b7d57SCorentin Labbe * UNMAP xpad
670801b7d57SCorentin Labbe * UNMAP res
671801b7d57SCorentin Labbe * if hmac < 2
672801b7d57SCorentin Labbe * UNMAP SRC
673801b7d57SCorentin Labbe *
674801b7d57SCorentin Labbe * if hmac = 1 then hmac = 2 goto retry
675801b7d57SCorentin Labbe */
676801b7d57SCorentin Labbe
677d9b45418SCorentin Labbe dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE);
678801b7d57SCorentin Labbe
679801b7d57SCorentin Labbe err_dma_pad:
680801b7d57SCorentin Labbe if (hmac > 0)
681801b7d57SCorentin Labbe dma_unmap_single(ss->dev, addr_xpad, bs, DMA_TO_DEVICE);
682801b7d57SCorentin Labbe err_dma_xpad:
683801b7d57SCorentin Labbe dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE);
684801b7d57SCorentin Labbe err_dma_result:
685801b7d57SCorentin Labbe if (hmac < 2)
686884b93c5SXiang Chen dma_unmap_sg(ss->dev, areq->src, sg_nents(areq->src),
687884b93c5SXiang Chen DMA_TO_DEVICE);
688801b7d57SCorentin Labbe if (hmac == 1 && !err) {
689801b7d57SCorentin Labbe hmac = 2;
690801b7d57SCorentin Labbe goto retry;
691801b7d57SCorentin Labbe }
692d9b45418SCorentin Labbe
693801b7d57SCorentin Labbe if (!err)
6944c19e8fbSHerbert Xu memcpy(areq->result, result, crypto_ahash_digestsize(tfm));
695d9b45418SCorentin Labbe theend:
696b169b376SCorentin Labbe local_bh_disable();
697d9b45418SCorentin Labbe crypto_finalize_hash_request(engine, breq, err);
698b169b376SCorentin Labbe local_bh_enable();
699d9b45418SCorentin Labbe return 0;
700d9b45418SCorentin Labbe }
701