1 /*-
2  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/kobj.h>
34 #include <sys/lock.h>
35 #include <sys/malloc.h>
36 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/rwlock.h>
39 #include <sys/smp.h>
40 
41 #include <blake2.h>
42 
43 #include <opencrypto/cryptodev.h>
44 #include <cryptodev_if.h>
45 
46 #if defined(__amd64__)
47 #include <machine/fpu.h>
48 #elif defined(__i386__)
49 #include <machine/npx.h>
50 #endif
51 
52 struct blake2_session {
53 	int algo;
54 	size_t klen;
55 	size_t mlen;
56 	uint8_t key[BLAKE2B_KEYBYTES];
57 };
58 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
59 
60 struct blake2_softc {
61 	bool	dying;
62 	int32_t cid;
63 	struct rwlock lock;
64 };
65 
66 static struct mtx_padalign *ctx_mtx;
67 static struct fpu_kern_ctx **ctx_fpu;
68 
69 #define ACQUIRE_CTX(i, ctx)					\
70 	do {							\
71 		(i) = PCPU_GET(cpuid);				\
72 		mtx_lock(&ctx_mtx[(i)]);			\
73 		(ctx) = ctx_fpu[(i)];				\
74 	} while (0)
75 #define RELEASE_CTX(i, ctx)					\
76 	do {							\
77 		mtx_unlock(&ctx_mtx[(i)]);			\
78 		(i) = -1;					\
79 		(ctx) = NULL;					\
80 	} while (0)
81 
82 static int blake2_newsession(device_t, crypto_session_t cses,
83     struct cryptoini *cri);
84 static int blake2_cipher_setup(struct blake2_session *ses,
85     struct cryptoini *authini);
86 static int blake2_cipher_process(struct blake2_session *ses,
87     struct cryptop *crp);
88 
89 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
90 
91 static void
92 blake2_identify(driver_t *drv, device_t parent)
93 {
94 
95 	/* NB: order 10 is so we get attached after h/w devices */
96 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
97 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
98 		panic("blaketwo: could not attach");
99 }
100 
101 static int
102 blake2_probe(device_t dev)
103 {
104 	device_set_desc(dev, "Blake2");
105 	return (0);
106 }
107 
108 static void
109 blake2_cleanctx(void)
110 {
111 	int i;
112 
113 	/* XXX - no way to return driverid */
114 	CPU_FOREACH(i) {
115 		if (ctx_fpu[i] != NULL) {
116 			mtx_destroy(&ctx_mtx[i]);
117 			fpu_kern_free_ctx(ctx_fpu[i]);
118 		}
119 		ctx_fpu[i] = NULL;
120 	}
121 	free(ctx_mtx, M_BLAKE2);
122 	ctx_mtx = NULL;
123 	free(ctx_fpu, M_BLAKE2);
124 	ctx_fpu = NULL;
125 }
126 
127 static int
128 blake2_attach(device_t dev)
129 {
130 	struct blake2_softc *sc;
131 	int i;
132 
133 	sc = device_get_softc(dev);
134 	sc->dying = false;
135 
136 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
137 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
138 	if (sc->cid < 0) {
139 		device_printf(dev, "Could not get crypto driver id.\n");
140 		return (ENOMEM);
141 	}
142 
143 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
144 	    M_WAITOK | M_ZERO);
145 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
146 	    M_WAITOK | M_ZERO);
147 
148 	CPU_FOREACH(i) {
149 		ctx_fpu[i] = fpu_kern_alloc_ctx(0);
150 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
151 	}
152 
153 	rw_init(&sc->lock, "blake2_lock");
154 
155 	crypto_register(sc->cid, CRYPTO_BLAKE2B, 0, 0);
156 	crypto_register(sc->cid, CRYPTO_BLAKE2S, 0, 0);
157 	return (0);
158 }
159 
160 static int
161 blake2_detach(device_t dev)
162 {
163 	struct blake2_softc *sc;
164 
165 	sc = device_get_softc(dev);
166 
167 	rw_wlock(&sc->lock);
168 	sc->dying = true;
169 	rw_wunlock(&sc->lock);
170 	crypto_unregister_all(sc->cid);
171 
172 	rw_destroy(&sc->lock);
173 
174 	blake2_cleanctx();
175 
176 	return (0);
177 }
178 
179 static int
180 blake2_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
181 {
182 	struct blake2_softc *sc;
183 	struct blake2_session *ses;
184 	struct cryptoini *authini;
185 	int error;
186 
187 	if (cri == NULL) {
188 		CRYPTDEB("no cri");
189 		return (EINVAL);
190 	}
191 
192 	sc = device_get_softc(dev);
193 
194 	authini = NULL;
195 	for (; cri != NULL; cri = cri->cri_next) {
196 		switch (cri->cri_alg) {
197 		case CRYPTO_BLAKE2B:
198 		case CRYPTO_BLAKE2S:
199 			if (authini != NULL) {
200 				CRYPTDEB("authini already set");
201 				return (EINVAL);
202 			}
203 			authini = cri;
204 			break;
205 		default:
206 			CRYPTDEB("unhandled algorithm");
207 			return (EINVAL);
208 		}
209 	}
210 	if (authini == NULL) {
211 		CRYPTDEB("no cipher");
212 		return (EINVAL);
213 	}
214 
215 	rw_wlock(&sc->lock);
216 	if (sc->dying) {
217 		rw_wunlock(&sc->lock);
218 		return (EINVAL);
219 	}
220 	rw_wunlock(&sc->lock);
221 
222 	ses = crypto_get_driver_session(cses);
223 
224 	ses->algo = authini->cri_alg;
225 	error = blake2_cipher_setup(ses, authini);
226 	if (error != 0) {
227 		CRYPTDEB("setup failed");
228 		return (error);
229 	}
230 
231 	return (0);
232 }
233 
234 static int
235 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
236 {
237 	struct blake2_session *ses;
238 	struct cryptodesc *crd, *authcrd;
239 	int error;
240 
241 	ses = NULL;
242 	error = 0;
243 	authcrd = NULL;
244 
245 	/* Sanity check. */
246 	if (crp == NULL)
247 		return (EINVAL);
248 
249 	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
250 		error = EINVAL;
251 		goto out;
252 	}
253 
254 	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
255 		switch (crd->crd_alg) {
256 		case CRYPTO_BLAKE2B:
257 		case CRYPTO_BLAKE2S:
258 			if (authcrd != NULL) {
259 				error = EINVAL;
260 				goto out;
261 			}
262 			authcrd = crd;
263 			break;
264 
265 		default:
266 			error = EINVAL;
267 			goto out;
268 		}
269 	}
270 
271 	ses = crypto_get_driver_session(crp->crp_session);
272 	error = blake2_cipher_process(ses, crp);
273 	if (error != 0)
274 		goto out;
275 
276 out:
277 	crp->crp_etype = error;
278 	crypto_done(crp);
279 	return (error);
280 }
281 
282 static device_method_t blake2_methods[] = {
283 	DEVMETHOD(device_identify, blake2_identify),
284 	DEVMETHOD(device_probe, blake2_probe),
285 	DEVMETHOD(device_attach, blake2_attach),
286 	DEVMETHOD(device_detach, blake2_detach),
287 
288 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
289 	DEVMETHOD(cryptodev_process, blake2_process),
290 
291 	DEVMETHOD_END
292 };
293 
294 static driver_t blake2_driver = {
295 	"blaketwo",
296 	blake2_methods,
297 	sizeof(struct blake2_softc),
298 };
299 static devclass_t blake2_devclass;
300 
301 DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
302 MODULE_VERSION(blake2, 1);
303 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
304 
305 static int
306 blake2_cipher_setup(struct blake2_session *ses, struct cryptoini *authini)
307 {
308 	int keylen;
309 
310 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
311 
312 	if (authini->cri_mlen < 0)
313 		return (EINVAL);
314 
315 	switch (ses->algo) {
316 	case CRYPTO_BLAKE2S:
317 		if (authini->cri_mlen != 0 &&
318 		    authini->cri_mlen > BLAKE2S_OUTBYTES)
319 			return (EINVAL);
320 		/* FALLTHROUGH */
321 	case CRYPTO_BLAKE2B:
322 		if (authini->cri_mlen != 0 &&
323 		    authini->cri_mlen > BLAKE2B_OUTBYTES)
324 			return (EINVAL);
325 
326 		if (authini->cri_klen % 8 != 0)
327 			return (EINVAL);
328 		keylen = authini->cri_klen / 8;
329 		if (keylen > sizeof(ses->key) ||
330 		    (ses->algo == CRYPTO_BLAKE2S && keylen > BLAKE2S_KEYBYTES))
331 			return (EINVAL);
332 		ses->klen = keylen;
333 		memcpy(ses->key, authini->cri_key, keylen);
334 		ses->mlen = authini->cri_mlen;
335 	}
336 	return (0);
337 }
338 
339 static int
340 blake2b_applicator(void *state, void *buf, u_int len)
341 {
342 	int rc;
343 
344 	rc = blake2b_update(state, buf, len);
345 	if (rc != 0)
346 		return (EINVAL);
347 	return (0);
348 }
349 
350 static int
351 blake2s_applicator(void *state, void *buf, u_int len)
352 {
353 	int rc;
354 
355 	rc = blake2s_update(state, buf, len);
356 	if (rc != 0)
357 		return (EINVAL);
358 	return (0);
359 }
360 
361 static int
362 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
363 {
364 	union {
365 		blake2b_state sb;
366 		blake2s_state ss;
367 	} bctx;
368 	char res[BLAKE2B_OUTBYTES];
369 	struct fpu_kern_ctx *ctx;
370 	int ctxidx;
371 	bool kt;
372 	struct cryptodesc *crd;
373 	int error, rc;
374 	size_t hashlen;
375 
376 	crd = crp->crp_desc;
377 	ctx = NULL;
378 	ctxidx = 0;
379 	error = EINVAL;
380 
381 	kt = is_fpu_kern_thread(0);
382 	if (!kt) {
383 		ACQUIRE_CTX(ctxidx, ctx);
384 		fpu_kern_enter(curthread, ctx,
385 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
386 	}
387 
388 	if (crd->crd_flags != 0)
389 		goto out;
390 
391 	switch (ses->algo) {
392 	case CRYPTO_BLAKE2B:
393 		if (ses->mlen != 0)
394 			hashlen = ses->mlen;
395 		else
396 			hashlen = BLAKE2B_OUTBYTES;
397 		if (ses->klen > 0)
398 			rc = blake2b_init_key(&bctx.sb, hashlen, ses->key, ses->klen);
399 		else
400 			rc = blake2b_init(&bctx.sb, hashlen);
401 		if (rc != 0)
402 			goto out;
403 		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
404 		    crd->crd_len, blake2b_applicator, &bctx.sb);
405 		if (error != 0)
406 			goto out;
407 		rc = blake2b_final(&bctx.sb, res, hashlen);
408 		if (rc != 0) {
409 			error = EINVAL;
410 			goto out;
411 		}
412 		break;
413 	case CRYPTO_BLAKE2S:
414 		if (ses->mlen != 0)
415 			hashlen = ses->mlen;
416 		else
417 			hashlen = BLAKE2S_OUTBYTES;
418 		if (ses->klen > 0)
419 			rc = blake2s_init_key(&bctx.ss, hashlen, ses->key, ses->klen);
420 		else
421 			rc = blake2s_init(&bctx.ss, hashlen);
422 		if (rc != 0)
423 			goto out;
424 		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
425 		    crd->crd_len, blake2s_applicator, &bctx.ss);
426 		if (error != 0)
427 			goto out;
428 		rc = blake2s_final(&bctx.ss, res, hashlen);
429 		if (rc != 0) {
430 			error = EINVAL;
431 			goto out;
432 		}
433 		break;
434 	default:
435 		panic("unreachable");
436 	}
437 
438 	crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
439 	    (void *)res);
440 
441 out:
442 	if (!kt) {
443 		fpu_kern_leave(curthread, ctx);
444 		RELEASE_CTX(ctxidx, ctx);
445 	}
446 	return (error);
447 }
448