1 /*-
2  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/kobj.h>
34 #include <sys/lock.h>
35 #include <sys/module.h>
36 #include <sys/malloc.h>
37 #include <sys/rwlock.h>
38 #include <sys/smp.h>
39 
40 #include <blake2.h>
41 
42 #include <opencrypto/cryptodev.h>
43 #include <cryptodev_if.h>
44 
45 #if defined(__amd64__)
46 #include <machine/fpu.h>
47 #elif defined(__i386__)
48 #include <machine/npx.h>
49 #endif
50 
51 struct blake2_session {
52 	int algo;
53 	size_t klen;
54 	size_t mlen;
55 	uint8_t key[BLAKE2B_KEYBYTES];
56 };
57 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
58 
59 struct blake2_softc {
60 	bool	dying;
61 	int32_t cid;
62 	struct rwlock lock;
63 };
64 
65 static struct mtx_padalign *ctx_mtx;
66 static struct fpu_kern_ctx **ctx_fpu;
67 
68 #define ACQUIRE_CTX(i, ctx)					\
69 	do {							\
70 		(i) = PCPU_GET(cpuid);				\
71 		mtx_lock(&ctx_mtx[(i)]);			\
72 		(ctx) = ctx_fpu[(i)];				\
73 	} while (0)
74 #define RELEASE_CTX(i, ctx)					\
75 	do {							\
76 		mtx_unlock(&ctx_mtx[(i)]);			\
77 		(i) = -1;					\
78 		(ctx) = NULL;					\
79 	} while (0)
80 
81 static int blake2_newsession(device_t, crypto_session_t cses,
82     struct cryptoini *cri);
83 static int blake2_cipher_setup(struct blake2_session *ses,
84     struct cryptoini *authini);
85 static int blake2_cipher_process(struct blake2_session *ses,
86     struct cryptop *crp);
87 
88 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
89 
90 static void
91 blake2_identify(driver_t *drv, device_t parent)
92 {
93 
94 	/* NB: order 10 is so we get attached after h/w devices */
95 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
96 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
97 		panic("blaketwo: could not attach");
98 }
99 
100 static int
101 blake2_probe(device_t dev)
102 {
103 	device_set_desc(dev, "Blake2");
104 	return (0);
105 }
106 
107 static void
108 blake2_cleanctx(void)
109 {
110 	int i;
111 
112 	/* XXX - no way to return driverid */
113 	CPU_FOREACH(i) {
114 		if (ctx_fpu[i] != NULL) {
115 			mtx_destroy(&ctx_mtx[i]);
116 			fpu_kern_free_ctx(ctx_fpu[i]);
117 		}
118 		ctx_fpu[i] = NULL;
119 	}
120 	free(ctx_mtx, M_BLAKE2);
121 	ctx_mtx = NULL;
122 	free(ctx_fpu, M_BLAKE2);
123 	ctx_fpu = NULL;
124 }
125 
126 static int
127 blake2_attach(device_t dev)
128 {
129 	struct blake2_softc *sc;
130 	int i;
131 
132 	sc = device_get_softc(dev);
133 	sc->dying = false;
134 
135 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
136 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
137 	if (sc->cid < 0) {
138 		device_printf(dev, "Could not get crypto driver id.\n");
139 		return (ENOMEM);
140 	}
141 
142 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
143 	    M_WAITOK | M_ZERO);
144 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
145 	    M_WAITOK | M_ZERO);
146 
147 	CPU_FOREACH(i) {
148 		ctx_fpu[i] = fpu_kern_alloc_ctx(0);
149 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
150 	}
151 
152 	rw_init(&sc->lock, "blake2_lock");
153 
154 	crypto_register(sc->cid, CRYPTO_BLAKE2B, 0, 0);
155 	crypto_register(sc->cid, CRYPTO_BLAKE2S, 0, 0);
156 	return (0);
157 }
158 
159 static int
160 blake2_detach(device_t dev)
161 {
162 	struct blake2_softc *sc;
163 
164 	sc = device_get_softc(dev);
165 
166 	rw_wlock(&sc->lock);
167 	sc->dying = true;
168 	rw_wunlock(&sc->lock);
169 	crypto_unregister_all(sc->cid);
170 
171 	rw_destroy(&sc->lock);
172 
173 	blake2_cleanctx();
174 
175 	return (0);
176 }
177 
178 static int
179 blake2_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
180 {
181 	struct blake2_softc *sc;
182 	struct blake2_session *ses;
183 	struct cryptoini *authini;
184 	int error;
185 
186 	if (cri == NULL) {
187 		CRYPTDEB("no cri");
188 		return (EINVAL);
189 	}
190 
191 	sc = device_get_softc(dev);
192 
193 	authini = NULL;
194 	for (; cri != NULL; cri = cri->cri_next) {
195 		switch (cri->cri_alg) {
196 		case CRYPTO_BLAKE2B:
197 		case CRYPTO_BLAKE2S:
198 			if (authini != NULL) {
199 				CRYPTDEB("authini already set");
200 				return (EINVAL);
201 			}
202 			authini = cri;
203 			break;
204 		default:
205 			CRYPTDEB("unhandled algorithm");
206 			return (EINVAL);
207 		}
208 	}
209 	if (authini == NULL) {
210 		CRYPTDEB("no cipher");
211 		return (EINVAL);
212 	}
213 
214 	rw_wlock(&sc->lock);
215 	if (sc->dying) {
216 		rw_wunlock(&sc->lock);
217 		return (EINVAL);
218 	}
219 	rw_wunlock(&sc->lock);
220 
221 	ses = crypto_get_driver_session(cses);
222 
223 	ses->algo = authini->cri_alg;
224 	error = blake2_cipher_setup(ses, authini);
225 	if (error != 0) {
226 		CRYPTDEB("setup failed");
227 		return (error);
228 	}
229 
230 	return (0);
231 }
232 
233 static int
234 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
235 {
236 	struct blake2_session *ses;
237 	struct cryptodesc *crd, *authcrd;
238 	int error;
239 
240 	ses = NULL;
241 	error = 0;
242 	authcrd = NULL;
243 
244 	/* Sanity check. */
245 	if (crp == NULL)
246 		return (EINVAL);
247 
248 	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
249 		error = EINVAL;
250 		goto out;
251 	}
252 
253 	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
254 		switch (crd->crd_alg) {
255 		case CRYPTO_BLAKE2B:
256 		case CRYPTO_BLAKE2S:
257 			if (authcrd != NULL) {
258 				error = EINVAL;
259 				goto out;
260 			}
261 			authcrd = crd;
262 			break;
263 
264 		default:
265 			error = EINVAL;
266 			goto out;
267 		}
268 	}
269 
270 	ses = crypto_get_driver_session(crp->crp_session);
271 	error = blake2_cipher_process(ses, crp);
272 	if (error != 0)
273 		goto out;
274 
275 out:
276 	crp->crp_etype = error;
277 	crypto_done(crp);
278 	return (error);
279 }
280 
281 static device_method_t blake2_methods[] = {
282 	DEVMETHOD(device_identify, blake2_identify),
283 	DEVMETHOD(device_probe, blake2_probe),
284 	DEVMETHOD(device_attach, blake2_attach),
285 	DEVMETHOD(device_detach, blake2_detach),
286 
287 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
288 	DEVMETHOD(cryptodev_process, blake2_process),
289 
290 	DEVMETHOD_END
291 };
292 
293 static driver_t blake2_driver = {
294 	"blaketwo",
295 	blake2_methods,
296 	sizeof(struct blake2_softc),
297 };
298 static devclass_t blake2_devclass;
299 
300 DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
301 MODULE_VERSION(blake2, 1);
302 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
303 
304 static int
305 blake2_cipher_setup(struct blake2_session *ses, struct cryptoini *authini)
306 {
307 	int keylen;
308 
309 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
310 
311 	if (authini->cri_mlen < 0)
312 		return (EINVAL);
313 
314 	switch (ses->algo) {
315 	case CRYPTO_BLAKE2S:
316 		if (authini->cri_mlen != 0 &&
317 		    authini->cri_mlen > BLAKE2S_OUTBYTES)
318 			return (EINVAL);
319 		/* FALLTHROUGH */
320 	case CRYPTO_BLAKE2B:
321 		if (authini->cri_mlen != 0 &&
322 		    authini->cri_mlen > BLAKE2B_OUTBYTES)
323 			return (EINVAL);
324 
325 		if (authini->cri_klen % 8 != 0)
326 			return (EINVAL);
327 		keylen = authini->cri_klen / 8;
328 		if (keylen > sizeof(ses->key) ||
329 		    (ses->algo == CRYPTO_BLAKE2S && keylen > BLAKE2S_KEYBYTES))
330 			return (EINVAL);
331 		ses->klen = keylen;
332 		memcpy(ses->key, authini->cri_key, keylen);
333 		ses->mlen = authini->cri_mlen;
334 	}
335 	return (0);
336 }
337 
338 static int
339 blake2b_applicator(void *state, void *buf, u_int len)
340 {
341 	int rc;
342 
343 	rc = blake2b_update(state, buf, len);
344 	if (rc != 0)
345 		return (EINVAL);
346 	return (0);
347 }
348 
349 static int
350 blake2s_applicator(void *state, void *buf, u_int len)
351 {
352 	int rc;
353 
354 	rc = blake2s_update(state, buf, len);
355 	if (rc != 0)
356 		return (EINVAL);
357 	return (0);
358 }
359 
360 static int
361 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
362 {
363 	union {
364 		blake2b_state sb;
365 		blake2s_state ss;
366 	} bctx;
367 	char res[BLAKE2B_OUTBYTES];
368 	struct fpu_kern_ctx *ctx;
369 	int ctxidx;
370 	bool kt;
371 	struct cryptodesc *crd;
372 	int error, rc;
373 	size_t hashlen;
374 
375 	crd = crp->crp_desc;
376 	ctx = NULL;
377 	ctxidx = 0;
378 	error = EINVAL;
379 
380 	kt = is_fpu_kern_thread(0);
381 	if (!kt) {
382 		ACQUIRE_CTX(ctxidx, ctx);
383 		fpu_kern_enter(curthread, ctx,
384 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
385 	}
386 
387 	if (crd->crd_flags != 0)
388 		goto out;
389 
390 	switch (ses->algo) {
391 	case CRYPTO_BLAKE2B:
392 		if (ses->mlen != 0)
393 			hashlen = ses->mlen;
394 		else
395 			hashlen = BLAKE2B_OUTBYTES;
396 		if (ses->klen > 0)
397 			rc = blake2b_init_key(&bctx.sb, hashlen, ses->key, ses->klen);
398 		else
399 			rc = blake2b_init(&bctx.sb, hashlen);
400 		if (rc != 0)
401 			goto out;
402 		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
403 		    crd->crd_len, blake2b_applicator, &bctx.sb);
404 		if (error != 0)
405 			goto out;
406 		rc = blake2b_final(&bctx.sb, res, hashlen);
407 		if (rc != 0) {
408 			error = EINVAL;
409 			goto out;
410 		}
411 		break;
412 	case CRYPTO_BLAKE2S:
413 		if (ses->mlen != 0)
414 			hashlen = ses->mlen;
415 		else
416 			hashlen = BLAKE2S_OUTBYTES;
417 		if (ses->klen > 0)
418 			rc = blake2s_init_key(&bctx.ss, hashlen, ses->key, ses->klen);
419 		else
420 			rc = blake2s_init(&bctx.ss, hashlen);
421 		if (rc != 0)
422 			goto out;
423 		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
424 		    crd->crd_len, blake2s_applicator, &bctx.ss);
425 		if (error != 0)
426 			goto out;
427 		rc = blake2s_final(&bctx.ss, res, hashlen);
428 		if (rc != 0) {
429 			error = EINVAL;
430 			goto out;
431 		}
432 		break;
433 	default:
434 		panic("unreachable");
435 	}
436 
437 	crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
438 	    (void *)res);
439 
440 out:
441 	if (!kt) {
442 		fpu_kern_leave(curthread, ctx);
443 		RELEASE_CTX(ctxidx, ctx);
444 	}
445 	return (error);
446 }
447