1 /*-
2  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/kobj.h>
34 #include <sys/lock.h>
35 #include <sys/malloc.h>
36 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/rwlock.h>
39 #include <sys/smp.h>
40 
41 #include <blake2.h>
42 
43 #include <opencrypto/cryptodev.h>
44 #include <cryptodev_if.h>
45 
46 #include <machine/fpu.h>
47 
48 struct blake2_session {
49 	size_t mlen;
50 };
51 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
52 
53 struct blake2_softc {
54 	bool	dying;
55 	int32_t cid;
56 	struct rwlock lock;
57 };
58 
59 static struct mtx_padalign *ctx_mtx;
60 static struct fpu_kern_ctx **ctx_fpu;
61 
62 #define ACQUIRE_CTX(i, ctx)					\
63 	do {							\
64 		(i) = PCPU_GET(cpuid);				\
65 		mtx_lock(&ctx_mtx[(i)]);			\
66 		(ctx) = ctx_fpu[(i)];				\
67 	} while (0)
68 #define RELEASE_CTX(i, ctx)					\
69 	do {							\
70 		mtx_unlock(&ctx_mtx[(i)]);			\
71 		(i) = -1;					\
72 		(ctx) = NULL;					\
73 	} while (0)
74 
75 static int blake2_cipher_setup(struct blake2_session *ses,
76     const struct crypto_session_params *csp);
77 static int blake2_cipher_process(struct blake2_session *ses,
78     struct cryptop *crp);
79 
80 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
81 
82 static void
83 blake2_identify(driver_t *drv, device_t parent)
84 {
85 
86 	/* NB: order 10 is so we get attached after h/w devices */
87 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
88 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
89 		panic("blaketwo: could not attach");
90 }
91 
92 static int
93 blake2_probe(device_t dev)
94 {
95 	device_set_desc(dev, "Blake2");
96 	return (0);
97 }
98 
99 static void
100 blake2_cleanctx(void)
101 {
102 	int i;
103 
104 	/* XXX - no way to return driverid */
105 	CPU_FOREACH(i) {
106 		if (ctx_fpu[i] != NULL) {
107 			mtx_destroy(&ctx_mtx[i]);
108 			fpu_kern_free_ctx(ctx_fpu[i]);
109 		}
110 		ctx_fpu[i] = NULL;
111 	}
112 	free(ctx_mtx, M_BLAKE2);
113 	ctx_mtx = NULL;
114 	free(ctx_fpu, M_BLAKE2);
115 	ctx_fpu = NULL;
116 }
117 
118 static int
119 blake2_attach(device_t dev)
120 {
121 	struct blake2_softc *sc;
122 	int i;
123 
124 	sc = device_get_softc(dev);
125 	sc->dying = false;
126 
127 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
128 	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
129 	    CRYPTOCAP_F_ACCEL_SOFTWARE);
130 	if (sc->cid < 0) {
131 		device_printf(dev, "Could not get crypto driver id.\n");
132 		return (ENOMEM);
133 	}
134 
135 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
136 	    M_WAITOK | M_ZERO);
137 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
138 	    M_WAITOK | M_ZERO);
139 
140 	CPU_FOREACH(i) {
141 #ifdef __amd64__
142 		ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
143 		    pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
144 #else
145 		ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
146 #endif
147 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
148 	}
149 
150 	rw_init(&sc->lock, "blake2_lock");
151 
152 	return (0);
153 }
154 
155 static int
156 blake2_detach(device_t dev)
157 {
158 	struct blake2_softc *sc;
159 
160 	sc = device_get_softc(dev);
161 
162 	rw_wlock(&sc->lock);
163 	sc->dying = true;
164 	rw_wunlock(&sc->lock);
165 	crypto_unregister_all(sc->cid);
166 
167 	rw_destroy(&sc->lock);
168 
169 	blake2_cleanctx();
170 
171 	return (0);
172 }
173 
174 static int
175 blake2_probesession(device_t dev, const struct crypto_session_params *csp)
176 {
177 
178 	if (csp->csp_flags != 0)
179 		return (EINVAL);
180 	switch (csp->csp_mode) {
181 	case CSP_MODE_DIGEST:
182 		switch (csp->csp_auth_alg) {
183 		case CRYPTO_BLAKE2B:
184 		case CRYPTO_BLAKE2S:
185 			break;
186 		default:
187 			return (EINVAL);
188 		}
189 		break;
190 	default:
191 		return (EINVAL);
192 	}
193 	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
194 }
195 
196 static int
197 blake2_newsession(device_t dev, crypto_session_t cses,
198     const struct crypto_session_params *csp)
199 {
200 	struct blake2_softc *sc;
201 	struct blake2_session *ses;
202 	int error;
203 
204 	sc = device_get_softc(dev);
205 
206 	ses = crypto_get_driver_session(cses);
207 
208 	rw_rlock(&sc->lock);
209 	if (sc->dying) {
210 		rw_runlock(&sc->lock);
211 		return (EINVAL);
212 	}
213 	rw_runlock(&sc->lock);
214 
215 	error = blake2_cipher_setup(ses, csp);
216 	if (error != 0) {
217 		CRYPTDEB("setup failed");
218 		return (error);
219 	}
220 
221 	return (0);
222 }
223 
224 static int
225 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
226 {
227 	struct blake2_session *ses;
228 	int error;
229 
230 	ses = crypto_get_driver_session(crp->crp_session);
231 	error = blake2_cipher_process(ses, crp);
232 
233 	crp->crp_etype = error;
234 	crypto_done(crp);
235 	return (0);
236 }
237 
238 static device_method_t blake2_methods[] = {
239 	DEVMETHOD(device_identify, blake2_identify),
240 	DEVMETHOD(device_probe, blake2_probe),
241 	DEVMETHOD(device_attach, blake2_attach),
242 	DEVMETHOD(device_detach, blake2_detach),
243 
244 	DEVMETHOD(cryptodev_probesession, blake2_probesession),
245 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
246 	DEVMETHOD(cryptodev_process, blake2_process),
247 
248 	DEVMETHOD_END
249 };
250 
251 static driver_t blake2_driver = {
252 	"blaketwo",
253 	blake2_methods,
254 	sizeof(struct blake2_softc),
255 };
256 
257 DRIVER_MODULE(blake2, nexus, blake2_driver, 0, 0);
258 MODULE_VERSION(blake2, 1);
259 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
260 
261 static bool
262 blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
263 {
264 
265 	if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
266 		return (klen <= BLAKE2S_KEYBYTES);
267 	else
268 		return (klen <= BLAKE2B_KEYBYTES);
269 }
270 
271 static int
272 blake2_cipher_setup(struct blake2_session *ses,
273     const struct crypto_session_params *csp)
274 {
275 	int hashlen;
276 
277 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
278 
279 	if (!blake2_check_klen(csp, csp->csp_auth_klen))
280 		return (EINVAL);
281 
282 	if (csp->csp_auth_mlen < 0)
283 		return (EINVAL);
284 
285 	switch (csp->csp_auth_alg) {
286 	case CRYPTO_BLAKE2S:
287 		hashlen = BLAKE2S_OUTBYTES;
288 		break;
289 	case CRYPTO_BLAKE2B:
290 		hashlen = BLAKE2B_OUTBYTES;
291 		break;
292 	default:
293 		return (EINVAL);
294 	}
295 
296 	if (csp->csp_auth_mlen > hashlen)
297 		return (EINVAL);
298 
299 	if (csp->csp_auth_mlen == 0)
300 		ses->mlen = hashlen;
301 	else
302 		ses->mlen = csp->csp_auth_mlen;
303 	return (0);
304 }
305 
306 static int
307 blake2b_applicator(void *state, const void *buf, u_int len)
308 {
309 	int rc;
310 
311 	rc = blake2b_update(state, buf, len);
312 	if (rc != 0)
313 		return (EINVAL);
314 	return (0);
315 }
316 
317 static int
318 blake2s_applicator(void *state, const void *buf, u_int len)
319 {
320 	int rc;
321 
322 	rc = blake2s_update(state, buf, len);
323 	if (rc != 0)
324 		return (EINVAL);
325 	return (0);
326 }
327 
328 static int
329 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
330 {
331 	union {
332 		blake2b_state sb;
333 		blake2s_state ss;
334 	} bctx;
335 	char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
336 	const struct crypto_session_params *csp;
337 	struct fpu_kern_ctx *ctx;
338 	const void *key;
339 	int ctxidx;
340 	bool kt;
341 	int error, rc;
342 	unsigned klen;
343 
344 	ctx = NULL;
345 	ctxidx = 0;
346 	error = EINVAL;
347 
348 	kt = is_fpu_kern_thread(0);
349 	if (!kt) {
350 		ACQUIRE_CTX(ctxidx, ctx);
351 		fpu_kern_enter(curthread, ctx,
352 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
353 	}
354 
355 	csp = crypto_get_params(crp->crp_session);
356 	if (crp->crp_auth_key != NULL)
357 		key = crp->crp_auth_key;
358 	else
359 		key = csp->csp_auth_key;
360 	klen = csp->csp_auth_klen;
361 	switch (csp->csp_auth_alg) {
362 	case CRYPTO_BLAKE2B:
363 		if (klen > 0)
364 			rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
365 		else
366 			rc = blake2b_init(&bctx.sb, ses->mlen);
367 		if (rc != 0)
368 			goto out;
369 		error = crypto_apply(crp, crp->crp_payload_start,
370 		    crp->crp_payload_length, blake2b_applicator, &bctx.sb);
371 		if (error != 0)
372 			goto out;
373 		rc = blake2b_final(&bctx.sb, res, ses->mlen);
374 		if (rc != 0) {
375 			error = EINVAL;
376 			goto out;
377 		}
378 		break;
379 	case CRYPTO_BLAKE2S:
380 		if (klen > 0)
381 			rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
382 		else
383 			rc = blake2s_init(&bctx.ss, ses->mlen);
384 		if (rc != 0)
385 			goto out;
386 		error = crypto_apply(crp, crp->crp_payload_start,
387 		    crp->crp_payload_length, blake2s_applicator, &bctx.ss);
388 		if (error != 0)
389 			goto out;
390 		rc = blake2s_final(&bctx.ss, res, ses->mlen);
391 		if (rc != 0) {
392 			error = EINVAL;
393 			goto out;
394 		}
395 		break;
396 	default:
397 		panic("unreachable");
398 	}
399 
400 	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
401 		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
402 		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
403 			return (EBADMSG);
404 	} else
405 		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
406 
407 out:
408 	if (!kt) {
409 		fpu_kern_leave(curthread, ctx);
410 		RELEASE_CTX(ctxidx, ctx);
411 	}
412 	return (error);
413 }
414