1 /*-
2  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 #include <sys/param.h>
29 #include <sys/bus.h>
30 #include <sys/kernel.h>
31 #include <sys/kobj.h>
32 #include <sys/lock.h>
33 #include <sys/malloc.h>
34 #include <sys/module.h>
35 #include <sys/mutex.h>
36 #include <sys/rwlock.h>
37 #include <sys/smp.h>
38 
39 #include <blake2.h>
40 
41 #include <opencrypto/cryptodev.h>
42 #include <cryptodev_if.h>
43 
44 #include <machine/fpu.h>
45 
46 struct blake2_session {
47 	size_t mlen;
48 };
49 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
50 
51 struct blake2_softc {
52 	bool	dying;
53 	int32_t cid;
54 	struct rwlock lock;
55 };
56 
57 static struct mtx_padalign *ctx_mtx;
58 static struct fpu_kern_ctx **ctx_fpu;
59 
60 #define ACQUIRE_CTX(i, ctx)					\
61 	do {							\
62 		(i) = PCPU_GET(cpuid);				\
63 		mtx_lock(&ctx_mtx[(i)]);			\
64 		(ctx) = ctx_fpu[(i)];				\
65 	} while (0)
66 #define RELEASE_CTX(i, ctx)					\
67 	do {							\
68 		mtx_unlock(&ctx_mtx[(i)]);			\
69 		(i) = -1;					\
70 		(ctx) = NULL;					\
71 	} while (0)
72 
73 static int blake2_cipher_setup(struct blake2_session *ses,
74     const struct crypto_session_params *csp);
75 static int blake2_cipher_process(struct blake2_session *ses,
76     struct cryptop *crp);
77 
78 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
79 
80 static void
81 blake2_identify(driver_t *drv, device_t parent)
82 {
83 
84 	/* NB: order 10 is so we get attached after h/w devices */
85 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
86 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
87 		panic("blaketwo: could not attach");
88 }
89 
90 static int
91 blake2_probe(device_t dev)
92 {
93 	device_set_desc(dev, "Blake2");
94 	return (0);
95 }
96 
97 static void
98 blake2_cleanctx(void)
99 {
100 	int i;
101 
102 	/* XXX - no way to return driverid */
103 	CPU_FOREACH(i) {
104 		if (ctx_fpu[i] != NULL) {
105 			mtx_destroy(&ctx_mtx[i]);
106 			fpu_kern_free_ctx(ctx_fpu[i]);
107 		}
108 		ctx_fpu[i] = NULL;
109 	}
110 	free(ctx_mtx, M_BLAKE2);
111 	ctx_mtx = NULL;
112 	free(ctx_fpu, M_BLAKE2);
113 	ctx_fpu = NULL;
114 }
115 
116 static int
117 blake2_attach(device_t dev)
118 {
119 	struct blake2_softc *sc;
120 	int i;
121 
122 	sc = device_get_softc(dev);
123 	sc->dying = false;
124 
125 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
126 	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
127 	    CRYPTOCAP_F_ACCEL_SOFTWARE);
128 	if (sc->cid < 0) {
129 		device_printf(dev, "Could not get crypto driver id.\n");
130 		return (ENOMEM);
131 	}
132 
133 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
134 	    M_WAITOK | M_ZERO);
135 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
136 	    M_WAITOK | M_ZERO);
137 
138 	CPU_FOREACH(i) {
139 #ifdef __amd64__
140 		ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
141 		    pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
142 #else
143 		ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
144 #endif
145 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
146 	}
147 
148 	rw_init(&sc->lock, "blake2_lock");
149 
150 	return (0);
151 }
152 
153 static int
154 blake2_detach(device_t dev)
155 {
156 	struct blake2_softc *sc;
157 
158 	sc = device_get_softc(dev);
159 
160 	rw_wlock(&sc->lock);
161 	sc->dying = true;
162 	rw_wunlock(&sc->lock);
163 	crypto_unregister_all(sc->cid);
164 
165 	rw_destroy(&sc->lock);
166 
167 	blake2_cleanctx();
168 
169 	return (0);
170 }
171 
172 static int
173 blake2_probesession(device_t dev, const struct crypto_session_params *csp)
174 {
175 
176 	if (csp->csp_flags != 0)
177 		return (EINVAL);
178 	switch (csp->csp_mode) {
179 	case CSP_MODE_DIGEST:
180 		switch (csp->csp_auth_alg) {
181 		case CRYPTO_BLAKE2B:
182 		case CRYPTO_BLAKE2S:
183 			break;
184 		default:
185 			return (EINVAL);
186 		}
187 		break;
188 	default:
189 		return (EINVAL);
190 	}
191 	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
192 }
193 
194 static int
195 blake2_newsession(device_t dev, crypto_session_t cses,
196     const struct crypto_session_params *csp)
197 {
198 	struct blake2_softc *sc;
199 	struct blake2_session *ses;
200 	int error;
201 
202 	sc = device_get_softc(dev);
203 
204 	ses = crypto_get_driver_session(cses);
205 
206 	rw_rlock(&sc->lock);
207 	if (sc->dying) {
208 		rw_runlock(&sc->lock);
209 		return (EINVAL);
210 	}
211 	rw_runlock(&sc->lock);
212 
213 	error = blake2_cipher_setup(ses, csp);
214 	if (error != 0) {
215 		CRYPTDEB("setup failed");
216 		return (error);
217 	}
218 
219 	return (0);
220 }
221 
222 static int
223 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
224 {
225 	struct blake2_session *ses;
226 	int error;
227 
228 	ses = crypto_get_driver_session(crp->crp_session);
229 	error = blake2_cipher_process(ses, crp);
230 
231 	crp->crp_etype = error;
232 	crypto_done(crp);
233 	return (0);
234 }
235 
236 static device_method_t blake2_methods[] = {
237 	DEVMETHOD(device_identify, blake2_identify),
238 	DEVMETHOD(device_probe, blake2_probe),
239 	DEVMETHOD(device_attach, blake2_attach),
240 	DEVMETHOD(device_detach, blake2_detach),
241 
242 	DEVMETHOD(cryptodev_probesession, blake2_probesession),
243 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
244 	DEVMETHOD(cryptodev_process, blake2_process),
245 
246 	DEVMETHOD_END
247 };
248 
249 static driver_t blake2_driver = {
250 	"blaketwo",
251 	blake2_methods,
252 	sizeof(struct blake2_softc),
253 };
254 
255 DRIVER_MODULE(blake2, nexus, blake2_driver, 0, 0);
256 MODULE_VERSION(blake2, 1);
257 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
258 
259 static bool
260 blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
261 {
262 
263 	if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
264 		return (klen <= BLAKE2S_KEYBYTES);
265 	else
266 		return (klen <= BLAKE2B_KEYBYTES);
267 }
268 
269 static int
270 blake2_cipher_setup(struct blake2_session *ses,
271     const struct crypto_session_params *csp)
272 {
273 	int hashlen;
274 
275 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
276 
277 	if (!blake2_check_klen(csp, csp->csp_auth_klen))
278 		return (EINVAL);
279 
280 	if (csp->csp_auth_mlen < 0)
281 		return (EINVAL);
282 
283 	switch (csp->csp_auth_alg) {
284 	case CRYPTO_BLAKE2S:
285 		hashlen = BLAKE2S_OUTBYTES;
286 		break;
287 	case CRYPTO_BLAKE2B:
288 		hashlen = BLAKE2B_OUTBYTES;
289 		break;
290 	default:
291 		return (EINVAL);
292 	}
293 
294 	if (csp->csp_auth_mlen > hashlen)
295 		return (EINVAL);
296 
297 	if (csp->csp_auth_mlen == 0)
298 		ses->mlen = hashlen;
299 	else
300 		ses->mlen = csp->csp_auth_mlen;
301 	return (0);
302 }
303 
304 static int
305 blake2b_applicator(void *state, const void *buf, u_int len)
306 {
307 	int rc;
308 
309 	rc = blake2b_update(state, buf, len);
310 	if (rc != 0)
311 		return (EINVAL);
312 	return (0);
313 }
314 
315 static int
316 blake2s_applicator(void *state, const void *buf, u_int len)
317 {
318 	int rc;
319 
320 	rc = blake2s_update(state, buf, len);
321 	if (rc != 0)
322 		return (EINVAL);
323 	return (0);
324 }
325 
326 static int
327 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
328 {
329 	union {
330 		blake2b_state sb;
331 		blake2s_state ss;
332 	} bctx;
333 	char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
334 	const struct crypto_session_params *csp;
335 	struct fpu_kern_ctx *ctx;
336 	const void *key;
337 	int ctxidx;
338 	bool kt;
339 	int error, rc;
340 	unsigned klen;
341 
342 	ctx = NULL;
343 	ctxidx = 0;
344 	error = EINVAL;
345 
346 	kt = is_fpu_kern_thread(0);
347 	if (!kt) {
348 		ACQUIRE_CTX(ctxidx, ctx);
349 		fpu_kern_enter(curthread, ctx,
350 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
351 	}
352 
353 	csp = crypto_get_params(crp->crp_session);
354 	if (crp->crp_auth_key != NULL)
355 		key = crp->crp_auth_key;
356 	else
357 		key = csp->csp_auth_key;
358 	klen = csp->csp_auth_klen;
359 	switch (csp->csp_auth_alg) {
360 	case CRYPTO_BLAKE2B:
361 		if (klen > 0)
362 			rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
363 		else
364 			rc = blake2b_init(&bctx.sb, ses->mlen);
365 		if (rc != 0)
366 			goto out;
367 		error = crypto_apply(crp, crp->crp_payload_start,
368 		    crp->crp_payload_length, blake2b_applicator, &bctx.sb);
369 		if (error != 0)
370 			goto out;
371 		rc = blake2b_final(&bctx.sb, res, ses->mlen);
372 		if (rc != 0) {
373 			error = EINVAL;
374 			goto out;
375 		}
376 		break;
377 	case CRYPTO_BLAKE2S:
378 		if (klen > 0)
379 			rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
380 		else
381 			rc = blake2s_init(&bctx.ss, ses->mlen);
382 		if (rc != 0)
383 			goto out;
384 		error = crypto_apply(crp, crp->crp_payload_start,
385 		    crp->crp_payload_length, blake2s_applicator, &bctx.ss);
386 		if (error != 0)
387 			goto out;
388 		rc = blake2s_final(&bctx.ss, res, ses->mlen);
389 		if (rc != 0) {
390 			error = EINVAL;
391 			goto out;
392 		}
393 		break;
394 	default:
395 		panic("unreachable");
396 	}
397 
398 	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
399 		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
400 		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
401 			return (EBADMSG);
402 	} else
403 		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
404 
405 out:
406 	if (!kt) {
407 		fpu_kern_leave(curthread, ctx);
408 		RELEASE_CTX(ctxidx, ctx);
409 	}
410 	return (error);
411 }
412