1f603807bSTomohiro Kusumi /*
2f603807bSTomohiro Kusumi * Copyright (c) 2010 The DragonFly Project. All rights reserved.
3f603807bSTomohiro Kusumi *
4f603807bSTomohiro Kusumi * This code is derived from software contributed to The DragonFly Project
5f603807bSTomohiro Kusumi * by Alex Hornung <ahornung@gmail.com>
6f603807bSTomohiro Kusumi *
7f603807bSTomohiro Kusumi * Redistribution and use in source and binary forms, with or without
8f603807bSTomohiro Kusumi * modification, are permitted provided that the following conditions
9f603807bSTomohiro Kusumi * are met:
10f603807bSTomohiro Kusumi *
11f603807bSTomohiro Kusumi * 1. Redistributions of source code must retain the above copyright
12f603807bSTomohiro Kusumi * notice, this list of conditions and the following disclaimer.
13f603807bSTomohiro Kusumi * 2. Redistributions in binary form must reproduce the above copyright
14f603807bSTomohiro Kusumi * notice, this list of conditions and the following disclaimer in
15f603807bSTomohiro Kusumi * the documentation and/or other materials provided with the
16f603807bSTomohiro Kusumi * distribution.
17f603807bSTomohiro Kusumi * 3. Neither the name of The DragonFly Project nor the names of its
18f603807bSTomohiro Kusumi * contributors may be used to endorse or promote products derived
19f603807bSTomohiro Kusumi * from this software without specific, prior written permission.
20f603807bSTomohiro Kusumi *
21f603807bSTomohiro Kusumi * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22f603807bSTomohiro Kusumi * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23f603807bSTomohiro Kusumi * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24f603807bSTomohiro Kusumi * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25f603807bSTomohiro Kusumi * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26f603807bSTomohiro Kusumi * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27f603807bSTomohiro Kusumi * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28f603807bSTomohiro Kusumi * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29f603807bSTomohiro Kusumi * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30f603807bSTomohiro Kusumi * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31f603807bSTomohiro Kusumi * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32f603807bSTomohiro Kusumi * SUCH DAMAGE.
33f603807bSTomohiro Kusumi */
34f603807bSTomohiro Kusumi
35f603807bSTomohiro Kusumi /*
36f603807bSTomohiro Kusumi * This file implements initial version of device-mapper crypt target.
37f603807bSTomohiro Kusumi */
38f603807bSTomohiro Kusumi #include <sys/endian.h>
39f603807bSTomohiro Kusumi
40f603807bSTomohiro Kusumi #include <sys/bio.h>
41f603807bSTomohiro Kusumi #include <sys/globaldata.h>
42f603807bSTomohiro Kusumi #include <sys/kerneldump.h>
43f603807bSTomohiro Kusumi #include <sys/malloc.h>
44f603807bSTomohiro Kusumi #include <sys/mpipe.h>
45f603807bSTomohiro Kusumi #include <sys/md5.h>
46f603807bSTomohiro Kusumi #include <crypto/sha1.h>
47f603807bSTomohiro Kusumi #include <crypto/sha2/sha2.h>
48f603807bSTomohiro Kusumi #include <opencrypto/cryptodev.h>
49f603807bSTomohiro Kusumi #include <opencrypto/rmd160.h>
50f603807bSTomohiro Kusumi #include <machine/cpufunc.h>
51f603807bSTomohiro Kusumi #include <cpu/atomic.h>
52f603807bSTomohiro Kusumi
53f603807bSTomohiro Kusumi #include <sys/ktr.h>
54*8345afadSMatthew Dillon #include <sys/spinlock2.h>
55f603807bSTomohiro Kusumi
56f603807bSTomohiro Kusumi #include <dev/disk/dm/dm.h>
57f603807bSTomohiro Kusumi MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
58f603807bSTomohiro Kusumi
59f603807bSTomohiro Kusumi KTR_INFO_MASTER(dmcrypt);
60f603807bSTomohiro Kusumi
61f603807bSTomohiro Kusumi #if !defined(KTR_DMCRYPT)
62f603807bSTomohiro Kusumi #define KTR_DMCRYPT KTR_ALL
63f603807bSTomohiro Kusumi #endif
64f603807bSTomohiro Kusumi
65f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
66f603807bSTomohiro Kusumi "crypto_dispatch(%p)", struct cryptop *crp);
67f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
68f603807bSTomohiro Kusumi "crypt_strategy(b_cmd = %d, bp = %p)", int cmd, struct buf *bp);
69f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
70f603807bSTomohiro Kusumi "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
71f603807bSTomohiro Kusumi struct cryptop *crp, struct buf *bp, int i, int sectors);
72f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
73f603807bSTomohiro Kusumi "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
74f603807bSTomohiro Kusumi struct cryptop *crp, struct buf *bp, int n);
75f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
76f603807bSTomohiro Kusumi "bio_write_done(bp = %p)", struct buf *bp);
77f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
78f603807bSTomohiro Kusumi "crypto_write_retry(crp = %p)", struct buf *bp);
79f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
80f603807bSTomohiro Kusumi "bio_read_done(bp = %p)", struct buf *bp);
81f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
82f603807bSTomohiro Kusumi "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
83f603807bSTomohiro Kusumi struct cryptop *crp, struct buf *bp, int i, int sectors);
84f603807bSTomohiro Kusumi KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
85f603807bSTomohiro Kusumi "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
86f603807bSTomohiro Kusumi struct cryptop *crp, struct buf *bp, int n);
87f603807bSTomohiro Kusumi
88f603807bSTomohiro Kusumi struct target_crypt_config;
89f603807bSTomohiro Kusumi
90f603807bSTomohiro Kusumi typedef void dispatch_t(void *);
91f603807bSTomohiro Kusumi typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
92f603807bSTomohiro Kusumi void *);
93f603807bSTomohiro Kusumi
94f603807bSTomohiro Kusumi typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
95f603807bSTomohiro Kusumi typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
96f603807bSTomohiro Kusumi
97f603807bSTomohiro Kusumi struct iv_generator {
98f603807bSTomohiro Kusumi const char *name;
99f603807bSTomohiro Kusumi ivgen_ctor_t *ctor;
100f603807bSTomohiro Kusumi ivgen_dtor_t *dtor;
101f603807bSTomohiro Kusumi ivgen_t *gen_iv;
102f603807bSTomohiro Kusumi };
103f603807bSTomohiro Kusumi
104*8345afadSMatthew Dillon struct essiv_ivgen_data {
105*8345afadSMatthew Dillon struct essiv_ivgen_data *next;
106*8345afadSMatthew Dillon void *ivpriv;
107*8345afadSMatthew Dillon void *opaque;
108*8345afadSMatthew Dillon struct cryptop crp;
109*8345afadSMatthew Dillon struct cryptodesc crd;
110*8345afadSMatthew Dillon };
111*8345afadSMatthew Dillon
112f603807bSTomohiro Kusumi struct essiv_ivgen_priv {
113f603807bSTomohiro Kusumi struct cryptoini crypto_session;
114*8345afadSMatthew Dillon struct spinlock ivdata_spin;
115*8345afadSMatthew Dillon struct essiv_ivgen_data *ivdata_base;
116f603807bSTomohiro Kusumi u_int64_t crypto_sid;
117f603807bSTomohiro Kusumi size_t keyhash_len;
118f603807bSTomohiro Kusumi u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
119f603807bSTomohiro Kusumi };
120f603807bSTomohiro Kusumi
121f603807bSTomohiro Kusumi typedef struct target_crypt_config {
122f603807bSTomohiro Kusumi size_t params_len;
123f603807bSTomohiro Kusumi dm_pdev_t *pdev;
124f603807bSTomohiro Kusumi char *status_str;
125f603807bSTomohiro Kusumi int crypto_alg;
126f603807bSTomohiro Kusumi int crypto_klen;
127f603807bSTomohiro Kusumi u_int8_t crypto_key[512>>3];
128f603807bSTomohiro Kusumi
129f603807bSTomohiro Kusumi u_int64_t crypto_sid;
130f603807bSTomohiro Kusumi u_int64_t block_offset;
131f603807bSTomohiro Kusumi int64_t iv_offset;
132f603807bSTomohiro Kusumi SHA512_CTX essivsha512_ctx;
133f603807bSTomohiro Kusumi
134f603807bSTomohiro Kusumi struct cryptoini crypto_session;
135f603807bSTomohiro Kusumi
136f603807bSTomohiro Kusumi struct iv_generator *ivgen;
137f603807bSTomohiro Kusumi void *ivgen_priv;
138f603807bSTomohiro Kusumi
139f603807bSTomohiro Kusumi struct malloc_pipe read_mpipe;
140f603807bSTomohiro Kusumi struct malloc_pipe write_mpipe;
141f603807bSTomohiro Kusumi } dm_target_crypt_config_t;
142f603807bSTomohiro Kusumi
143f603807bSTomohiro Kusumi struct dmtc_helper {
144f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
145f603807bSTomohiro Kusumi caddr_t free_addr;
146f603807bSTomohiro Kusumi caddr_t orig_buf;
147f603807bSTomohiro Kusumi caddr_t data_buf;
148f603807bSTomohiro Kusumi };
149f603807bSTomohiro Kusumi
150f603807bSTomohiro Kusumi struct dmtc_dump_helper {
151f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
152f603807bSTomohiro Kusumi void *data;
153f603807bSTomohiro Kusumi size_t length;
154f603807bSTomohiro Kusumi off_t offset;
155f603807bSTomohiro Kusumi
156f603807bSTomohiro Kusumi int sectors;
157f603807bSTomohiro Kusumi int *ident;
158f603807bSTomohiro Kusumi
159f603807bSTomohiro Kusumi struct cryptodesc crd[128];
160f603807bSTomohiro Kusumi struct cryptop crp[128];
161f603807bSTomohiro Kusumi u_char space[65536];
162f603807bSTomohiro Kusumi };
163f603807bSTomohiro Kusumi
164f603807bSTomohiro Kusumi #define DMTC_BUF_SIZE_WRITE \
165*8345afadSMatthew Dillon (MAXPHYS + sizeof(struct dmtc_helper) + \
166*8345afadSMatthew Dillon MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc)))
167f603807bSTomohiro Kusumi #define DMTC_BUF_SIZE_READ \
168*8345afadSMatthew Dillon (sizeof(struct dmtc_helper) + \
169*8345afadSMatthew Dillon MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc)))
170f603807bSTomohiro Kusumi
171f603807bSTomohiro Kusumi static void dmtc_crypto_dispatch(void *arg);
172f603807bSTomohiro Kusumi static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
173f603807bSTomohiro Kusumi struct dmtc_dump_helper *dump_helper);
174f603807bSTomohiro Kusumi static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
175f603807bSTomohiro Kusumi struct bio *bio);
176f603807bSTomohiro Kusumi static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
177f603807bSTomohiro Kusumi struct bio *bio);
178f603807bSTomohiro Kusumi static void dmtc_bio_read_done(struct bio *bio);
179f603807bSTomohiro Kusumi static void dmtc_bio_write_done(struct bio *bio);
180f603807bSTomohiro Kusumi static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
181f603807bSTomohiro Kusumi static int dmtc_crypto_cb_read_done(struct cryptop *crp);
182f603807bSTomohiro Kusumi static int dmtc_crypto_cb_write_done(struct cryptop *crp);
183f603807bSTomohiro Kusumi
184f603807bSTomohiro Kusumi static ivgen_ctor_t essiv_ivgen_ctor;
185f603807bSTomohiro Kusumi static ivgen_dtor_t essiv_ivgen_dtor;
186f603807bSTomohiro Kusumi static ivgen_t essiv_ivgen;
187f603807bSTomohiro Kusumi static ivgen_t plain_ivgen;
188f603807bSTomohiro Kusumi static ivgen_t plain64_ivgen;
189f603807bSTomohiro Kusumi
190f603807bSTomohiro Kusumi static struct iv_generator ivgens[] = {
191f603807bSTomohiro Kusumi { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
192f603807bSTomohiro Kusumi .gen_iv = essiv_ivgen },
193f603807bSTomohiro Kusumi { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
194f603807bSTomohiro Kusumi { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
195f603807bSTomohiro Kusumi { NULL, NULL, NULL, NULL }
196f603807bSTomohiro Kusumi };
197f603807bSTomohiro Kusumi
198*8345afadSMatthew Dillon /*
199*8345afadSMatthew Dillon * Number of crypto buffers. All crypto buffers will be preallocated
200*8345afadSMatthew Dillon * in order to avoid kmalloc() deadlocks in critical low-memory paging
201*8345afadSMatthew Dillon * paths.
202*8345afadSMatthew Dillon */
203*8345afadSMatthew Dillon static __inline int
dmtc_get_nmax(void)204*8345afadSMatthew Dillon dmtc_get_nmax(void)
205*8345afadSMatthew Dillon {
206*8345afadSMatthew Dillon int nmax;
207f603807bSTomohiro Kusumi
208*8345afadSMatthew Dillon nmax = (physmem * 2 / 1000 * PAGE_SIZE) /
209*8345afadSMatthew Dillon (DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
210*8345afadSMatthew Dillon
211*8345afadSMatthew Dillon if (nmax < 2)
212*8345afadSMatthew Dillon nmax = 2;
213*8345afadSMatthew Dillon if (nmax > 8 + ncpus * 2)
214*8345afadSMatthew Dillon nmax = 8 + ncpus * 2;
215*8345afadSMatthew Dillon
216*8345afadSMatthew Dillon return nmax;
217*8345afadSMatthew Dillon }
218*8345afadSMatthew Dillon
219*8345afadSMatthew Dillon /*
220*8345afadSMatthew Dillon * Initialize the crypto buffer mpipe. Preallocate all crypto buffers
221*8345afadSMatthew Dillon * to avoid making any kmalloc()s in the critical path.
222*8345afadSMatthew Dillon */
223f603807bSTomohiro Kusumi static void
dmtc_init_mpipe(struct target_crypt_config * priv)224f603807bSTomohiro Kusumi dmtc_init_mpipe(struct target_crypt_config *priv)
225f603807bSTomohiro Kusumi {
226f603807bSTomohiro Kusumi int nmax;
227f603807bSTomohiro Kusumi
228*8345afadSMatthew Dillon nmax = dmtc_get_nmax();
229f603807bSTomohiro Kusumi
230*8345afadSMatthew Dillon kprintf("dm_target_crypt: Setting %d mpipe buffers\n", nmax);
231f603807bSTomohiro Kusumi
232f603807bSTomohiro Kusumi mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
233*8345afadSMatthew Dillon nmax, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
234f603807bSTomohiro Kusumi mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
235*8345afadSMatthew Dillon nmax, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
236f603807bSTomohiro Kusumi }
237f603807bSTomohiro Kusumi
238f603807bSTomohiro Kusumi static void
dmtc_destroy_mpipe(struct target_crypt_config * priv)239f603807bSTomohiro Kusumi dmtc_destroy_mpipe(struct target_crypt_config *priv)
240f603807bSTomohiro Kusumi {
241f603807bSTomohiro Kusumi mpipe_done(&priv->write_mpipe);
242f603807bSTomohiro Kusumi mpipe_done(&priv->read_mpipe);
243f603807bSTomohiro Kusumi }
244f603807bSTomohiro Kusumi
245f603807bSTomohiro Kusumi /*
246f603807bSTomohiro Kusumi * Overwrite private information (in buf) to avoid leaking it
247f603807bSTomohiro Kusumi */
248f603807bSTomohiro Kusumi static void
dmtc_crypto_clear(void * buf,size_t len)249f603807bSTomohiro Kusumi dmtc_crypto_clear(void *buf, size_t len)
250f603807bSTomohiro Kusumi {
251f603807bSTomohiro Kusumi memset(buf, 0xFF, len);
252f603807bSTomohiro Kusumi bzero(buf, len);
253f603807bSTomohiro Kusumi }
254f603807bSTomohiro Kusumi
255f603807bSTomohiro Kusumi /*
256f603807bSTomohiro Kusumi * ESSIV IV Generator Routines
257f603807bSTomohiro Kusumi */
258f603807bSTomohiro Kusumi static int
essiv_ivgen_ctor(struct target_crypt_config * priv,char * iv_hash,void ** p_ivpriv)259f603807bSTomohiro Kusumi essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
260f603807bSTomohiro Kusumi {
261f603807bSTomohiro Kusumi struct essiv_ivgen_priv *ivpriv;
262f603807bSTomohiro Kusumi u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
263f603807bSTomohiro Kusumi unsigned int klen, hashlen;
264f603807bSTomohiro Kusumi int error;
265*8345afadSMatthew Dillon int nmax;
266f603807bSTomohiro Kusumi
267f603807bSTomohiro Kusumi klen = (priv->crypto_klen >> 3);
268f603807bSTomohiro Kusumi
269f603807bSTomohiro Kusumi if (iv_hash == NULL)
270f603807bSTomohiro Kusumi return EINVAL;
271f603807bSTomohiro Kusumi
272f603807bSTomohiro Kusumi if (!strcmp(iv_hash, "sha1")) {
273f603807bSTomohiro Kusumi SHA1_CTX ctx;
274f603807bSTomohiro Kusumi
275f603807bSTomohiro Kusumi hashlen = SHA1_RESULTLEN;
276f603807bSTomohiro Kusumi SHA1Init(&ctx);
277f603807bSTomohiro Kusumi SHA1Update(&ctx, priv->crypto_key, klen);
278f603807bSTomohiro Kusumi SHA1Final(crypto_keyhash, &ctx);
279f603807bSTomohiro Kusumi } else if (!strcmp(iv_hash, "sha256")) {
280f603807bSTomohiro Kusumi SHA256_CTX ctx;
281f603807bSTomohiro Kusumi
282f603807bSTomohiro Kusumi hashlen = SHA256_DIGEST_LENGTH;
283f603807bSTomohiro Kusumi SHA256_Init(&ctx);
284f603807bSTomohiro Kusumi SHA256_Update(&ctx, priv->crypto_key, klen);
285f603807bSTomohiro Kusumi SHA256_Final(crypto_keyhash, &ctx);
286f603807bSTomohiro Kusumi } else if (!strcmp(iv_hash, "sha384")) {
287f603807bSTomohiro Kusumi SHA384_CTX ctx;
288f603807bSTomohiro Kusumi
289f603807bSTomohiro Kusumi hashlen = SHA384_DIGEST_LENGTH;
290f603807bSTomohiro Kusumi SHA384_Init(&ctx);
291f603807bSTomohiro Kusumi SHA384_Update(&ctx, priv->crypto_key, klen);
292f603807bSTomohiro Kusumi SHA384_Final(crypto_keyhash, &ctx);
293f603807bSTomohiro Kusumi } else if (!strcmp(iv_hash, "sha512")) {
294f603807bSTomohiro Kusumi SHA512_CTX ctx;
295f603807bSTomohiro Kusumi
296f603807bSTomohiro Kusumi hashlen = SHA512_DIGEST_LENGTH;
297f603807bSTomohiro Kusumi SHA512_Init(&ctx);
298f603807bSTomohiro Kusumi SHA512_Update(&ctx, priv->crypto_key, klen);
299f603807bSTomohiro Kusumi SHA512_Final(crypto_keyhash, &ctx);
300f603807bSTomohiro Kusumi } else if (!strcmp(iv_hash, "md5")) {
301f603807bSTomohiro Kusumi MD5_CTX ctx;
302f603807bSTomohiro Kusumi
303f603807bSTomohiro Kusumi hashlen = MD5_DIGEST_LENGTH;
304f603807bSTomohiro Kusumi MD5Init(&ctx);
305f603807bSTomohiro Kusumi MD5Update(&ctx, priv->crypto_key, klen);
306f603807bSTomohiro Kusumi MD5Final(crypto_keyhash, &ctx);
307f603807bSTomohiro Kusumi } else if (!strcmp(iv_hash, "rmd160") ||
308f603807bSTomohiro Kusumi !strcmp(iv_hash, "ripemd160")) {
309f603807bSTomohiro Kusumi RMD160_CTX ctx;
310f603807bSTomohiro Kusumi
311f603807bSTomohiro Kusumi hashlen = 160/8;
312f603807bSTomohiro Kusumi RMD160Init(&ctx);
313f603807bSTomohiro Kusumi RMD160Update(&ctx, priv->crypto_key, klen);
314f603807bSTomohiro Kusumi RMD160Final(crypto_keyhash, &ctx);
315f603807bSTomohiro Kusumi } else {
316f603807bSTomohiro Kusumi return EINVAL;
317f603807bSTomohiro Kusumi }
318f603807bSTomohiro Kusumi
319f603807bSTomohiro Kusumi /* Convert hashlen to bits */
320f603807bSTomohiro Kusumi hashlen <<= 3;
321f603807bSTomohiro Kusumi
322f603807bSTomohiro Kusumi ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
323f603807bSTomohiro Kusumi M_WAITOK | M_ZERO);
324f603807bSTomohiro Kusumi memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
325f603807bSTomohiro Kusumi ivpriv->keyhash_len = sizeof(crypto_keyhash);
326f603807bSTomohiro Kusumi dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
327f603807bSTomohiro Kusumi
328f603807bSTomohiro Kusumi ivpriv->crypto_session.cri_alg = priv->crypto_alg;
329f603807bSTomohiro Kusumi ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
330f603807bSTomohiro Kusumi ivpriv->crypto_session.cri_klen = hashlen;
331f603807bSTomohiro Kusumi ivpriv->crypto_session.cri_mlen = 0;
332f603807bSTomohiro Kusumi ivpriv->crypto_session.cri_next = NULL;
333f603807bSTomohiro Kusumi
334f603807bSTomohiro Kusumi /*
335f603807bSTomohiro Kusumi * XXX: in principle we also need to check if the block size of the
336f603807bSTomohiro Kusumi * cipher is a valid iv size for the block cipher.
337f603807bSTomohiro Kusumi */
338f603807bSTomohiro Kusumi
339f603807bSTomohiro Kusumi error = crypto_newsession(&ivpriv->crypto_sid,
340f603807bSTomohiro Kusumi &ivpriv->crypto_session,
341f603807bSTomohiro Kusumi CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
342f603807bSTomohiro Kusumi if (error) {
343f603807bSTomohiro Kusumi kprintf("dm_target_crypt: Error during crypto_newsession "
344f603807bSTomohiro Kusumi "for essiv_ivgen, error = %d\n",
345f603807bSTomohiro Kusumi error);
346f603807bSTomohiro Kusumi dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
347f603807bSTomohiro Kusumi kfree(ivpriv, M_DMCRYPT);
348f603807bSTomohiro Kusumi return ENOTSUP;
349f603807bSTomohiro Kusumi }
350f603807bSTomohiro Kusumi
351*8345afadSMatthew Dillon /*
352*8345afadSMatthew Dillon * mpipe for 512-byte ivgen elements, make sure there are enough
353*8345afadSMatthew Dillon * to cover all in-flight read and write buffers.
354*8345afadSMatthew Dillon */
355*8345afadSMatthew Dillon nmax = dmtc_get_nmax() * (int)(MAXPHYS / DEV_BSIZE) * 2;
356f603807bSTomohiro Kusumi
357*8345afadSMatthew Dillon spin_init(&ivpriv->ivdata_spin, "ivdata");
358*8345afadSMatthew Dillon
359*8345afadSMatthew Dillon while (nmax) {
360*8345afadSMatthew Dillon struct essiv_ivgen_data *ivdata;
361*8345afadSMatthew Dillon
362*8345afadSMatthew Dillon ivdata = kmalloc(sizeof(*ivdata), M_DMCRYPT, M_WAITOK|M_ZERO);
363*8345afadSMatthew Dillon ivdata->next = ivpriv->ivdata_base;
364*8345afadSMatthew Dillon ivpriv->ivdata_base = ivdata;
365*8345afadSMatthew Dillon --nmax;
366*8345afadSMatthew Dillon }
367f603807bSTomohiro Kusumi *p_ivpriv = ivpriv;
368*8345afadSMatthew Dillon
369f603807bSTomohiro Kusumi return 0;
370f603807bSTomohiro Kusumi }
371f603807bSTomohiro Kusumi
372f603807bSTomohiro Kusumi static int
essiv_ivgen_dtor(struct target_crypt_config * priv,void * arg)373f603807bSTomohiro Kusumi essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
374f603807bSTomohiro Kusumi {
375f603807bSTomohiro Kusumi struct essiv_ivgen_priv *ivpriv;
376*8345afadSMatthew Dillon struct essiv_ivgen_data *ivdata;
377f603807bSTomohiro Kusumi
378f603807bSTomohiro Kusumi ivpriv = (struct essiv_ivgen_priv *)arg;
379f603807bSTomohiro Kusumi KKASSERT(ivpriv != NULL);
380f603807bSTomohiro Kusumi
381f603807bSTomohiro Kusumi crypto_freesession(ivpriv->crypto_sid);
382f603807bSTomohiro Kusumi
383*8345afadSMatthew Dillon while ((ivdata = ivpriv->ivdata_base) != NULL) {
384*8345afadSMatthew Dillon ivpriv->ivdata_base = ivdata->next;
385*8345afadSMatthew Dillon kfree(ivdata, M_DMCRYPT);
386*8345afadSMatthew Dillon }
387*8345afadSMatthew Dillon spin_uninit(&ivpriv->ivdata_spin);
388f603807bSTomohiro Kusumi
389f603807bSTomohiro Kusumi dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
390f603807bSTomohiro Kusumi kfree(ivpriv, M_DMCRYPT);
391f603807bSTomohiro Kusumi
392f603807bSTomohiro Kusumi return 0;
393f603807bSTomohiro Kusumi }
394f603807bSTomohiro Kusumi
395f603807bSTomohiro Kusumi static int
essiv_ivgen_done(struct cryptop * crp)396f603807bSTomohiro Kusumi essiv_ivgen_done(struct cryptop *crp)
397f603807bSTomohiro Kusumi {
398f603807bSTomohiro Kusumi struct essiv_ivgen_priv *ivpriv;
399*8345afadSMatthew Dillon struct essiv_ivgen_data *ivdata;
400f603807bSTomohiro Kusumi void *opaque;
401f603807bSTomohiro Kusumi
402f603807bSTomohiro Kusumi
403f603807bSTomohiro Kusumi if (crp->crp_etype == EAGAIN)
404f603807bSTomohiro Kusumi return crypto_dispatch(crp);
405f603807bSTomohiro Kusumi
406f603807bSTomohiro Kusumi if (crp->crp_etype != 0) {
407f603807bSTomohiro Kusumi kprintf("dm_target_crypt: essiv_ivgen_done, "
408f603807bSTomohiro Kusumi "crp->crp_etype = %d\n", crp->crp_etype);
409f603807bSTomohiro Kusumi }
410f603807bSTomohiro Kusumi
411*8345afadSMatthew Dillon ivdata = (void *)crp->crp_opaque;
412*8345afadSMatthew Dillon
413f603807bSTomohiro Kusumi /*
414f603807bSTomohiro Kusumi * In-memory structure is:
415f603807bSTomohiro Kusumi * | ivpriv | opaque | crp | crd |
416f603807bSTomohiro Kusumi * | (void *) | (void *) | (cryptop) | (cryptodesc) |
417f603807bSTomohiro Kusumi */
418*8345afadSMatthew Dillon ivpriv = ivdata->ivpriv;
419*8345afadSMatthew Dillon opaque = ivdata->opaque;
420f603807bSTomohiro Kusumi
421*8345afadSMatthew Dillon spin_lock(&ivpriv->ivdata_spin);
422*8345afadSMatthew Dillon ivdata->next = ivpriv->ivdata_base;
423*8345afadSMatthew Dillon ivpriv->ivdata_base = ivdata;
424*8345afadSMatthew Dillon spin_unlock(&ivpriv->ivdata_spin);
425*8345afadSMatthew Dillon
426f603807bSTomohiro Kusumi dmtc_crypto_dispatch(opaque);
427*8345afadSMatthew Dillon
428f603807bSTomohiro Kusumi return 0;
429f603807bSTomohiro Kusumi }
430f603807bSTomohiro Kusumi
431f603807bSTomohiro Kusumi static void
essiv_ivgen(dm_target_crypt_config_t * priv,u_int8_t * iv,size_t iv_len,off_t sector,void * opaque)432f603807bSTomohiro Kusumi essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
433f603807bSTomohiro Kusumi size_t iv_len, off_t sector, void *opaque)
434f603807bSTomohiro Kusumi {
435f603807bSTomohiro Kusumi struct essiv_ivgen_priv *ivpriv;
436*8345afadSMatthew Dillon struct essiv_ivgen_data *ivdata;
437f603807bSTomohiro Kusumi struct cryptodesc *crd;
438f603807bSTomohiro Kusumi struct cryptop *crp;
439f603807bSTomohiro Kusumi int error;
440f603807bSTomohiro Kusumi
441f603807bSTomohiro Kusumi ivpriv = priv->ivgen_priv;
442f603807bSTomohiro Kusumi KKASSERT(ivpriv != NULL);
443f603807bSTomohiro Kusumi
444f603807bSTomohiro Kusumi /*
445*8345afadSMatthew Dillon * We preallocated all necessary ivdata's, so pull one off and use
446*8345afadSMatthew Dillon * it.
447f603807bSTomohiro Kusumi */
448*8345afadSMatthew Dillon spin_lock(&ivpriv->ivdata_spin);
449*8345afadSMatthew Dillon ivdata = ivpriv->ivdata_base;
450*8345afadSMatthew Dillon ivpriv->ivdata_base = ivdata->next;
451*8345afadSMatthew Dillon spin_unlock(&ivpriv->ivdata_spin);
452*8345afadSMatthew Dillon
453*8345afadSMatthew Dillon KKASSERT(ivdata != NULL);
454*8345afadSMatthew Dillon
455*8345afadSMatthew Dillon ivdata->ivpriv = ivpriv;
456*8345afadSMatthew Dillon ivdata->opaque = opaque;
457*8345afadSMatthew Dillon crp = &ivdata->crp;
458*8345afadSMatthew Dillon crd = &ivdata->crd;
459f603807bSTomohiro Kusumi
460f603807bSTomohiro Kusumi bzero(iv, iv_len);
461f603807bSTomohiro Kusumi bzero(crd, sizeof(struct cryptodesc));
462f603807bSTomohiro Kusumi bzero(crp, sizeof(struct cryptop));
463f603807bSTomohiro Kusumi *((off_t *)iv) = htole64(sector + priv->iv_offset);
464f603807bSTomohiro Kusumi crp->crp_buf = (caddr_t)iv;
465f603807bSTomohiro Kusumi
466f603807bSTomohiro Kusumi crp->crp_sid = ivpriv->crypto_sid;
467f603807bSTomohiro Kusumi crp->crp_ilen = crp->crp_olen = iv_len;
468f603807bSTomohiro Kusumi
469*8345afadSMatthew Dillon crp->crp_opaque = (caddr_t)ivdata;
470f603807bSTomohiro Kusumi
471f603807bSTomohiro Kusumi crp->crp_callback = essiv_ivgen_done;
472f603807bSTomohiro Kusumi
473f603807bSTomohiro Kusumi crp->crp_desc = crd;
474f603807bSTomohiro Kusumi crp->crp_etype = 0;
475f603807bSTomohiro Kusumi crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
476f603807bSTomohiro Kusumi
477f603807bSTomohiro Kusumi crd->crd_alg = priv->crypto_alg;
478f603807bSTomohiro Kusumi #if 0
479f603807bSTomohiro Kusumi crd->crd_key = (caddr_t)priv->crypto_keyhash;
480f603807bSTomohiro Kusumi crd->crd_klen = priv->crypto_klen;
481f603807bSTomohiro Kusumi #endif
482f603807bSTomohiro Kusumi
483f603807bSTomohiro Kusumi bzero(crd->crd_iv, sizeof(crd->crd_iv));
484f603807bSTomohiro Kusumi
485f603807bSTomohiro Kusumi crd->crd_skip = 0;
486f603807bSTomohiro Kusumi crd->crd_len = iv_len;
487f603807bSTomohiro Kusumi crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
488f603807bSTomohiro Kusumi crd->crd_flags |= CRD_F_ENCRYPT;
489f603807bSTomohiro Kusumi crd->crd_next = NULL;
490f603807bSTomohiro Kusumi
491f603807bSTomohiro Kusumi error = crypto_dispatch(crp);
492f603807bSTomohiro Kusumi if (error)
493f603807bSTomohiro Kusumi kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
494f603807bSTomohiro Kusumi }
495f603807bSTomohiro Kusumi
496f603807bSTomohiro Kusumi
497f603807bSTomohiro Kusumi static void
plain_ivgen(dm_target_crypt_config_t * priv,u_int8_t * iv,size_t iv_len,off_t sector,void * opaque)498f603807bSTomohiro Kusumi plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
499f603807bSTomohiro Kusumi size_t iv_len, off_t sector, void *opaque)
500f603807bSTomohiro Kusumi {
501f603807bSTomohiro Kusumi bzero(iv, iv_len);
502f603807bSTomohiro Kusumi *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
503f603807bSTomohiro Kusumi dmtc_crypto_dispatch(opaque);
504f603807bSTomohiro Kusumi }
505f603807bSTomohiro Kusumi
506f603807bSTomohiro Kusumi static void
plain64_ivgen(dm_target_crypt_config_t * priv,u_int8_t * iv,size_t iv_len,off_t sector,void * opaque)507f603807bSTomohiro Kusumi plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
508f603807bSTomohiro Kusumi size_t iv_len, off_t sector, void *opaque)
509f603807bSTomohiro Kusumi {
510f603807bSTomohiro Kusumi bzero(iv, iv_len);
511f603807bSTomohiro Kusumi *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
512f603807bSTomohiro Kusumi dmtc_crypto_dispatch(opaque);
513f603807bSTomohiro Kusumi }
514f603807bSTomohiro Kusumi
515f603807bSTomohiro Kusumi #if 0
516f603807bSTomohiro Kusumi static void
517f603807bSTomohiro Kusumi geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
518f603807bSTomohiro Kusumi size_t iv_len, off_t sector, void *opaque)
519f603807bSTomohiro Kusumi {
520f603807bSTomohiro Kusumi
521f603807bSTomohiro Kusumi SHA512_CTX ctx512;
522f603807bSTomohiro Kusumi u_int8_t md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
523f603807bSTomohiro Kusumi
524f603807bSTomohiro Kusumi memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
525f603807bSTomohiro Kusumi SHA512_Update(&ctx512, (u_int8_t*)§or, sizeof(off_t));
526f603807bSTomohiro Kusumi SHA512_Final(md, &ctx512);
527f603807bSTomohiro Kusumi
528f603807bSTomohiro Kusumi memcpy(iv, md, iv_len);
529f603807bSTomohiro Kusumi dmtc_crypto_dispatch(opaque);
530f603807bSTomohiro Kusumi }
531f603807bSTomohiro Kusumi #endif
532f603807bSTomohiro Kusumi
533f603807bSTomohiro Kusumi /*
534f603807bSTomohiro Kusumi * Init function called from dm_table_load_ioctl.
535f603807bSTomohiro Kusumi * cryptsetup actually passes us this:
536f603807bSTomohiro Kusumi * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
537f603807bSTomohiro Kusumi */
538f603807bSTomohiro Kusumi static int
hex2key(char * hex,size_t key_len,u_int8_t * key)539f603807bSTomohiro Kusumi hex2key(char *hex, size_t key_len, u_int8_t *key)
540f603807bSTomohiro Kusumi {
541f603807bSTomohiro Kusumi char hex_buf[3];
542f603807bSTomohiro Kusumi size_t key_idx;
543f603807bSTomohiro Kusumi
544f603807bSTomohiro Kusumi hex_buf[2] = 0;
545f603807bSTomohiro Kusumi for (key_idx = 0; key_idx < key_len; ++key_idx) {
546f603807bSTomohiro Kusumi hex_buf[0] = *hex++;
547f603807bSTomohiro Kusumi hex_buf[1] = *hex++;
548f603807bSTomohiro Kusumi key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
549f603807bSTomohiro Kusumi }
550f603807bSTomohiro Kusumi hex_buf[0] = 0;
551f603807bSTomohiro Kusumi hex_buf[1] = 0;
552f603807bSTomohiro Kusumi
553f603807bSTomohiro Kusumi return 0;
554f603807bSTomohiro Kusumi }
555f603807bSTomohiro Kusumi
556f603807bSTomohiro Kusumi static int
dm_target_crypt_init(dm_table_entry_t * table_en,int argc,char ** argv)557f603807bSTomohiro Kusumi dm_target_crypt_init(dm_table_entry_t *table_en, int argc, char **argv)
558f603807bSTomohiro Kusumi {
559f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
560f603807bSTomohiro Kusumi size_t len;
561f603807bSTomohiro Kusumi char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
562f603807bSTomohiro Kusumi char *status_str;
563f603807bSTomohiro Kusumi int i, klen, error;
564f603807bSTomohiro Kusumi uint64_t iv_offset, block_offset;
565f603807bSTomohiro Kusumi
566f603807bSTomohiro Kusumi if (argc != 5) {
567f603807bSTomohiro Kusumi kprintf("dm_target_crypt: not enough arguments, "
568f603807bSTomohiro Kusumi "need exactly 5\n");
569f603807bSTomohiro Kusumi return EINVAL;
570f603807bSTomohiro Kusumi }
571f603807bSTomohiro Kusumi
572f603807bSTomohiro Kusumi len = 0;
573f603807bSTomohiro Kusumi for (i = 0; i < argc; i++) {
574f603807bSTomohiro Kusumi len += strlen(argv[i]);
575f603807bSTomohiro Kusumi len++;
576f603807bSTomohiro Kusumi }
577f603807bSTomohiro Kusumi /* len is strlen() of input string +1 */
578f603807bSTomohiro Kusumi status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
579f603807bSTomohiro Kusumi
580f603807bSTomohiro Kusumi crypto_alg = strsep(&argv[0], "-");
581f603807bSTomohiro Kusumi crypto_mode = strsep(&argv[0], "-");
582f603807bSTomohiro Kusumi iv_opt = strsep(&argv[0], "-");
583f603807bSTomohiro Kusumi iv_mode = strsep(&iv_opt, ":");
584f603807bSTomohiro Kusumi key = argv[1];
585f603807bSTomohiro Kusumi iv_offset = strtouq(argv[2], NULL, 0);
586f603807bSTomohiro Kusumi dev = argv[3];
587f603807bSTomohiro Kusumi block_offset = strtouq(argv[4], NULL, 0);
588f603807bSTomohiro Kusumi /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
589f603807bSTomohiro Kusumi klen = strlen(key) << 2;
590f603807bSTomohiro Kusumi
591f603807bSTomohiro Kusumi #if 0
592f603807bSTomohiro Kusumi kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
593f603807bSTomohiro Kusumi "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
594f603807bSTomohiro Kusumi "block_offset=%ju\n",
595f603807bSTomohiro Kusumi dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
596f603807bSTomohiro Kusumi block_offset);
597f603807bSTomohiro Kusumi #endif
598f603807bSTomohiro Kusumi
599f603807bSTomohiro Kusumi priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
600f603807bSTomohiro Kusumi
601f603807bSTomohiro Kusumi /* Insert dmp to global pdev list */
602f603807bSTomohiro Kusumi if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
603f603807bSTomohiro Kusumi kprintf("dm_target_crypt: dm_pdev_insert failed\n");
604f603807bSTomohiro Kusumi kfree(status_str, M_DMCRYPT);
605f603807bSTomohiro Kusumi return ENOENT;
606f603807bSTomohiro Kusumi }
607f603807bSTomohiro Kusumi
608f603807bSTomohiro Kusumi /*
609f603807bSTomohiro Kusumi * This code checks for valid combinations of algorithm and mode.
610f603807bSTomohiro Kusumi * Currently supported options are:
611f603807bSTomohiro Kusumi *
612f603807bSTomohiro Kusumi * *-cbc
613f603807bSTomohiro Kusumi * aes-xts
614f603807bSTomohiro Kusumi * twofish-xts
615f603807bSTomohiro Kusumi * serpent-xts
616f603807bSTomohiro Kusumi */
617f603807bSTomohiro Kusumi if ((strcmp(crypto_mode, "cbc") != 0) &&
618f603807bSTomohiro Kusumi !((strcmp(crypto_mode, "xts") == 0) &&
619f603807bSTomohiro Kusumi ((strcmp(crypto_alg, "aes") == 0) ||
620f603807bSTomohiro Kusumi (strcmp(crypto_alg, "twofish") == 0) ||
621f603807bSTomohiro Kusumi (strcmp(crypto_alg, "serpent") == 0))))
622f603807bSTomohiro Kusumi {
623f603807bSTomohiro Kusumi kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
624f603807bSTomohiro Kusumi " aes-xts, twofish-xts and serpent-xts, "
625f603807bSTomohiro Kusumi "invalid mode '%s-%s'\n",
626f603807bSTomohiro Kusumi crypto_alg, crypto_mode);
627f603807bSTomohiro Kusumi goto notsup;
628f603807bSTomohiro Kusumi }
629f603807bSTomohiro Kusumi
630f603807bSTomohiro Kusumi if (!strcmp(crypto_alg, "aes")) {
631f603807bSTomohiro Kusumi if (!strcmp(crypto_mode, "xts")) {
632f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_AES_XTS;
633f603807bSTomohiro Kusumi if (klen != 256 && klen != 512)
634f603807bSTomohiro Kusumi goto notsup;
635f603807bSTomohiro Kusumi } else if (!strcmp(crypto_mode, "cbc")) {
636f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_AES_CBC;
637f603807bSTomohiro Kusumi if (klen != 128 && klen != 192 && klen != 256)
638f603807bSTomohiro Kusumi goto notsup;
639f603807bSTomohiro Kusumi } else {
640f603807bSTomohiro Kusumi goto notsup;
641f603807bSTomohiro Kusumi }
642f603807bSTomohiro Kusumi priv->crypto_klen = klen;
643f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "twofish")) {
644f603807bSTomohiro Kusumi if (!strcmp(crypto_mode, "xts")) {
645f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_TWOFISH_XTS;
646f603807bSTomohiro Kusumi if (klen != 256 && klen != 512)
647f603807bSTomohiro Kusumi goto notsup;
648f603807bSTomohiro Kusumi } else if (!strcmp(crypto_mode, "cbc")) {
649f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_TWOFISH_CBC;
650f603807bSTomohiro Kusumi if (klen != 128 && klen != 192 && klen != 256)
651f603807bSTomohiro Kusumi goto notsup;
652f603807bSTomohiro Kusumi } else {
653f603807bSTomohiro Kusumi goto notsup;
654f603807bSTomohiro Kusumi }
655f603807bSTomohiro Kusumi priv->crypto_klen = klen;
656f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "serpent")) {
657f603807bSTomohiro Kusumi if (!strcmp(crypto_mode, "xts")) {
658f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_SERPENT_XTS;
659f603807bSTomohiro Kusumi if (klen != 256 && klen != 512)
660f603807bSTomohiro Kusumi goto notsup;
661f603807bSTomohiro Kusumi } else if (!strcmp(crypto_mode, "cbc")) {
662f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_SERPENT_CBC;
663f603807bSTomohiro Kusumi if (klen != 128 && klen != 192 && klen != 256)
664f603807bSTomohiro Kusumi goto notsup;
665f603807bSTomohiro Kusumi } else {
666f603807bSTomohiro Kusumi goto notsup;
667f603807bSTomohiro Kusumi }
668f603807bSTomohiro Kusumi priv->crypto_klen = klen;
669f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "blowfish")) {
670f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_BLF_CBC;
671f603807bSTomohiro Kusumi if (klen < 128 || klen > 448 || (klen % 8) != 0)
672f603807bSTomohiro Kusumi goto notsup;
673f603807bSTomohiro Kusumi priv->crypto_klen = klen;
674f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "3des") ||
675f603807bSTomohiro Kusumi !strncmp(crypto_alg, "des3", 4)) {
676f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_3DES_CBC;
677f603807bSTomohiro Kusumi if (klen != 168)
678f603807bSTomohiro Kusumi goto notsup;
679f603807bSTomohiro Kusumi priv->crypto_klen = 168;
680f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "camellia")) {
681f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
682f603807bSTomohiro Kusumi if (klen != 128 && klen != 192 && klen != 256)
683f603807bSTomohiro Kusumi goto notsup;
684f603807bSTomohiro Kusumi priv->crypto_klen = klen;
685f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "skipjack")) {
686f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
687f603807bSTomohiro Kusumi if (klen != 80)
688f603807bSTomohiro Kusumi goto notsup;
689f603807bSTomohiro Kusumi priv->crypto_klen = 80;
690f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "cast5")) {
691f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_CAST_CBC;
692f603807bSTomohiro Kusumi if (klen != 128)
693f603807bSTomohiro Kusumi goto notsup;
694f603807bSTomohiro Kusumi priv->crypto_klen = 128;
695f603807bSTomohiro Kusumi } else if (!strcmp(crypto_alg, "null")) {
696f603807bSTomohiro Kusumi priv->crypto_alg = CRYPTO_NULL_CBC;
697f603807bSTomohiro Kusumi if (klen != 128)
698f603807bSTomohiro Kusumi goto notsup;
699f603807bSTomohiro Kusumi priv->crypto_klen = 128;
700f603807bSTomohiro Kusumi } else {
701f603807bSTomohiro Kusumi kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
702f603807bSTomohiro Kusumi crypto_alg);
703f603807bSTomohiro Kusumi goto notsup;
704f603807bSTomohiro Kusumi }
705f603807bSTomohiro Kusumi
706f603807bSTomohiro Kusumi /* Save length of param string */
707f603807bSTomohiro Kusumi priv->params_len = len;
708f603807bSTomohiro Kusumi priv->block_offset = block_offset;
709f603807bSTomohiro Kusumi priv->iv_offset = iv_offset - block_offset;
710f603807bSTomohiro Kusumi
711f603807bSTomohiro Kusumi dm_table_add_deps(table_en, priv->pdev);
712f603807bSTomohiro Kusumi
71341a68322STomohiro Kusumi dm_table_init_target(table_en, priv);
714f603807bSTomohiro Kusumi
715f603807bSTomohiro Kusumi error = hex2key(key, priv->crypto_klen >> 3,
716f603807bSTomohiro Kusumi (u_int8_t *)priv->crypto_key);
717f603807bSTomohiro Kusumi
718f603807bSTomohiro Kusumi if (error) {
719f603807bSTomohiro Kusumi kprintf("dm_target_crypt: hex2key failed, "
720f603807bSTomohiro Kusumi "invalid key format\n");
721f603807bSTomohiro Kusumi goto notsup;
722f603807bSTomohiro Kusumi }
723f603807bSTomohiro Kusumi
724f603807bSTomohiro Kusumi /* Handle cmd */
725f603807bSTomohiro Kusumi for(i = 0; ivgens[i].name != NULL; i++) {
726f603807bSTomohiro Kusumi if (!strcmp(iv_mode, ivgens[i].name))
727f603807bSTomohiro Kusumi break;
728f603807bSTomohiro Kusumi }
729f603807bSTomohiro Kusumi
730f603807bSTomohiro Kusumi if (ivgens[i].name == NULL) {
731f603807bSTomohiro Kusumi kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
732f603807bSTomohiro Kusumi iv_mode);
733f603807bSTomohiro Kusumi goto notsup;
734f603807bSTomohiro Kusumi }
735f603807bSTomohiro Kusumi
736f603807bSTomohiro Kusumi /* Call our ivgen constructor */
737f603807bSTomohiro Kusumi if (ivgens[i].ctor != NULL) {
738f603807bSTomohiro Kusumi error = ivgens[i].ctor(priv, iv_opt,
739f603807bSTomohiro Kusumi &priv->ivgen_priv);
740f603807bSTomohiro Kusumi if (error) {
741f603807bSTomohiro Kusumi kprintf("dm_target_crypt: ctor for '%s' failed\n",
742f603807bSTomohiro Kusumi ivgens[i].name);
743f603807bSTomohiro Kusumi goto notsup;
744f603807bSTomohiro Kusumi }
745f603807bSTomohiro Kusumi }
746f603807bSTomohiro Kusumi
747f603807bSTomohiro Kusumi priv->ivgen = &ivgens[i];
748f603807bSTomohiro Kusumi
749f603807bSTomohiro Kusumi priv->crypto_session.cri_alg = priv->crypto_alg;
750f603807bSTomohiro Kusumi priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
751f603807bSTomohiro Kusumi priv->crypto_session.cri_klen = priv->crypto_klen;
752f603807bSTomohiro Kusumi priv->crypto_session.cri_mlen = 0;
753f603807bSTomohiro Kusumi priv->crypto_session.cri_next = NULL;
754f603807bSTomohiro Kusumi
755f603807bSTomohiro Kusumi error = crypto_newsession(&priv->crypto_sid,
756f603807bSTomohiro Kusumi &priv->crypto_session,
757f603807bSTomohiro Kusumi CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
758f603807bSTomohiro Kusumi if (error) {
759f603807bSTomohiro Kusumi kprintf("dm_target_crypt: Error during crypto_newsession, "
760f603807bSTomohiro Kusumi "error = %d\n",
761f603807bSTomohiro Kusumi error);
762f603807bSTomohiro Kusumi goto notsup;
763f603807bSTomohiro Kusumi }
764f603807bSTomohiro Kusumi
765f603807bSTomohiro Kusumi memset(key, '0', strlen(key));
766f603807bSTomohiro Kusumi if (iv_opt) {
767f603807bSTomohiro Kusumi ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
768f603807bSTomohiro Kusumi crypto_alg, crypto_mode, iv_mode, iv_opt,
769f603807bSTomohiro Kusumi key, iv_offset, dev, block_offset);
770f603807bSTomohiro Kusumi } else {
771f603807bSTomohiro Kusumi ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
772f603807bSTomohiro Kusumi crypto_alg, crypto_mode, iv_mode,
773f603807bSTomohiro Kusumi key, iv_offset, dev, block_offset);
774f603807bSTomohiro Kusumi }
775f603807bSTomohiro Kusumi priv->status_str = status_str;
776f603807bSTomohiro Kusumi
777f603807bSTomohiro Kusumi /* Initialize mpipes */
778f603807bSTomohiro Kusumi dmtc_init_mpipe(priv);
779f603807bSTomohiro Kusumi
780f603807bSTomohiro Kusumi return 0;
781f603807bSTomohiro Kusumi
782f603807bSTomohiro Kusumi notsup:
783f603807bSTomohiro Kusumi kprintf("dm_target_crypt: ENOTSUP\n");
784f603807bSTomohiro Kusumi kfree(status_str, M_DMCRYPT);
785f603807bSTomohiro Kusumi return ENOTSUP;
786f603807bSTomohiro Kusumi }
787f603807bSTomohiro Kusumi
788f603807bSTomohiro Kusumi /* Table routine called to get params string. */
789f603807bSTomohiro Kusumi static char *
dm_target_crypt_table(void * target_config)790f603807bSTomohiro Kusumi dm_target_crypt_table(void *target_config)
791f603807bSTomohiro Kusumi {
792f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
793f603807bSTomohiro Kusumi char *params;
794f603807bSTomohiro Kusumi
795f603807bSTomohiro Kusumi priv = target_config;
796f603807bSTomohiro Kusumi
797f603807bSTomohiro Kusumi params = dm_alloc_string(DM_MAX_PARAMS_SIZE);
798f603807bSTomohiro Kusumi
799f603807bSTomohiro Kusumi ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
800f603807bSTomohiro Kusumi priv->status_str);
801f603807bSTomohiro Kusumi
802f603807bSTomohiro Kusumi return params;
803f603807bSTomohiro Kusumi }
804f603807bSTomohiro Kusumi
805f603807bSTomohiro Kusumi static int
dm_target_crypt_destroy(dm_table_entry_t * table_en)806f603807bSTomohiro Kusumi dm_target_crypt_destroy(dm_table_entry_t *table_en)
807f603807bSTomohiro Kusumi {
808f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
809f603807bSTomohiro Kusumi
810f603807bSTomohiro Kusumi /*
811f603807bSTomohiro Kusumi * Disconnect the crypt config before unbusying the target.
812f603807bSTomohiro Kusumi */
813f603807bSTomohiro Kusumi priv = table_en->target_config;
814f603807bSTomohiro Kusumi if (priv == NULL)
815f603807bSTomohiro Kusumi return 0;
816f603807bSTomohiro Kusumi dm_pdev_decr(priv->pdev);
817f603807bSTomohiro Kusumi
818f603807bSTomohiro Kusumi /*
819f603807bSTomohiro Kusumi * Clean up the crypt config
820f603807bSTomohiro Kusumi *
821f603807bSTomohiro Kusumi * Overwrite the private information before freeing memory to
822f603807bSTomohiro Kusumi * avoid leaking it.
823f603807bSTomohiro Kusumi */
824f603807bSTomohiro Kusumi if (priv->status_str) {
825f603807bSTomohiro Kusumi dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
826f603807bSTomohiro Kusumi kfree(priv->status_str, M_DMCRYPT);
827f603807bSTomohiro Kusumi crypto_freesession(priv->crypto_sid);
828f603807bSTomohiro Kusumi }
829f603807bSTomohiro Kusumi
830f603807bSTomohiro Kusumi if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
831f603807bSTomohiro Kusumi priv->ivgen->dtor(priv, priv->ivgen_priv);
832f603807bSTomohiro Kusumi }
833f603807bSTomohiro Kusumi
834f603807bSTomohiro Kusumi /* Destroy mpipes */
835f603807bSTomohiro Kusumi dmtc_destroy_mpipe(priv);
836f603807bSTomohiro Kusumi
837f603807bSTomohiro Kusumi dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
838f603807bSTomohiro Kusumi kfree(priv, M_DMCRYPT);
839f603807bSTomohiro Kusumi
840f603807bSTomohiro Kusumi return 0;
841f603807bSTomohiro Kusumi }
842f603807bSTomohiro Kusumi
843f603807bSTomohiro Kusumi /************************************************************************
844f603807bSTomohiro Kusumi * STRATEGY SUPPORT FUNCTIONS *
845f603807bSTomohiro Kusumi ************************************************************************
846f603807bSTomohiro Kusumi *
847f603807bSTomohiro Kusumi * READ PATH: doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
848f603807bSTomohiro Kusumi * WRITE PATH: crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
849f603807bSTomohiro Kusumi */
850f603807bSTomohiro Kusumi
851f603807bSTomohiro Kusumi /*
852f603807bSTomohiro Kusumi * Wrapper around crypto_dispatch() to match dispatch_t type
853f603807bSTomohiro Kusumi */
854f603807bSTomohiro Kusumi static void
dmtc_crypto_dispatch(void * arg)855f603807bSTomohiro Kusumi dmtc_crypto_dispatch(void *arg)
856f603807bSTomohiro Kusumi {
857f603807bSTomohiro Kusumi struct cryptop *crp;
858f603807bSTomohiro Kusumi
859f603807bSTomohiro Kusumi crp = (struct cryptop *)arg;
860f603807bSTomohiro Kusumi KKASSERT(crp != NULL);
861f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_dispatch, crp);
862f603807bSTomohiro Kusumi crypto_dispatch(crp);
863f603807bSTomohiro Kusumi }
864f603807bSTomohiro Kusumi
865f603807bSTomohiro Kusumi /*
866f603807bSTomohiro Kusumi * Start IO operation, called from dmstrategy routine.
867f603807bSTomohiro Kusumi */
868f603807bSTomohiro Kusumi static int
dm_target_crypt_strategy(dm_table_entry_t * table_en,struct buf * bp)869f603807bSTomohiro Kusumi dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
870f603807bSTomohiro Kusumi {
871f603807bSTomohiro Kusumi struct bio *bio;
872f603807bSTomohiro Kusumi
873f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
874f603807bSTomohiro Kusumi priv = table_en->target_config;
875f603807bSTomohiro Kusumi
876f603807bSTomohiro Kusumi /* Get rid of stuff we can't really handle */
877f603807bSTomohiro Kusumi if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
878f603807bSTomohiro Kusumi if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
879f603807bSTomohiro Kusumi kprintf("dm_target_crypt_strategy: can't really "
880f603807bSTomohiro Kusumi "handle bp->b_bcount = %d\n",
881f603807bSTomohiro Kusumi bp->b_bcount);
882f603807bSTomohiro Kusumi bp->b_error = EINVAL;
883f603807bSTomohiro Kusumi bp->b_flags |= B_ERROR | B_INVAL;
884f603807bSTomohiro Kusumi biodone(&bp->b_bio1);
885f603807bSTomohiro Kusumi return 0;
886f603807bSTomohiro Kusumi }
887f603807bSTomohiro Kusumi }
888f603807bSTomohiro Kusumi
889f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
890f603807bSTomohiro Kusumi
891f603807bSTomohiro Kusumi switch (bp->b_cmd) {
892f603807bSTomohiro Kusumi case BUF_CMD_READ:
893f603807bSTomohiro Kusumi bio = push_bio(&bp->b_bio1);
894f603807bSTomohiro Kusumi bio->bio_offset = bp->b_bio1.bio_offset +
895f603807bSTomohiro Kusumi priv->block_offset * DEV_BSIZE;
896f603807bSTomohiro Kusumi bio->bio_caller_info1.ptr = priv;
897f603807bSTomohiro Kusumi bio->bio_done = dmtc_bio_read_done;
898f603807bSTomohiro Kusumi vn_strategy(priv->pdev->pdev_vnode, bio);
899f603807bSTomohiro Kusumi break;
900f603807bSTomohiro Kusumi case BUF_CMD_WRITE:
901f603807bSTomohiro Kusumi bio = push_bio(&bp->b_bio1);
902f603807bSTomohiro Kusumi bio->bio_offset = bp->b_bio1.bio_offset +
903f603807bSTomohiro Kusumi priv->block_offset * DEV_BSIZE;
904f603807bSTomohiro Kusumi bio->bio_caller_info1.ptr = priv;
905f603807bSTomohiro Kusumi dmtc_crypto_write_start(priv, bio);
906f603807bSTomohiro Kusumi break;
907f603807bSTomohiro Kusumi default:
908f603807bSTomohiro Kusumi vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
909f603807bSTomohiro Kusumi break;
910f603807bSTomohiro Kusumi }
911f603807bSTomohiro Kusumi return 0;
912f603807bSTomohiro Kusumi }
913f603807bSTomohiro Kusumi
914f603807bSTomohiro Kusumi /*
915f603807bSTomohiro Kusumi * STRATEGY READ PATH PART 1/3 (after read BIO completes)
916f603807bSTomohiro Kusumi */
917f603807bSTomohiro Kusumi static void
dmtc_bio_read_done(struct bio * bio)918f603807bSTomohiro Kusumi dmtc_bio_read_done(struct bio *bio)
919f603807bSTomohiro Kusumi {
920f603807bSTomohiro Kusumi struct bio *obio;
921f603807bSTomohiro Kusumi
922f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
923f603807bSTomohiro Kusumi
924f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
925f603807bSTomohiro Kusumi
926f603807bSTomohiro Kusumi /*
927f603807bSTomohiro Kusumi * If a read error occurs we shortcut the operation, otherwise
928f603807bSTomohiro Kusumi * go on to stage 2.
929f603807bSTomohiro Kusumi */
930f603807bSTomohiro Kusumi if (bio->bio_buf->b_flags & B_ERROR) {
931f603807bSTomohiro Kusumi obio = pop_bio(bio);
932f603807bSTomohiro Kusumi biodone(obio);
933f603807bSTomohiro Kusumi } else {
934f603807bSTomohiro Kusumi priv = bio->bio_caller_info1.ptr;
935f603807bSTomohiro Kusumi dmtc_crypto_read_start(priv, bio);
936f603807bSTomohiro Kusumi }
937f603807bSTomohiro Kusumi }
938f603807bSTomohiro Kusumi
939f603807bSTomohiro Kusumi /*
940f603807bSTomohiro Kusumi * STRATEGY READ PATH PART 2/3
941f603807bSTomohiro Kusumi */
942f603807bSTomohiro Kusumi static void
dmtc_crypto_read_retry(void * arg1,void * arg2)943f603807bSTomohiro Kusumi dmtc_crypto_read_retry(void *arg1, void *arg2)
944f603807bSTomohiro Kusumi {
945f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv = arg1;
946f603807bSTomohiro Kusumi struct bio *bio = arg2;
947f603807bSTomohiro Kusumi
948f603807bSTomohiro Kusumi dmtc_crypto_read_start(priv, bio);
949f603807bSTomohiro Kusumi }
950f603807bSTomohiro Kusumi
951f603807bSTomohiro Kusumi static void
dmtc_crypto_read_start(dm_target_crypt_config_t * priv,struct bio * bio)952f603807bSTomohiro Kusumi dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
953f603807bSTomohiro Kusumi {
954f603807bSTomohiro Kusumi struct dmtc_helper *dmtc;
955f603807bSTomohiro Kusumi struct cryptodesc *crd;
956f603807bSTomohiro Kusumi struct cryptop *crp;
957f603807bSTomohiro Kusumi int i, bytes, sectors, sz;
958f603807bSTomohiro Kusumi off_t isector;
959f603807bSTomohiro Kusumi u_char *ptr, *space;
960f603807bSTomohiro Kusumi
961f603807bSTomohiro Kusumi /*
962f603807bSTomohiro Kusumi * Note: b_resid no good after read I/O, it will be 0, use
963f603807bSTomohiro Kusumi * b_bcount.
964f603807bSTomohiro Kusumi */
965f603807bSTomohiro Kusumi bytes = bio->bio_buf->b_bcount;
966f603807bSTomohiro Kusumi isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
967f603807bSTomohiro Kusumi sectors = bytes / DEV_BSIZE; /* Number of sectors */
968f603807bSTomohiro Kusumi sz = sectors * (sizeof(*crp) + sizeof(*crd));
969f603807bSTomohiro Kusumi
970f603807bSTomohiro Kusumi /*
971f603807bSTomohiro Kusumi * For reads with bogus page we can't decrypt in place as stuff
972f603807bSTomohiro Kusumi * can get ripped out from under us.
973f603807bSTomohiro Kusumi *
974f603807bSTomohiro Kusumi * XXX actually it looks like we can, and in any case the initial
975f603807bSTomohiro Kusumi * read already completed and threw crypted data into the buffer
976f603807bSTomohiro Kusumi * cache buffer. Disable for now.
977f603807bSTomohiro Kusumi */
978f603807bSTomohiro Kusumi space = mpipe_alloc_callback(&priv->read_mpipe,
979f603807bSTomohiro Kusumi dmtc_crypto_read_retry, priv, bio);
980f603807bSTomohiro Kusumi if (space == NULL)
981f603807bSTomohiro Kusumi return;
982f603807bSTomohiro Kusumi
983f603807bSTomohiro Kusumi dmtc = (struct dmtc_helper *)space;
984f603807bSTomohiro Kusumi dmtc->free_addr = space;
985f603807bSTomohiro Kusumi space += sizeof(struct dmtc_helper);
986f603807bSTomohiro Kusumi dmtc->orig_buf = NULL;
987f603807bSTomohiro Kusumi dmtc->data_buf = bio->bio_buf->b_data;
988f603807bSTomohiro Kusumi dmtc->priv = priv;
989f603807bSTomohiro Kusumi bio->bio_caller_info2.ptr = dmtc;
990f603807bSTomohiro Kusumi bio->bio_buf->b_error = 0;
991f603807bSTomohiro Kusumi
992f603807bSTomohiro Kusumi /*
993f603807bSTomohiro Kusumi * Load crypto descriptors (crp/crd loop)
994f603807bSTomohiro Kusumi */
995f603807bSTomohiro Kusumi bzero(space, sz);
996f603807bSTomohiro Kusumi ptr = space;
997f603807bSTomohiro Kusumi bio->bio_caller_info3.value = sectors;
998f603807bSTomohiro Kusumi cpu_sfence();
999f603807bSTomohiro Kusumi #if 0
1000f603807bSTomohiro Kusumi kprintf("Read, bytes = %d (b_bcount), "
1001f603807bSTomohiro Kusumi "sectors = %d (bio = %p, b_cmd = %d)\n",
1002f603807bSTomohiro Kusumi bytes, sectors, bio, bio->bio_buf->b_cmd);
1003f603807bSTomohiro Kusumi #endif
1004f603807bSTomohiro Kusumi for (i = 0; i < sectors; i++) {
1005f603807bSTomohiro Kusumi crp = (struct cryptop *)ptr;
1006f603807bSTomohiro Kusumi ptr += sizeof(*crp);
1007f603807bSTomohiro Kusumi crd = (struct cryptodesc *)ptr;
1008f603807bSTomohiro Kusumi ptr += sizeof (*crd);
1009f603807bSTomohiro Kusumi
1010f603807bSTomohiro Kusumi crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1011f603807bSTomohiro Kusumi
1012f603807bSTomohiro Kusumi crp->crp_sid = priv->crypto_sid;
1013f603807bSTomohiro Kusumi crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1014f603807bSTomohiro Kusumi
1015f603807bSTomohiro Kusumi crp->crp_opaque = (void *)bio;
1016f603807bSTomohiro Kusumi
1017f603807bSTomohiro Kusumi crp->crp_callback = dmtc_crypto_cb_read_done;
1018f603807bSTomohiro Kusumi crp->crp_desc = crd;
1019f603807bSTomohiro Kusumi crp->crp_etype = 0;
1020f603807bSTomohiro Kusumi crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1021f603807bSTomohiro Kusumi CRYPTO_F_BATCH;
1022f603807bSTomohiro Kusumi
1023f603807bSTomohiro Kusumi crd->crd_alg = priv->crypto_alg;
1024f603807bSTomohiro Kusumi #if 0
1025f603807bSTomohiro Kusumi crd->crd_key = (caddr_t)priv->crypto_key;
1026f603807bSTomohiro Kusumi crd->crd_klen = priv->crypto_klen;
1027f603807bSTomohiro Kusumi #endif
1028f603807bSTomohiro Kusumi
1029f603807bSTomohiro Kusumi crd->crd_skip = 0;
1030f603807bSTomohiro Kusumi crd->crd_len = DEV_BSIZE /* XXX */;
1031f603807bSTomohiro Kusumi crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1032f603807bSTomohiro Kusumi crd->crd_next = NULL;
1033f603807bSTomohiro Kusumi
1034f603807bSTomohiro Kusumi crd->crd_flags &= ~CRD_F_ENCRYPT;
1035f603807bSTomohiro Kusumi
1036f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
1037f603807bSTomohiro Kusumi sectors);
1038f603807bSTomohiro Kusumi
1039f603807bSTomohiro Kusumi /*
1040f603807bSTomohiro Kusumi * Note: last argument is used to generate salt(?) and is
1041f603807bSTomohiro Kusumi * a 64 bit value, but the original code passed an
1042f603807bSTomohiro Kusumi * int. Changing it now will break pre-existing
1043f603807bSTomohiro Kusumi * crypt volumes.
1044f603807bSTomohiro Kusumi */
1045f603807bSTomohiro Kusumi priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1046f603807bSTomohiro Kusumi isector + i, crp);
1047f603807bSTomohiro Kusumi }
1048f603807bSTomohiro Kusumi }
1049f603807bSTomohiro Kusumi
1050f603807bSTomohiro Kusumi /*
1051f603807bSTomohiro Kusumi * STRATEGY READ PATH PART 3/3
1052f603807bSTomohiro Kusumi */
1053f603807bSTomohiro Kusumi static int
dmtc_crypto_cb_read_done(struct cryptop * crp)1054f603807bSTomohiro Kusumi dmtc_crypto_cb_read_done(struct cryptop *crp)
1055f603807bSTomohiro Kusumi {
1056f603807bSTomohiro Kusumi struct dmtc_helper *dmtc;
1057f603807bSTomohiro Kusumi struct bio *bio, *obio;
1058f603807bSTomohiro Kusumi int n;
1059f603807bSTomohiro Kusumi
1060f603807bSTomohiro Kusumi if (crp->crp_etype == EAGAIN)
1061f603807bSTomohiro Kusumi return crypto_dispatch(crp);
1062f603807bSTomohiro Kusumi
1063f603807bSTomohiro Kusumi bio = (struct bio *)crp->crp_opaque;
1064f603807bSTomohiro Kusumi KKASSERT(bio != NULL);
1065f603807bSTomohiro Kusumi
1066f603807bSTomohiro Kusumi /*
1067f603807bSTomohiro Kusumi * Cumulative error
1068f603807bSTomohiro Kusumi */
1069f603807bSTomohiro Kusumi if (crp->crp_etype) {
1070f603807bSTomohiro Kusumi kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1071f603807bSTomohiro Kusumi "crp_etype = %d\n",
1072f603807bSTomohiro Kusumi crp->crp_etype);
1073f603807bSTomohiro Kusumi bio->bio_buf->b_error = crp->crp_etype;
1074f603807bSTomohiro Kusumi }
1075f603807bSTomohiro Kusumi
1076f603807bSTomohiro Kusumi /*
1077f603807bSTomohiro Kusumi * On the last chunk of the decryption we do any required copybacks
1078f603807bSTomohiro Kusumi * and complete the I/O.
1079f603807bSTomohiro Kusumi */
1080f603807bSTomohiro Kusumi n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1081f603807bSTomohiro Kusumi #if 0
1082f603807bSTomohiro Kusumi kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1083f603807bSTomohiro Kusumi #endif
1084f603807bSTomohiro Kusumi
1085f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1086f603807bSTomohiro Kusumi
1087f603807bSTomohiro Kusumi if (n == 1) {
1088f603807bSTomohiro Kusumi /*
1089f603807bSTomohiro Kusumi * For the B_HASBOGUS case we didn't decrypt in place,
1090f603807bSTomohiro Kusumi * so we need to copy stuff back into the buf.
1091f603807bSTomohiro Kusumi *
1092f603807bSTomohiro Kusumi * (disabled for now).
1093f603807bSTomohiro Kusumi */
1094f603807bSTomohiro Kusumi dmtc = bio->bio_caller_info2.ptr;
1095f603807bSTomohiro Kusumi if (bio->bio_buf->b_error) {
1096f603807bSTomohiro Kusumi bio->bio_buf->b_flags |= B_ERROR;
1097f603807bSTomohiro Kusumi }
1098f603807bSTomohiro Kusumi #if 0
1099f603807bSTomohiro Kusumi else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1100f603807bSTomohiro Kusumi memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1101f603807bSTomohiro Kusumi bio->bio_buf->b_bcount);
1102f603807bSTomohiro Kusumi }
1103f603807bSTomohiro Kusumi #endif
1104f603807bSTomohiro Kusumi mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1105f603807bSTomohiro Kusumi obio = pop_bio(bio);
1106f603807bSTomohiro Kusumi biodone(obio);
1107f603807bSTomohiro Kusumi }
1108f603807bSTomohiro Kusumi return 0;
1109f603807bSTomohiro Kusumi }
1110f603807bSTomohiro Kusumi /* END OF STRATEGY READ SECTION */
1111f603807bSTomohiro Kusumi
1112f603807bSTomohiro Kusumi /*
1113f603807bSTomohiro Kusumi * STRATEGY WRITE PATH PART 1/3
1114f603807bSTomohiro Kusumi */
1115f603807bSTomohiro Kusumi
1116f603807bSTomohiro Kusumi static void
dmtc_crypto_write_retry(void * arg1,void * arg2)1117f603807bSTomohiro Kusumi dmtc_crypto_write_retry(void *arg1, void *arg2)
1118f603807bSTomohiro Kusumi {
1119f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv = arg1;
1120f603807bSTomohiro Kusumi struct bio *bio = arg2;
1121f603807bSTomohiro Kusumi
1122f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1123f603807bSTomohiro Kusumi
1124f603807bSTomohiro Kusumi dmtc_crypto_write_start(priv, bio);
1125f603807bSTomohiro Kusumi }
1126f603807bSTomohiro Kusumi
1127f603807bSTomohiro Kusumi static void
dmtc_crypto_write_start(dm_target_crypt_config_t * priv,struct bio * bio)1128f603807bSTomohiro Kusumi dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1129f603807bSTomohiro Kusumi {
1130f603807bSTomohiro Kusumi struct dmtc_helper *dmtc;
1131f603807bSTomohiro Kusumi struct cryptodesc *crd;
1132f603807bSTomohiro Kusumi struct cryptop *crp;
1133f603807bSTomohiro Kusumi int i, bytes, sectors, sz;
1134f603807bSTomohiro Kusumi off_t isector;
1135f603807bSTomohiro Kusumi u_char *ptr, *space;
1136f603807bSTomohiro Kusumi
1137f603807bSTomohiro Kusumi /*
1138f603807bSTomohiro Kusumi * Use b_bcount for consistency
1139f603807bSTomohiro Kusumi */
1140f603807bSTomohiro Kusumi bytes = bio->bio_buf->b_bcount;
1141f603807bSTomohiro Kusumi
1142f603807bSTomohiro Kusumi isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
1143f603807bSTomohiro Kusumi sectors = bytes / DEV_BSIZE; /* Number of sectors */
1144f603807bSTomohiro Kusumi sz = sectors * (sizeof(*crp) + sizeof(*crd));
1145f603807bSTomohiro Kusumi
1146f603807bSTomohiro Kusumi /*
1147f603807bSTomohiro Kusumi * For writes and reads with bogus page don't decrypt in place.
1148f603807bSTomohiro Kusumi */
1149f603807bSTomohiro Kusumi space = mpipe_alloc_callback(&priv->write_mpipe,
1150f603807bSTomohiro Kusumi dmtc_crypto_write_retry, priv, bio);
1151f603807bSTomohiro Kusumi if (space == NULL)
1152f603807bSTomohiro Kusumi return;
1153f603807bSTomohiro Kusumi
1154f603807bSTomohiro Kusumi dmtc = (struct dmtc_helper *)space;
1155f603807bSTomohiro Kusumi dmtc->free_addr = space;
1156f603807bSTomohiro Kusumi space += sizeof(struct dmtc_helper);
1157f603807bSTomohiro Kusumi memcpy(space + sz, bio->bio_buf->b_data, bytes);
1158f603807bSTomohiro Kusumi
1159f603807bSTomohiro Kusumi bio->bio_caller_info2.ptr = dmtc;
1160f603807bSTomohiro Kusumi bio->bio_buf->b_error = 0;
1161f603807bSTomohiro Kusumi
1162f603807bSTomohiro Kusumi dmtc->orig_buf = bio->bio_buf->b_data;
1163f603807bSTomohiro Kusumi dmtc->data_buf = space + sz;
1164f603807bSTomohiro Kusumi dmtc->priv = priv;
1165f603807bSTomohiro Kusumi
1166f603807bSTomohiro Kusumi /*
1167f603807bSTomohiro Kusumi * Load crypto descriptors (crp/crd loop)
1168f603807bSTomohiro Kusumi */
1169f603807bSTomohiro Kusumi bzero(space, sz);
1170f603807bSTomohiro Kusumi ptr = space;
1171f603807bSTomohiro Kusumi bio->bio_caller_info3.value = sectors;
1172f603807bSTomohiro Kusumi cpu_sfence();
1173f603807bSTomohiro Kusumi #if 0
1174f603807bSTomohiro Kusumi kprintf("Write, bytes = %d (b_bcount), "
1175f603807bSTomohiro Kusumi "sectors = %d (bio = %p, b_cmd = %d)\n",
1176f603807bSTomohiro Kusumi bytes, sectors, bio, bio->bio_buf->b_cmd);
1177f603807bSTomohiro Kusumi #endif
1178f603807bSTomohiro Kusumi for (i = 0; i < sectors; i++) {
1179f603807bSTomohiro Kusumi crp = (struct cryptop *)ptr;
1180f603807bSTomohiro Kusumi ptr += sizeof(*crp);
1181f603807bSTomohiro Kusumi crd = (struct cryptodesc *)ptr;
1182f603807bSTomohiro Kusumi ptr += sizeof (*crd);
1183f603807bSTomohiro Kusumi
1184f603807bSTomohiro Kusumi crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1185f603807bSTomohiro Kusumi
1186f603807bSTomohiro Kusumi crp->crp_sid = priv->crypto_sid;
1187f603807bSTomohiro Kusumi crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1188f603807bSTomohiro Kusumi
1189f603807bSTomohiro Kusumi crp->crp_opaque = (void *)bio;
1190f603807bSTomohiro Kusumi
1191f603807bSTomohiro Kusumi crp->crp_callback = dmtc_crypto_cb_write_done;
1192f603807bSTomohiro Kusumi crp->crp_desc = crd;
1193f603807bSTomohiro Kusumi crp->crp_etype = 0;
1194f603807bSTomohiro Kusumi crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1195f603807bSTomohiro Kusumi CRYPTO_F_BATCH;
1196f603807bSTomohiro Kusumi
1197f603807bSTomohiro Kusumi crd->crd_alg = priv->crypto_alg;
1198f603807bSTomohiro Kusumi #if 0
1199f603807bSTomohiro Kusumi crd->crd_key = (caddr_t)priv->crypto_key;
1200f603807bSTomohiro Kusumi crd->crd_klen = priv->crypto_klen;
1201f603807bSTomohiro Kusumi #endif
1202f603807bSTomohiro Kusumi
1203f603807bSTomohiro Kusumi crd->crd_skip = 0;
1204f603807bSTomohiro Kusumi crd->crd_len = DEV_BSIZE /* XXX */;
1205f603807bSTomohiro Kusumi crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1206f603807bSTomohiro Kusumi crd->crd_next = NULL;
1207f603807bSTomohiro Kusumi
1208f603807bSTomohiro Kusumi crd->crd_flags |= CRD_F_ENCRYPT;
1209f603807bSTomohiro Kusumi
1210f603807bSTomohiro Kusumi /*
1211f603807bSTomohiro Kusumi * Note: last argument is used to generate salt(?) and is
1212f603807bSTomohiro Kusumi * a 64 bit value, but the original code passed an
1213f603807bSTomohiro Kusumi * int. Changing it now will break pre-existing
1214f603807bSTomohiro Kusumi * crypt volumes.
1215f603807bSTomohiro Kusumi */
1216f603807bSTomohiro Kusumi
1217f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1218f603807bSTomohiro Kusumi i, sectors);
1219f603807bSTomohiro Kusumi
1220f603807bSTomohiro Kusumi priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1221f603807bSTomohiro Kusumi isector + i, crp);
1222f603807bSTomohiro Kusumi }
1223f603807bSTomohiro Kusumi }
1224f603807bSTomohiro Kusumi
1225f603807bSTomohiro Kusumi /*
1226f603807bSTomohiro Kusumi * STRATEGY WRITE PATH PART 2/3
1227f603807bSTomohiro Kusumi */
1228f603807bSTomohiro Kusumi static int
dmtc_crypto_cb_write_done(struct cryptop * crp)1229f603807bSTomohiro Kusumi dmtc_crypto_cb_write_done(struct cryptop *crp)
1230f603807bSTomohiro Kusumi {
1231f603807bSTomohiro Kusumi struct dmtc_helper *dmtc;
1232f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
1233f603807bSTomohiro Kusumi struct bio *bio, *obio;
1234f603807bSTomohiro Kusumi int n;
1235f603807bSTomohiro Kusumi
1236f603807bSTomohiro Kusumi if (crp->crp_etype == EAGAIN)
1237f603807bSTomohiro Kusumi return crypto_dispatch(crp);
1238f603807bSTomohiro Kusumi
1239f603807bSTomohiro Kusumi bio = (struct bio *)crp->crp_opaque;
1240f603807bSTomohiro Kusumi KKASSERT(bio != NULL);
1241f603807bSTomohiro Kusumi
1242f603807bSTomohiro Kusumi /*
1243f603807bSTomohiro Kusumi * Cumulative error
1244f603807bSTomohiro Kusumi */
1245f603807bSTomohiro Kusumi if (crp->crp_etype != 0) {
1246f603807bSTomohiro Kusumi kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1247f603807bSTomohiro Kusumi "crp_etype = %d\n",
1248f603807bSTomohiro Kusumi crp->crp_etype);
1249f603807bSTomohiro Kusumi bio->bio_buf->b_error = crp->crp_etype;
1250f603807bSTomohiro Kusumi }
1251f603807bSTomohiro Kusumi
1252f603807bSTomohiro Kusumi /*
1253f603807bSTomohiro Kusumi * On the last chunk of the encryption we issue the write
1254f603807bSTomohiro Kusumi */
1255f603807bSTomohiro Kusumi n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1256f603807bSTomohiro Kusumi #if 0
1257f603807bSTomohiro Kusumi kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1258f603807bSTomohiro Kusumi #endif
1259f603807bSTomohiro Kusumi
1260f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1261f603807bSTomohiro Kusumi
1262f603807bSTomohiro Kusumi if (n == 1) {
1263f603807bSTomohiro Kusumi dmtc = bio->bio_caller_info2.ptr;
1264f603807bSTomohiro Kusumi priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1265f603807bSTomohiro Kusumi
1266f603807bSTomohiro Kusumi if (bio->bio_buf->b_error) {
1267f603807bSTomohiro Kusumi bio->bio_buf->b_flags |= B_ERROR;
1268f603807bSTomohiro Kusumi mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1269f603807bSTomohiro Kusumi obio = pop_bio(bio);
1270f603807bSTomohiro Kusumi biodone(obio);
1271f603807bSTomohiro Kusumi } else {
1272f603807bSTomohiro Kusumi dmtc->orig_buf = bio->bio_buf->b_data;
1273f603807bSTomohiro Kusumi bio->bio_buf->b_data = dmtc->data_buf;
1274f603807bSTomohiro Kusumi bio->bio_done = dmtc_bio_write_done;
1275f603807bSTomohiro Kusumi vn_strategy(priv->pdev->pdev_vnode, bio);
1276f603807bSTomohiro Kusumi }
1277f603807bSTomohiro Kusumi }
1278f603807bSTomohiro Kusumi return 0;
1279f603807bSTomohiro Kusumi }
1280f603807bSTomohiro Kusumi
1281f603807bSTomohiro Kusumi /*
1282f603807bSTomohiro Kusumi * STRATEGY WRITE PATH PART 3/3
1283f603807bSTomohiro Kusumi */
1284f603807bSTomohiro Kusumi static void
dmtc_bio_write_done(struct bio * bio)1285f603807bSTomohiro Kusumi dmtc_bio_write_done(struct bio *bio)
1286f603807bSTomohiro Kusumi {
1287f603807bSTomohiro Kusumi struct dmtc_helper *dmtc;
1288f603807bSTomohiro Kusumi struct bio *obio;
1289f603807bSTomohiro Kusumi
1290f603807bSTomohiro Kusumi dmtc = bio->bio_caller_info2.ptr;
1291f603807bSTomohiro Kusumi bio->bio_buf->b_data = dmtc->orig_buf;
1292f603807bSTomohiro Kusumi mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1293f603807bSTomohiro Kusumi
1294f603807bSTomohiro Kusumi KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1295f603807bSTomohiro Kusumi
1296f603807bSTomohiro Kusumi obio = pop_bio(bio);
1297f603807bSTomohiro Kusumi biodone(obio);
1298f603807bSTomohiro Kusumi }
1299f603807bSTomohiro Kusumi /* END OF STRATEGY WRITE SECTION */
1300f603807bSTomohiro Kusumi
1301f603807bSTomohiro Kusumi
1302f603807bSTomohiro Kusumi
1303f603807bSTomohiro Kusumi /* DUMPING MAGIC */
1304f603807bSTomohiro Kusumi
1305f603807bSTomohiro Kusumi extern int tsleep_crypto_dump;
1306f603807bSTomohiro Kusumi
1307f603807bSTomohiro Kusumi static int
dm_target_crypt_dump(dm_table_entry_t * table_en,void * data,size_t length,off_t offset)1308f603807bSTomohiro Kusumi dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1309f603807bSTomohiro Kusumi {
1310f603807bSTomohiro Kusumi static struct dmtc_dump_helper dump_helper;
1311f603807bSTomohiro Kusumi dm_target_crypt_config_t *priv;
1312f603807bSTomohiro Kusumi int id;
1313f603807bSTomohiro Kusumi static int first_call = 1;
1314f603807bSTomohiro Kusumi
1315f603807bSTomohiro Kusumi priv = table_en->target_config;
1316f603807bSTomohiro Kusumi
1317f603807bSTomohiro Kusumi if (first_call) {
1318f603807bSTomohiro Kusumi first_call = 0;
1319f603807bSTomohiro Kusumi dump_reactivate_cpus();
1320f603807bSTomohiro Kusumi }
1321f603807bSTomohiro Kusumi
1322f603807bSTomohiro Kusumi /* Magically enable tsleep */
1323f603807bSTomohiro Kusumi tsleep_crypto_dump = 1;
1324f603807bSTomohiro Kusumi id = 0;
1325f603807bSTomohiro Kusumi
1326f603807bSTomohiro Kusumi /*
1327f603807bSTomohiro Kusumi * 0 length means flush buffers and return
1328f603807bSTomohiro Kusumi */
1329f603807bSTomohiro Kusumi if (length == 0) {
1330f603807bSTomohiro Kusumi if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1331f603807bSTomohiro Kusumi tsleep_crypto_dump = 0;
1332f603807bSTomohiro Kusumi return ENXIO;
1333f603807bSTomohiro Kusumi }
1334f603807bSTomohiro Kusumi dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1335f603807bSTomohiro Kusumi data, 0, offset, 0);
1336f603807bSTomohiro Kusumi tsleep_crypto_dump = 0;
1337f603807bSTomohiro Kusumi return 0;
1338f603807bSTomohiro Kusumi }
1339f603807bSTomohiro Kusumi
1340f603807bSTomohiro Kusumi bzero(&dump_helper, sizeof(dump_helper));
1341f603807bSTomohiro Kusumi dump_helper.priv = priv;
1342f603807bSTomohiro Kusumi dump_helper.data = data;
1343f603807bSTomohiro Kusumi dump_helper.length = length;
1344f603807bSTomohiro Kusumi dump_helper.offset = offset +
1345f603807bSTomohiro Kusumi priv->block_offset * DEV_BSIZE;
1346f603807bSTomohiro Kusumi dump_helper.ident = &id;
1347f603807bSTomohiro Kusumi dmtc_crypto_dump_start(priv, &dump_helper);
1348f603807bSTomohiro Kusumi
1349f603807bSTomohiro Kusumi /*
1350f603807bSTomohiro Kusumi * Hackery to make stuff appear synchronous. The crypto callback will
1351f603807bSTomohiro Kusumi * set id to 1 and call wakeup on it. If the request completed
1352f603807bSTomohiro Kusumi * synchronously, id will be 1 and we won't bother to sleep. If not,
1353f603807bSTomohiro Kusumi * the crypto request will complete asynchronously and we sleep until
1354f603807bSTomohiro Kusumi * it's done.
1355f603807bSTomohiro Kusumi */
1356f603807bSTomohiro Kusumi if (id == 0)
1357f603807bSTomohiro Kusumi tsleep(&dump_helper, 0, "cryptdump", 0);
1358f603807bSTomohiro Kusumi
1359f603807bSTomohiro Kusumi dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1360f603807bSTomohiro Kusumi dump_helper.offset);
1361f603807bSTomohiro Kusumi
1362f603807bSTomohiro Kusumi dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1363f603807bSTomohiro Kusumi dump_helper.space, 0, dump_helper.offset,
1364f603807bSTomohiro Kusumi dump_helper.length);
1365f603807bSTomohiro Kusumi
1366f603807bSTomohiro Kusumi tsleep_crypto_dump = 0;
1367f603807bSTomohiro Kusumi return 0;
1368f603807bSTomohiro Kusumi }
1369f603807bSTomohiro Kusumi
1370f603807bSTomohiro Kusumi static void
dmtc_crypto_dump_start(dm_target_crypt_config_t * priv,struct dmtc_dump_helper * dump_helper)1371f603807bSTomohiro Kusumi dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1372f603807bSTomohiro Kusumi {
1373f603807bSTomohiro Kusumi struct cryptodesc *crd;
1374f603807bSTomohiro Kusumi struct cryptop *crp;
1375f603807bSTomohiro Kusumi int i, bytes, sectors;
1376f603807bSTomohiro Kusumi off_t isector;
1377f603807bSTomohiro Kusumi
1378f603807bSTomohiro Kusumi bytes = dump_helper->length;
1379f603807bSTomohiro Kusumi
1380f603807bSTomohiro Kusumi isector = dump_helper->offset / DEV_BSIZE; /* ivgen salt base? */
1381f603807bSTomohiro Kusumi sectors = bytes / DEV_BSIZE; /* Number of sectors */
1382f603807bSTomohiro Kusumi dump_helper->sectors = sectors;
1383f603807bSTomohiro Kusumi #if 0
1384f603807bSTomohiro Kusumi kprintf("Dump, bytes = %d, "
1385f603807bSTomohiro Kusumi "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1386f603807bSTomohiro Kusumi #endif
1387f603807bSTomohiro Kusumi KKASSERT(dump_helper->length <= 65536);
1388f603807bSTomohiro Kusumi
1389f603807bSTomohiro Kusumi memcpy(dump_helper->space, dump_helper->data, bytes);
1390f603807bSTomohiro Kusumi
1391f603807bSTomohiro Kusumi cpu_sfence();
1392f603807bSTomohiro Kusumi
1393f603807bSTomohiro Kusumi for (i = 0; i < sectors; i++) {
1394f603807bSTomohiro Kusumi crp = &dump_helper->crp[i];
1395f603807bSTomohiro Kusumi crd = &dump_helper->crd[i];
1396f603807bSTomohiro Kusumi
1397f603807bSTomohiro Kusumi crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1398f603807bSTomohiro Kusumi
1399f603807bSTomohiro Kusumi crp->crp_sid = priv->crypto_sid;
1400f603807bSTomohiro Kusumi crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1401f603807bSTomohiro Kusumi
1402f603807bSTomohiro Kusumi crp->crp_opaque = (void *)dump_helper;
1403f603807bSTomohiro Kusumi
1404f603807bSTomohiro Kusumi crp->crp_callback = dmtc_crypto_cb_dump_done;
1405f603807bSTomohiro Kusumi crp->crp_desc = crd;
1406f603807bSTomohiro Kusumi crp->crp_etype = 0;
1407f603807bSTomohiro Kusumi crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1408f603807bSTomohiro Kusumi CRYPTO_F_BATCH;
1409f603807bSTomohiro Kusumi
1410f603807bSTomohiro Kusumi crd->crd_alg = priv->crypto_alg;
1411f603807bSTomohiro Kusumi
1412f603807bSTomohiro Kusumi crd->crd_skip = 0;
1413f603807bSTomohiro Kusumi crd->crd_len = DEV_BSIZE /* XXX */;
1414f603807bSTomohiro Kusumi crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1415f603807bSTomohiro Kusumi crd->crd_next = NULL;
1416f603807bSTomohiro Kusumi
1417f603807bSTomohiro Kusumi crd->crd_flags |= CRD_F_ENCRYPT;
1418f603807bSTomohiro Kusumi
1419f603807bSTomohiro Kusumi /*
1420f603807bSTomohiro Kusumi * Note: last argument is used to generate salt(?) and is
1421f603807bSTomohiro Kusumi * a 64 bit value, but the original code passed an
1422f603807bSTomohiro Kusumi * int. Changing it now will break pre-existing
1423f603807bSTomohiro Kusumi * crypt volumes.
1424f603807bSTomohiro Kusumi */
1425f603807bSTomohiro Kusumi priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1426f603807bSTomohiro Kusumi isector + i, crp);
1427f603807bSTomohiro Kusumi }
1428f603807bSTomohiro Kusumi }
1429f603807bSTomohiro Kusumi
1430f603807bSTomohiro Kusumi static int
dmtc_crypto_cb_dump_done(struct cryptop * crp)1431f603807bSTomohiro Kusumi dmtc_crypto_cb_dump_done(struct cryptop *crp)
1432f603807bSTomohiro Kusumi {
1433f603807bSTomohiro Kusumi struct dmtc_dump_helper *dump_helper;
1434f603807bSTomohiro Kusumi int n;
1435f603807bSTomohiro Kusumi
1436f603807bSTomohiro Kusumi if (crp->crp_etype == EAGAIN)
1437f603807bSTomohiro Kusumi return crypto_dispatch(crp);
1438f603807bSTomohiro Kusumi
1439f603807bSTomohiro Kusumi dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1440f603807bSTomohiro Kusumi KKASSERT(dump_helper != NULL);
1441f603807bSTomohiro Kusumi
1442f603807bSTomohiro Kusumi if (crp->crp_etype != 0) {
1443f603807bSTomohiro Kusumi kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1444f603807bSTomohiro Kusumi "crp_etype = %d\n",
1445f603807bSTomohiro Kusumi crp->crp_etype);
1446f603807bSTomohiro Kusumi return crp->crp_etype;
1447f603807bSTomohiro Kusumi }
1448f603807bSTomohiro Kusumi
1449f603807bSTomohiro Kusumi /*
1450f603807bSTomohiro Kusumi * On the last chunk of the encryption we return control
1451f603807bSTomohiro Kusumi */
1452f603807bSTomohiro Kusumi n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1453f603807bSTomohiro Kusumi
1454f603807bSTomohiro Kusumi if (n == 1) {
1455f603807bSTomohiro Kusumi atomic_add_int(dump_helper->ident, 1);
1456f603807bSTomohiro Kusumi wakeup(dump_helper);
1457f603807bSTomohiro Kusumi }
1458f603807bSTomohiro Kusumi
1459f603807bSTomohiro Kusumi return 0;
1460f603807bSTomohiro Kusumi }
1461f603807bSTomohiro Kusumi
1462f603807bSTomohiro Kusumi static int
dmtc_mod_handler(module_t mod,int type,void * unused)1463f603807bSTomohiro Kusumi dmtc_mod_handler(module_t mod, int type, void *unused)
1464f603807bSTomohiro Kusumi {
1465f603807bSTomohiro Kusumi dm_target_t *dmt = NULL;
1466f603807bSTomohiro Kusumi int err = 0;
1467f603807bSTomohiro Kusumi
1468f603807bSTomohiro Kusumi switch (type) {
1469f603807bSTomohiro Kusumi case MOD_LOAD:
1470f603807bSTomohiro Kusumi if ((dmt = dm_target_lookup("crypt")) != NULL) {
1471f603807bSTomohiro Kusumi dm_target_unbusy(dmt);
1472f603807bSTomohiro Kusumi return EEXIST;
1473f603807bSTomohiro Kusumi }
1474f603807bSTomohiro Kusumi dmt = dm_target_alloc("crypt");
1475f603807bSTomohiro Kusumi dmt->version[0] = 1;
1476f603807bSTomohiro Kusumi dmt->version[1] = 6;
1477f603807bSTomohiro Kusumi dmt->version[2] = 0;
1478f603807bSTomohiro Kusumi dmt->init = &dm_target_crypt_init;
1479f603807bSTomohiro Kusumi dmt->destroy = &dm_target_crypt_destroy;
1480f603807bSTomohiro Kusumi dmt->strategy = &dm_target_crypt_strategy;
1481f603807bSTomohiro Kusumi dmt->table = &dm_target_crypt_table;
1482f603807bSTomohiro Kusumi dmt->dump = &dm_target_crypt_dump;
1483f603807bSTomohiro Kusumi
1484f603807bSTomohiro Kusumi err = dm_target_insert(dmt);
1485f603807bSTomohiro Kusumi if (!err)
1486f603807bSTomohiro Kusumi kprintf("dm_target_crypt: Successfully initialized\n");
1487f603807bSTomohiro Kusumi break;
1488f603807bSTomohiro Kusumi
1489f603807bSTomohiro Kusumi case MOD_UNLOAD:
1490f603807bSTomohiro Kusumi err = dm_target_remove("crypt");
1491f603807bSTomohiro Kusumi if (err == 0) {
1492f603807bSTomohiro Kusumi kprintf("dm_target_crypt: unloaded\n");
1493f603807bSTomohiro Kusumi }
1494f603807bSTomohiro Kusumi break;
1495f603807bSTomohiro Kusumi }
1496f603807bSTomohiro Kusumi
1497f603807bSTomohiro Kusumi return err;
1498f603807bSTomohiro Kusumi }
1499f603807bSTomohiro Kusumi
1500f603807bSTomohiro Kusumi DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);
1501