1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/linker.h> 34 #include <sys/module.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/bio.h> 38 #include <sys/sysctl.h> 39 #include <sys/kthread.h> 40 #include <sys/proc.h> 41 #include <sys/sched.h> 42 #include <sys/smp.h> 43 #include <sys/vnode.h> 44 45 #include <vm/uma.h> 46 47 #include <geom/geom.h> 48 #include <geom/geom_dbg.h> 49 #include <geom/eli/g_eli.h> 50 #include <geom/eli/pkcs5v2.h> 51 52 /* 53 * The data layout description when integrity verification is configured. 54 * 55 * One of the most important assumption here is that authenticated data and its 56 * HMAC has to be stored in the same place (namely in the same sector) to make 57 * it work reliable. 58 * The problem is that file systems work only with sectors that are multiple of 59 * 512 bytes and a power of two number. 60 * My idea to implement it is as follows. 61 * Let's store HMAC in sector. This is a must. This leaves us 480 bytes for 62 * data. We can't use that directly (ie. we can't create provider with 480 bytes 63 * sector size). We need another sector from where we take only 32 bytes of data 64 * and we store HMAC of this data as well. This takes two sectors from the 65 * original provider at the input and leaves us one sector of authenticated data 66 * at the output. Not very efficient, but you got the idea. 67 * Now, let's assume, we want to create provider with 4096 bytes sector. 68 * To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we 69 * need nine 512-bytes sectors at the input to get one 4096-bytes sector at the 70 * output. That's better. With 4096 bytes sector we can use 89% of size of the 71 * original provider. I find it as an acceptable cost. 72 * The reliability comes from the fact, that every HMAC stored inside the sector 73 * is calculated only for the data in the same sector, so its impossible to 74 * write new data and leave old HMAC or vice versa. 75 * 76 * And here is the picture: 77 * 78 * da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ 79 * |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |256b | 80 * |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data | 81 * +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+ 82 * |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |288 bytes | 83 * +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ |224 unused| 84 * +----------+ 85 * da0.eli: +----+----+----+----+----+----+----+----+----+ 86 * |480b|480b|480b|480b|480b|480b|480b|480b|256b| 87 * +----+----+----+----+----+----+----+----+----+ 88 * | 4096 bytes | 89 * +--------------------------------------------+ 90 * 91 * PS. You can use any sector size with geli(8). My example is using 4kB, 92 * because it's most efficient. For 8kB sectors you need 2 extra sectors, 93 * so the cost is the same as for 4kB sectors. 94 */ 95 96 /* 97 * Code paths: 98 * BIO_READ: 99 * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver 100 * BIO_WRITE: 101 * g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 102 */ 103 104 /* 105 * Here we generate key for HMAC. Every sector has its own HMAC key, so it is 106 * not possible to copy sectors. 107 * We cannot depend on fact, that every sector has its own IV, because different 108 * IV doesn't change HMAC, when we use encrypt-then-authenticate method. 109 */ 110 static void 111 g_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key) 112 { 113 SHA256_CTX ctx; 114 115 /* Copy precalculated SHA256 context. */ 116 bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx)); 117 SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset)); 118 SHA256_Final(key, &ctx); 119 } 120 121 /* 122 * The function is called after we read and decrypt data. 123 * 124 * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver 125 */ 126 static int 127 g_eli_auth_read_done(struct cryptop *crp) 128 { 129 struct g_eli_softc *sc; 130 struct bio *bp; 131 132 if (crp->crp_etype == EAGAIN) { 133 if (g_eli_crypto_rerun(crp) == 0) 134 return (0); 135 } 136 bp = (struct bio *)crp->crp_opaque; 137 bp->bio_inbed++; 138 sc = bp->bio_to->geom->softc; 139 if (crp->crp_etype == 0) { 140 bp->bio_completed += crp->crp_payload_length; 141 G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).", 142 bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed); 143 } else { 144 u_int nsec, decr_secsize, encr_secsize, rel_sec; 145 int *errorp; 146 147 /* Sectorsize of decrypted provider eg. 4096. */ 148 decr_secsize = bp->bio_to->sectorsize; 149 /* The real sectorsize of encrypted provider, eg. 512. */ 150 encr_secsize = 151 LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 152 /* Number of sectors from decrypted provider, eg. 2. */ 153 nsec = bp->bio_length / decr_secsize; 154 /* Number of sectors from encrypted provider, eg. 18. */ 155 nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 156 /* Which relative sector this request decrypted. */ 157 rel_sec = ((crp->crp_buf.cb_buf + crp->crp_payload_start) - 158 (char *)bp->bio_driver2) / encr_secsize; 159 160 errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec + 161 sizeof(int) * rel_sec); 162 *errorp = crp->crp_etype; 163 G_ELI_DEBUG(1, 164 "Crypto READ request failed (%d/%d) error=%d.", 165 bp->bio_inbed, bp->bio_children, crp->crp_etype); 166 if (bp->bio_error == 0 || bp->bio_error == EINTEGRITY) 167 bp->bio_error = crp->crp_etype == EBADMSG ? 168 EINTEGRITY : crp->crp_etype; 169 } 170 if (crp->crp_cipher_key != NULL) 171 g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); 172 crypto_freereq(crp); 173 /* 174 * Do we have all sectors already? 175 */ 176 if (bp->bio_inbed < bp->bio_children) 177 return (0); 178 179 if (bp->bio_error == 0) { 180 u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 181 u_char *srcdata, *dstdata; 182 183 /* Sectorsize of decrypted provider eg. 4096. */ 184 decr_secsize = bp->bio_to->sectorsize; 185 /* The real sectorsize of encrypted provider, eg. 512. */ 186 encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 187 /* Number of data bytes in one encrypted sector, eg. 480. */ 188 data_secsize = sc->sc_data_per_sector; 189 /* Number of sectors from decrypted provider, eg. 2. */ 190 nsec = bp->bio_length / decr_secsize; 191 /* Number of sectors from encrypted provider, eg. 18. */ 192 nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 193 /* Last sector number in every big sector, eg. 9. */ 194 lsec = sc->sc_bytes_per_sector / encr_secsize; 195 196 srcdata = bp->bio_driver2; 197 dstdata = bp->bio_data; 198 199 for (i = 1; i <= nsec; i++) { 200 data_secsize = sc->sc_data_per_sector; 201 if ((i % lsec) == 0) 202 data_secsize = decr_secsize % data_secsize; 203 bcopy(srcdata + sc->sc_alen, dstdata, data_secsize); 204 srcdata += encr_secsize; 205 dstdata += data_secsize; 206 } 207 } else if (bp->bio_error == EINTEGRITY) { 208 u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 209 int *errorp; 210 off_t coroff, corsize, dstoff; 211 212 /* Sectorsize of decrypted provider eg. 4096. */ 213 decr_secsize = bp->bio_to->sectorsize; 214 /* The real sectorsize of encrypted provider, eg. 512. */ 215 encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 216 /* Number of data bytes in one encrypted sector, eg. 480. */ 217 data_secsize = sc->sc_data_per_sector; 218 /* Number of sectors from decrypted provider, eg. 2. */ 219 nsec = bp->bio_length / decr_secsize; 220 /* Number of sectors from encrypted provider, eg. 18. */ 221 nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 222 /* Last sector number in every big sector, eg. 9. */ 223 lsec = sc->sc_bytes_per_sector / encr_secsize; 224 225 errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec); 226 coroff = -1; 227 corsize = 0; 228 dstoff = bp->bio_offset; 229 230 for (i = 1; i <= nsec; i++) { 231 data_secsize = sc->sc_data_per_sector; 232 if ((i % lsec) == 0) 233 data_secsize = decr_secsize % data_secsize; 234 if (errorp[i - 1] == EBADMSG) { 235 /* 236 * Corruption detected, remember the offset if 237 * this is the first corrupted sector and 238 * increase size. 239 */ 240 if (coroff == -1) 241 coroff = dstoff; 242 corsize += data_secsize; 243 } else { 244 /* 245 * No corruption, good. 246 * Report previous corruption if there was one. 247 */ 248 if (coroff != -1) { 249 G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " 250 "bytes of data at offset %jd.", 251 sc->sc_name, (intmax_t)corsize, 252 (intmax_t)coroff); 253 coroff = -1; 254 corsize = 0; 255 } 256 } 257 dstoff += data_secsize; 258 } 259 /* Report previous corruption if there was one. */ 260 if (coroff != -1) { 261 G_ELI_DEBUG(0, "%s: Failed to authenticate %jd " 262 "bytes of data at offset %jd.", 263 sc->sc_name, (intmax_t)corsize, (intmax_t)coroff); 264 } 265 } 266 g_eli_free_data(bp); 267 if (bp->bio_error != 0) { 268 if (bp->bio_error != EINTEGRITY) { 269 G_ELI_LOGREQ(0, bp, 270 "Crypto READ request failed (error=%d).", 271 bp->bio_error); 272 } 273 bp->bio_completed = 0; 274 } 275 /* 276 * Read is finished, send it up. 277 */ 278 g_io_deliver(bp, bp->bio_error); 279 atomic_subtract_int(&sc->sc_inflight, 1); 280 return (0); 281 } 282 283 /* 284 * The function is called after data encryption. 285 * 286 * g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver 287 */ 288 static int 289 g_eli_auth_write_done(struct cryptop *crp) 290 { 291 struct g_eli_softc *sc; 292 struct g_consumer *cp; 293 struct bio *bp, *cbp, *cbp2; 294 u_int nsec; 295 296 if (crp->crp_etype == EAGAIN) { 297 if (g_eli_crypto_rerun(crp) == 0) 298 return (0); 299 } 300 bp = (struct bio *)crp->crp_opaque; 301 bp->bio_inbed++; 302 if (crp->crp_etype == 0) { 303 G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).", 304 bp->bio_inbed, bp->bio_children); 305 } else { 306 G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.", 307 bp->bio_inbed, bp->bio_children, crp->crp_etype); 308 if (bp->bio_error == 0) 309 bp->bio_error = crp->crp_etype; 310 } 311 sc = bp->bio_to->geom->softc; 312 if (crp->crp_cipher_key != NULL) 313 g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key)); 314 crypto_freereq(crp); 315 /* 316 * All sectors are already encrypted? 317 */ 318 if (bp->bio_inbed < bp->bio_children) 319 return (0); 320 if (bp->bio_error != 0) { 321 G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).", 322 bp->bio_error); 323 g_eli_free_data(bp); 324 cbp = bp->bio_driver1; 325 bp->bio_driver1 = NULL; 326 g_destroy_bio(cbp); 327 g_io_deliver(bp, bp->bio_error); 328 atomic_subtract_int(&sc->sc_inflight, 1); 329 return (0); 330 } 331 cp = LIST_FIRST(&sc->sc_geom->consumer); 332 cbp = bp->bio_driver1; 333 bp->bio_driver1 = NULL; 334 cbp->bio_to = cp->provider; 335 cbp->bio_done = g_eli_write_done; 336 337 /* Number of sectors from decrypted provider, eg. 1. */ 338 nsec = bp->bio_length / bp->bio_to->sectorsize; 339 /* Number of sectors from encrypted provider, eg. 9. */ 340 nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; 341 342 cbp->bio_length = cp->provider->sectorsize * nsec; 343 cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 344 cbp->bio_data = bp->bio_driver2; 345 346 /* 347 * We write more than what is requested, so we have to be ready to write 348 * more than maxphys. 349 */ 350 cbp2 = NULL; 351 if (cbp->bio_length > maxphys) { 352 cbp2 = g_duplicate_bio(bp); 353 cbp2->bio_length = cbp->bio_length - maxphys; 354 cbp2->bio_data = cbp->bio_data + maxphys; 355 cbp2->bio_offset = cbp->bio_offset + maxphys; 356 cbp2->bio_to = cp->provider; 357 cbp2->bio_done = g_eli_write_done; 358 cbp->bio_length = maxphys; 359 } 360 /* 361 * Send encrypted data to the provider. 362 */ 363 G_ELI_LOGREQ(2, cbp, "Sending request."); 364 bp->bio_inbed = 0; 365 bp->bio_children = (cbp2 != NULL ? 2 : 1); 366 g_io_request(cbp, cp); 367 if (cbp2 != NULL) { 368 G_ELI_LOGREQ(2, cbp2, "Sending request."); 369 g_io_request(cbp2, cp); 370 } 371 return (0); 372 } 373 374 void 375 g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp) 376 { 377 struct g_consumer *cp; 378 struct bio *cbp, *cbp2; 379 size_t size; 380 off_t nsec; 381 382 G_ELI_SETWORKER(bp->bio_pflags, 0); 383 384 cp = LIST_FIRST(&sc->sc_geom->consumer); 385 cbp = bp->bio_driver1; 386 bp->bio_driver1 = NULL; 387 cbp->bio_to = cp->provider; 388 cbp->bio_done = g_eli_read_done; 389 390 /* Number of sectors from decrypted provider, eg. 1. */ 391 nsec = bp->bio_length / bp->bio_to->sectorsize; 392 /* Number of sectors from encrypted provider, eg. 9. */ 393 nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize; 394 395 cbp->bio_length = cp->provider->sectorsize * nsec; 396 size = cbp->bio_length; 397 size += sizeof(int) * nsec; 398 size += G_ELI_AUTH_SECKEYLEN * nsec; 399 cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 400 if (!g_eli_alloc_data(bp, size)) { 401 G_ELI_LOGREQ(0, bp, "Crypto auth read request failed (ENOMEM)"); 402 g_destroy_bio(cbp); 403 bp->bio_error = ENOMEM; 404 g_io_deliver(bp, bp->bio_error); 405 atomic_subtract_int(&sc->sc_inflight, 1); 406 return; 407 } 408 cbp->bio_data = bp->bio_driver2; 409 410 /* Clear the error array. */ 411 memset((char *)bp->bio_driver2 + cbp->bio_length, 0, 412 sizeof(int) * nsec); 413 414 /* 415 * We read more than what is requested, so we have to be ready to read 416 * more than maxphys. 417 */ 418 cbp2 = NULL; 419 if (cbp->bio_length > maxphys) { 420 cbp2 = g_duplicate_bio(bp); 421 cbp2->bio_length = cbp->bio_length - maxphys; 422 cbp2->bio_data = cbp->bio_data + maxphys; 423 cbp2->bio_offset = cbp->bio_offset + maxphys; 424 cbp2->bio_to = cp->provider; 425 cbp2->bio_done = g_eli_read_done; 426 cbp->bio_length = maxphys; 427 } 428 /* 429 * Read encrypted data from provider. 430 */ 431 G_ELI_LOGREQ(2, cbp, "Sending request."); 432 g_io_request(cbp, cp); 433 if (cbp2 != NULL) { 434 G_ELI_LOGREQ(2, cbp2, "Sending request."); 435 g_io_request(cbp2, cp); 436 } 437 } 438 439 /* 440 * This is the main function responsible for cryptography (ie. communication 441 * with crypto(9) subsystem). 442 * 443 * BIO_READ: 444 * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> G_ELI_AUTH_RUN -> g_eli_auth_read_done -> g_io_deliver 445 * BIO_WRITE: 446 * g_eli_start -> G_ELI_AUTH_RUN -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver 447 */ 448 void 449 g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp) 450 { 451 struct g_eli_softc *sc; 452 struct cryptopq crpq; 453 struct cryptop *crp; 454 u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize; 455 off_t dstoff; 456 u_char *p, *data, *authkey, *plaindata; 457 int error __diagused; 458 bool batch; 459 460 G_ELI_LOGREQ(3, bp, "%s", __func__); 461 462 G_ELI_SETWORKER(bp->bio_pflags, wr->w_number); 463 sc = wr->w_softc; 464 /* Sectorsize of decrypted provider eg. 4096. */ 465 decr_secsize = bp->bio_to->sectorsize; 466 /* The real sectorsize of encrypted provider, eg. 512. */ 467 encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize; 468 /* Number of data bytes in one encrypted sector, eg. 480. */ 469 data_secsize = sc->sc_data_per_sector; 470 /* Number of sectors from decrypted provider, eg. 2. */ 471 nsec = bp->bio_length / decr_secsize; 472 /* Number of sectors from encrypted provider, eg. 18. */ 473 nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize; 474 /* Last sector number in every big sector, eg. 9. */ 475 lsec = sc->sc_bytes_per_sector / encr_secsize; 476 /* Destination offset, used for IV generation. */ 477 dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector; 478 479 plaindata = bp->bio_data; 480 if (bp->bio_cmd == BIO_READ) { 481 data = bp->bio_driver2; 482 p = data + encr_secsize * nsec; 483 p += sizeof(int) * nsec; 484 } else { 485 size_t size; 486 487 size = encr_secsize * nsec; 488 size += G_ELI_AUTH_SECKEYLEN * nsec; 489 size += sizeof(uintptr_t); /* Space for alignment. */ 490 if (!g_eli_alloc_data(bp, size)) { 491 G_ELI_LOGREQ(0, bp, "Crypto request failed (ENOMEM)"); 492 if (bp->bio_driver1 != NULL) { 493 g_destroy_bio(bp->bio_driver1); 494 bp->bio_driver1 = NULL; 495 } 496 bp->bio_error = ENOMEM; 497 g_io_deliver(bp, bp->bio_error); 498 if (sc != NULL) 499 atomic_subtract_int(&sc->sc_inflight, 1); 500 return; 501 } 502 data = bp->bio_driver2; 503 p = data + encr_secsize * nsec; 504 } 505 bp->bio_inbed = 0; 506 bp->bio_children = nsec; 507 508 #if defined(__mips_n64) || defined(__mips_o64) 509 p = (char *)roundup((uintptr_t)p, sizeof(uintptr_t)); 510 #endif 511 512 TAILQ_INIT(&crpq); 513 batch = atomic_load_int(&g_eli_batch) != 0; 514 515 for (i = 1; i <= nsec; i++, dstoff += encr_secsize) { 516 crp = crypto_getreq(wr->w_sid, M_WAITOK); 517 authkey = (u_char *)p; p += G_ELI_AUTH_SECKEYLEN; 518 519 data_secsize = sc->sc_data_per_sector; 520 if ((i % lsec) == 0) { 521 data_secsize = decr_secsize % data_secsize; 522 /* 523 * Last encrypted sector of each decrypted sector is 524 * only partially filled. 525 */ 526 if (bp->bio_cmd == BIO_WRITE) 527 memset(data + sc->sc_alen + data_secsize, 0, 528 encr_secsize - sc->sc_alen - data_secsize); 529 } else if (data_secsize + sc->sc_alen != encr_secsize) { 530 /* 531 * If the HMAC size is not a multiple of 128 bits, the 532 * per-sector data size is rounded down to ensure that 533 * encryption can be performed without requiring any 534 * padding. In this case, each sector contains unused 535 * bytes. 536 */ 537 if (bp->bio_cmd == BIO_WRITE) 538 memset(data + sc->sc_alen + data_secsize, 0, 539 encr_secsize - sc->sc_alen - data_secsize); 540 } 541 542 if (bp->bio_cmd == BIO_WRITE) { 543 bcopy(plaindata, data + sc->sc_alen, data_secsize); 544 plaindata += data_secsize; 545 } 546 547 crypto_use_buf(crp, data, sc->sc_alen + data_secsize); 548 crp->crp_opaque = (void *)bp; 549 data += encr_secsize; 550 crp->crp_flags = CRYPTO_F_CBIFSYNC; 551 if (bp->bio_cmd == BIO_WRITE) { 552 crp->crp_callback = g_eli_auth_write_done; 553 crp->crp_op = CRYPTO_OP_ENCRYPT | 554 CRYPTO_OP_COMPUTE_DIGEST; 555 } else { 556 crp->crp_callback = g_eli_auth_read_done; 557 crp->crp_op = CRYPTO_OP_DECRYPT | 558 CRYPTO_OP_VERIFY_DIGEST; 559 } 560 561 crp->crp_digest_start = 0; 562 crp->crp_payload_start = sc->sc_alen; 563 crp->crp_payload_length = data_secsize; 564 if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) { 565 crp->crp_cipher_key = g_eli_key_hold(sc, dstoff, 566 encr_secsize); 567 } 568 if (g_eli_ivlen(sc->sc_ealgo) != 0) { 569 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 570 g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv, 571 sizeof(crp->crp_iv)); 572 } 573 574 g_eli_auth_keygen(sc, dstoff, authkey); 575 crp->crp_auth_key = authkey; 576 577 if (batch) { 578 TAILQ_INSERT_TAIL(&crpq, crp, crp_next); 579 } else { 580 error = crypto_dispatch(crp); 581 KASSERT(error == 0, 582 ("crypto_dispatch() failed (error=%d)", error)); 583 } 584 } 585 586 if (batch) 587 crypto_dispatch_batch(&crpq, 0); 588 } 589