1 /* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16 /* 17 * Copyright (c) 2017, Datto, Inc. All rights reserved. 18 */ 19 20 #include <sys/zio_crypt.h> 21 #include <sys/dmu.h> 22 #include <sys/dmu_objset.h> 23 #include <sys/dnode.h> 24 #include <sys/fs/zfs.h> 25 #include <sys/zio.h> 26 #include <sys/zil.h> 27 #include <sys/sha2.h> 28 #include <sys/hkdf.h> 29 30 /* 31 * This file is responsible for handling all of the details of generating 32 * encryption parameters and performing encryption and authentication. 33 * 34 * BLOCK ENCRYPTION PARAMETERS: 35 * Encryption /Authentication Algorithm Suite (crypt): 36 * The encryption algorithm, mode, and key length we are going to use. We 37 * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit 38 * keys. All authentication is currently done with SHA512-HMAC. 39 * 40 * Plaintext: 41 * The unencrypted data that we want to encrypt. 42 * 43 * Initialization Vector (IV): 44 * An initialization vector for the encryption algorithms. This is used to 45 * "tweak" the encryption algorithms so that two blocks of the same data are 46 * encrypted into different ciphertext outputs, thus obfuscating block patterns. 47 * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is 48 * never reused with the same encryption key. This value is stored unencrypted 49 * and must simply be provided to the decryption function. We use a 96 bit IV 50 * (as recommended by NIST) for all block encryption. For non-dedup blocks we 51 * derive the IV randomly. The first 64 bits of the IV are stored in the second 52 * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of 53 * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits 54 * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count 55 * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of 56 * level 0 blocks is the number of allocated dnodes in that block. The on-disk 57 * format supports at most 2^15 slots per L0 dnode block, because the maximum 58 * block size is 16MB (2^24). In either case, for level 0 blocks this number 59 * will still be smaller than UINT32_MAX so it is safe to store the IV in the 60 * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count 61 * for the dnode code. 62 * 63 * Master key: 64 * This is the most important secret data of an encrypted dataset. It is used 65 * along with the salt to generate that actual encryption keys via HKDF. We 66 * do not use the master key to directly encrypt any data because there are 67 * theoretical limits on how much data can actually be safely encrypted with 68 * any encryption mode. The master key is stored encrypted on disk with the 69 * user's wrapping key. Its length is determined by the encryption algorithm. 70 * For details on how this is stored see the block comment in dsl_crypt.c 71 * 72 * Salt: 73 * Used as an input to the HKDF function, along with the master key. We use a 74 * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt 75 * can be used for encrypting many blocks, so we cache the current salt and the 76 * associated derived key in zio_crypt_t so we do not need to derive it again 77 * needlessly. 78 * 79 * Encryption Key: 80 * A secret binary key, generated from an HKDF function used to encrypt and 81 * decrypt data. 82 * 83 * Message Authentication Code (MAC) 84 * The MAC is an output of authenticated encryption modes such as AES-GCM and 85 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted 86 * data on disk and return garbage to the application. Effectively, it is a 87 * checksum that can not be reproduced by an attacker. We store the MAC in the 88 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated 89 * regular checksum of the ciphertext which can be used for scrubbing. 90 * 91 * OBJECT AUTHENTICATION: 92 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because 93 * they contain some info that always needs to be readable. To prevent this 94 * data from being altered, we authenticate this data using SHA512-HMAC. This 95 * will produce a MAC (similar to the one produced via encryption) which can 96 * be used to verify the object was not modified. HMACs do not require key 97 * rotation or IVs, so we can keep up to the full 3 copies of authenticated 98 * data. 99 * 100 * ZIL ENCRYPTION: 101 * ZIL blocks have their bp written to disk ahead of the associated data, so we 102 * cannot store the MAC there as we normally do. For these blocks the MAC is 103 * stored in the embedded checksum within the zil_chain_t header. The salt and 104 * IV are generated for the block on bp allocation instead of at encryption 105 * time. In addition, ZIL blocks have some pieces that must be left in plaintext 106 * for claiming even though all of the sensitive user data still needs to be 107 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which 108 * pieces of the block need to be encrypted. All data that is not encrypted is 109 * authenticated using the AAD mechanisms that the supported encryption modes 110 * provide for. In order to preserve the semantics of the ZIL for encrypted 111 * datasets, the ZIL is not protected at the objset level as described below. 112 * 113 * DNODE ENCRYPTION: 114 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left 115 * in plaintext for scrubbing and claiming, but the bonus buffers might contain 116 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing 117 * which pieces of the block need to be encrypted. For more details about 118 * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). 119 * 120 * OBJECT SET AUTHENTICATION: 121 * Up to this point, everything we have encrypted and authenticated has been 122 * at level 0 (or -2 for the ZIL). If we did not do any further work the 123 * on-disk format would be susceptible to attacks that deleted or rearranged 124 * the order of level 0 blocks. Ideally, the cleanest solution would be to 125 * maintain a tree of authentication MACs going up the bp tree. However, this 126 * presents a problem for raw sends. Send files do not send information about 127 * indirect blocks so there would be no convenient way to transfer the MACs and 128 * they cannot be recalculated on the receive side without the master key which 129 * would defeat one of the purposes of raw sends in the first place. Instead, 130 * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs 131 * from the level below. We also include some portable fields from blk_prop such 132 * as the lsize and compression algorithm to prevent the data from being 133 * misinterpreted. 134 * 135 * At the objset level, we maintain 2 separate 256 bit MACs in the 136 * objset_phys_t. The first one is "portable" and is the logical root of the 137 * MAC tree maintained in the metadnode's bps. The second, is "local" and is 138 * used as the root MAC for the user accounting objects, which are also not 139 * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload 140 * of the send file. The useraccounting code ensures that the useraccounting 141 * info is not present upon a receive, so the local MAC can simply be cleared 142 * out at that time. For more info about objset_phys_t authentication, see 143 * zio_crypt_do_objset_hmacs(). 144 * 145 * CONSIDERATIONS FOR DEDUP: 146 * In order for dedup to work, blocks that we want to dedup with one another 147 * need to use the same IV and encryption key, so that they will have the same 148 * ciphertext. Normally, one should never reuse an IV with the same encryption 149 * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both 150 * blocks. In this case, however, since we are using the same plaintext as 151 * well all that we end up with is a duplicate of the original ciphertext we 152 * already had. As a result, an attacker with read access to the raw disk will 153 * be able to tell which blocks are the same but this information is given away 154 * by dedup anyway. In order to get the same IVs and encryption keys for 155 * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC 156 * here so that a reproducible checksum of the plaintext is never available to 157 * the attacker. The HMAC key is kept alongside the master key, encrypted on 158 * disk. The first 64 bits of the HMAC are used in place of the random salt, and 159 * the next 96 bits are used as the IV. As a result of this mechanism, dedup 160 * will only work within a clone family since encrypted dedup requires use of 161 * the same master and HMAC keys. 162 */ 163 164 /* 165 * After encrypting many blocks with the same key we may start to run up 166 * against the theoretical limits of how much data can securely be encrypted 167 * with a single key using the supported encryption modes. The most obvious 168 * limitation is that our risk of generating 2 equivalent 96 bit IVs increases 169 * the more IVs we generate (which both GCM and CCM modes strictly forbid). 170 * This risk actually grows surprisingly quickly over time according to the 171 * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have 172 * generated n IVs with a cryptographically secure RNG, the approximate 173 * probability p(n) of a collision is given as: 174 * 175 * p(n) ~= e^(-n*(n-1)/(2*(2^96))) 176 * 177 * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] 178 * 179 * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion 180 * we must not write more than 398,065,730 blocks with the same encryption key. 181 * Therefore, we rotate our keys after 400,000,000 blocks have been written by 182 * generating a new random 64 bit salt for our HKDF encryption key generation 183 * function. 184 */ 185 #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 186 #define ZFS_CURRENT_MAX_SALT_USES \ 187 (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) 188 unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; 189 190 /* 191 * Set to a nonzero value to cause zio_do_crypt_uio() to fail 1/this many 192 * calls, to test decryption error handling code paths. 193 */ 194 uint64_t zio_decrypt_fail_fraction = 0; 195 196 typedef struct blkptr_auth_buf { 197 uint64_t bab_prop; /* blk_prop - portable mask */ 198 uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ 199 uint64_t bab_pad; /* reserved for future use */ 200 } blkptr_auth_buf_t; 201 202 zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { 203 {"", ZC_TYPE_NONE, 0, "inherit"}, 204 {"", ZC_TYPE_NONE, 0, "on"}, 205 {"", ZC_TYPE_NONE, 0, "off"}, 206 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, 207 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, 208 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, 209 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, 210 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, 211 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} 212 }; 213 214 static void 215 zio_crypt_key_destroy_early(zio_crypt_key_t *key) 216 { 217 rw_destroy(&key->zk_salt_lock); 218 219 /* free crypto templates */ 220 bzero(&key->zk_session, sizeof (key->zk_session)); 221 222 /* zero out sensitive data */ 223 bzero(key, sizeof (zio_crypt_key_t)); 224 } 225 226 void 227 zio_crypt_key_destroy(zio_crypt_key_t *key) 228 { 229 230 freebsd_crypt_freesession(&key->zk_session); 231 zio_crypt_key_destroy_early(key); 232 } 233 234 int 235 zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) 236 { 237 int ret; 238 crypto_mechanism_t mech __unused; 239 uint_t keydata_len; 240 zio_crypt_info_t *ci = NULL; 241 242 ASSERT3P(key, !=, NULL); 243 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 244 245 ci = &zio_crypt_table[crypt]; 246 if (ci->ci_crypt_type != ZC_TYPE_GCM && 247 ci->ci_crypt_type != ZC_TYPE_CCM) 248 return (ENOTSUP); 249 250 keydata_len = zio_crypt_table[crypt].ci_keylen; 251 bzero(key, sizeof (zio_crypt_key_t)); 252 rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); 253 254 /* fill keydata buffers and salt with random data */ 255 ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); 256 if (ret != 0) 257 goto error; 258 259 ret = random_get_bytes(key->zk_master_keydata, keydata_len); 260 if (ret != 0) 261 goto error; 262 263 ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); 264 if (ret != 0) 265 goto error; 266 267 ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); 268 if (ret != 0) 269 goto error; 270 271 /* derive the current key from the master key */ 272 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 273 key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, 274 keydata_len); 275 if (ret != 0) 276 goto error; 277 278 /* initialize keys for the ICP */ 279 key->zk_current_key.ck_format = CRYPTO_KEY_RAW; 280 key->zk_current_key.ck_data = key->zk_current_keydata; 281 key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); 282 283 key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; 284 key->zk_hmac_key.ck_data = &key->zk_hmac_key; 285 key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); 286 287 ci = &zio_crypt_table[crypt]; 288 if (ci->ci_crypt_type != ZC_TYPE_GCM && 289 ci->ci_crypt_type != ZC_TYPE_CCM) 290 return (ENOTSUP); 291 292 ret = freebsd_crypt_newsession(&key->zk_session, ci, 293 &key->zk_current_key); 294 if (ret) 295 goto error; 296 297 key->zk_crypt = crypt; 298 key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; 299 key->zk_salt_count = 0; 300 301 return (0); 302 303 error: 304 zio_crypt_key_destroy_early(key); 305 return (ret); 306 } 307 308 static int 309 zio_crypt_key_change_salt(zio_crypt_key_t *key) 310 { 311 int ret = 0; 312 uint8_t salt[ZIO_DATA_SALT_LEN]; 313 crypto_mechanism_t mech __unused; 314 315 uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; 316 317 /* generate a new salt */ 318 ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); 319 if (ret != 0) 320 goto error; 321 322 rw_enter(&key->zk_salt_lock, RW_WRITER); 323 324 /* someone beat us to the salt rotation, just unlock and return */ 325 if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) 326 goto out_unlock; 327 328 /* derive the current key from the master key and the new salt */ 329 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 330 salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); 331 if (ret != 0) 332 goto out_unlock; 333 334 /* assign the salt and reset the usage count */ 335 bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); 336 key->zk_salt_count = 0; 337 338 freebsd_crypt_freesession(&key->zk_session); 339 ret = freebsd_crypt_newsession(&key->zk_session, 340 &zio_crypt_table[key->zk_crypt], &key->zk_current_key); 341 if (ret != 0) 342 goto out_unlock; 343 344 rw_exit(&key->zk_salt_lock); 345 346 return (0); 347 348 out_unlock: 349 rw_exit(&key->zk_salt_lock); 350 error: 351 return (ret); 352 } 353 354 /* See comment above zfs_key_max_salt_uses definition for details */ 355 int 356 zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) 357 { 358 int ret; 359 boolean_t salt_change; 360 361 rw_enter(&key->zk_salt_lock, RW_READER); 362 363 bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); 364 salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= 365 ZFS_CURRENT_MAX_SALT_USES); 366 367 rw_exit(&key->zk_salt_lock); 368 369 if (salt_change) { 370 ret = zio_crypt_key_change_salt(key); 371 if (ret != 0) 372 goto error; 373 } 374 375 return (0); 376 377 error: 378 return (ret); 379 } 380 381 void *failed_decrypt_buf; 382 int failed_decrypt_size; 383 384 /* 385 * This function handles all encryption and decryption in zfs. When 386 * encrypting it expects puio to reference the plaintext and cuio to 387 * reference the ciphertext. cuio must have enough space for the 388 * ciphertext + room for a MAC. datalen should be the length of the 389 * plaintext / ciphertext alone. 390 */ 391 /* 392 * The implementation for FreeBSD's OpenCrypto. 393 * 394 * The big difference between ICP and FOC is that FOC uses a single 395 * buffer for input and output. This means that (for AES-GCM, the 396 * only one supported right now) the source must be copied into the 397 * destination, and the destination must have the AAD, and the tag/MAC, 398 * already associated with it. (Both implementations can use a uio.) 399 * 400 * Since the auth data is part of the iovec array, all we need to know 401 * is the length: 0 means there's no AAD. 402 * 403 */ 404 static int 405 zio_do_crypt_uio_opencrypto(boolean_t encrypt, freebsd_crypt_session_t *sess, 406 uint64_t crypt, crypto_key_t *key, uint8_t *ivbuf, uint_t datalen, 407 zfs_uio_t *uio, uint_t auth_len) 408 { 409 zio_crypt_info_t *ci; 410 int ret; 411 412 ci = &zio_crypt_table[crypt]; 413 if (ci->ci_crypt_type != ZC_TYPE_GCM && 414 ci->ci_crypt_type != ZC_TYPE_CCM) 415 return (ENOTSUP); 416 417 418 ret = freebsd_crypt_uio(encrypt, sess, ci, uio, key, ivbuf, 419 datalen, auth_len); 420 if (ret != 0) { 421 #ifdef FCRYPTO_DEBUG 422 printf("%s(%d): Returning error %s\n", 423 __FUNCTION__, __LINE__, encrypt ? "EIO" : "ECKSUM"); 424 #endif 425 ret = SET_ERROR(encrypt ? EIO : ECKSUM); 426 } 427 428 return (ret); 429 } 430 431 int 432 zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, 433 uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) 434 { 435 int ret; 436 uint64_t aad[3]; 437 /* 438 * With OpenCrypto in FreeBSD, the same buffer is used for 439 * input and output. Also, the AAD (for AES-GMC at least) 440 * needs to logically go in front. 441 */ 442 zfs_uio_t cuio; 443 struct uio cuio_s; 444 iovec_t iovecs[4]; 445 uint64_t crypt = key->zk_crypt; 446 uint_t enc_len, keydata_len, aad_len; 447 448 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 449 ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); 450 451 zfs_uio_init(&cuio, &cuio_s); 452 453 keydata_len = zio_crypt_table[crypt].ci_keylen; 454 455 /* generate iv for wrapping the master and hmac key */ 456 ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); 457 if (ret != 0) 458 goto error; 459 460 /* 461 * Since we only support one buffer, we need to copy 462 * the plain text (source) to the cipher buffer (dest). 463 * We set iovecs[0] -- the authentication data -- below. 464 */ 465 bcopy((void*)key->zk_master_keydata, keydata_out, keydata_len); 466 bcopy((void*)key->zk_hmac_keydata, hmac_keydata_out, 467 SHA512_HMAC_KEYLEN); 468 iovecs[1].iov_base = keydata_out; 469 iovecs[1].iov_len = keydata_len; 470 iovecs[2].iov_base = hmac_keydata_out; 471 iovecs[2].iov_len = SHA512_HMAC_KEYLEN; 472 iovecs[3].iov_base = mac; 473 iovecs[3].iov_len = WRAPPING_MAC_LEN; 474 475 /* 476 * Although we don't support writing to the old format, we do 477 * support rewrapping the key so that the user can move and 478 * quarantine datasets on the old format. 479 */ 480 if (key->zk_version == 0) { 481 aad_len = sizeof (uint64_t); 482 aad[0] = LE_64(key->zk_guid); 483 } else { 484 ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 485 aad_len = sizeof (uint64_t) * 3; 486 aad[0] = LE_64(key->zk_guid); 487 aad[1] = LE_64(crypt); 488 aad[2] = LE_64(key->zk_version); 489 } 490 491 iovecs[0].iov_base = aad; 492 iovecs[0].iov_len = aad_len; 493 enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; 494 495 GET_UIO_STRUCT(&cuio)->uio_iov = iovecs; 496 zfs_uio_iovcnt(&cuio) = 4; 497 zfs_uio_segflg(&cuio) = UIO_SYSSPACE; 498 499 /* encrypt the keys and store the resulting ciphertext and mac */ 500 ret = zio_do_crypt_uio_opencrypto(B_TRUE, NULL, crypt, cwkey, 501 iv, enc_len, &cuio, aad_len); 502 if (ret != 0) 503 goto error; 504 505 return (0); 506 507 error: 508 return (ret); 509 } 510 511 int 512 zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, 513 uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, 514 uint8_t *mac, zio_crypt_key_t *key) 515 { 516 int ret; 517 uint64_t aad[3]; 518 /* 519 * With OpenCrypto in FreeBSD, the same buffer is used for 520 * input and output. Also, the AAD (for AES-GMC at least) 521 * needs to logically go in front. 522 */ 523 zfs_uio_t cuio; 524 struct uio cuio_s; 525 iovec_t iovecs[4]; 526 void *src, *dst; 527 uint_t enc_len, keydata_len, aad_len; 528 529 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 530 ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); 531 532 keydata_len = zio_crypt_table[crypt].ci_keylen; 533 rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); 534 535 zfs_uio_init(&cuio, &cuio_s); 536 537 /* 538 * Since we only support one buffer, we need to copy 539 * the encrypted buffer (source) to the plain buffer 540 * (dest). We set iovecs[0] -- the authentication data -- 541 * below. 542 */ 543 dst = key->zk_master_keydata; 544 src = keydata; 545 546 bcopy(src, dst, keydata_len); 547 548 dst = key->zk_hmac_keydata; 549 src = hmac_keydata; 550 bcopy(src, dst, SHA512_HMAC_KEYLEN); 551 552 iovecs[1].iov_base = key->zk_master_keydata; 553 iovecs[1].iov_len = keydata_len; 554 iovecs[2].iov_base = key->zk_hmac_keydata; 555 iovecs[2].iov_len = SHA512_HMAC_KEYLEN; 556 iovecs[3].iov_base = mac; 557 iovecs[3].iov_len = WRAPPING_MAC_LEN; 558 559 if (version == 0) { 560 aad_len = sizeof (uint64_t); 561 aad[0] = LE_64(guid); 562 } else { 563 ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 564 aad_len = sizeof (uint64_t) * 3; 565 aad[0] = LE_64(guid); 566 aad[1] = LE_64(crypt); 567 aad[2] = LE_64(version); 568 } 569 570 enc_len = keydata_len + SHA512_HMAC_KEYLEN; 571 iovecs[0].iov_base = aad; 572 iovecs[0].iov_len = aad_len; 573 574 GET_UIO_STRUCT(&cuio)->uio_iov = iovecs; 575 zfs_uio_iovcnt(&cuio) = 4; 576 zfs_uio_segflg(&cuio) = UIO_SYSSPACE; 577 578 /* decrypt the keys and store the result in the output buffers */ 579 ret = zio_do_crypt_uio_opencrypto(B_FALSE, NULL, crypt, cwkey, 580 iv, enc_len, &cuio, aad_len); 581 582 if (ret != 0) 583 goto error; 584 585 /* generate a fresh salt */ 586 ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); 587 if (ret != 0) 588 goto error; 589 590 /* derive the current key from the master key */ 591 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 592 key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, 593 keydata_len); 594 if (ret != 0) 595 goto error; 596 597 /* initialize keys for ICP */ 598 key->zk_current_key.ck_format = CRYPTO_KEY_RAW; 599 key->zk_current_key.ck_data = key->zk_current_keydata; 600 key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); 601 602 key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; 603 key->zk_hmac_key.ck_data = key->zk_hmac_keydata; 604 key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); 605 606 ret = freebsd_crypt_newsession(&key->zk_session, 607 &zio_crypt_table[crypt], &key->zk_current_key); 608 if (ret != 0) 609 goto error; 610 611 key->zk_crypt = crypt; 612 key->zk_version = version; 613 key->zk_guid = guid; 614 key->zk_salt_count = 0; 615 616 return (0); 617 618 error: 619 zio_crypt_key_destroy_early(key); 620 return (ret); 621 } 622 623 int 624 zio_crypt_generate_iv(uint8_t *ivbuf) 625 { 626 int ret; 627 628 /* randomly generate the IV */ 629 ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); 630 if (ret != 0) 631 goto error; 632 633 return (0); 634 635 error: 636 bzero(ivbuf, ZIO_DATA_IV_LEN); 637 return (ret); 638 } 639 640 int 641 zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, 642 uint8_t *digestbuf, uint_t digestlen) 643 { 644 uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; 645 646 ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); 647 648 crypto_mac(&key->zk_hmac_key, data, datalen, 649 raw_digestbuf, SHA512_DIGEST_LENGTH); 650 651 bcopy(raw_digestbuf, digestbuf, digestlen); 652 653 return (0); 654 } 655 656 int 657 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, 658 uint_t datalen, uint8_t *ivbuf, uint8_t *salt) 659 { 660 int ret; 661 uint8_t digestbuf[SHA512_DIGEST_LENGTH]; 662 663 ret = zio_crypt_do_hmac(key, data, datalen, 664 digestbuf, SHA512_DIGEST_LENGTH); 665 if (ret != 0) 666 return (ret); 667 668 bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); 669 bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); 670 671 return (0); 672 } 673 674 /* 675 * The following functions are used to encode and decode encryption parameters 676 * into blkptr_t and zil_header_t. The ICP wants to use these parameters as 677 * byte strings, which normally means that these strings would not need to deal 678 * with byteswapping at all. However, both blkptr_t and zil_header_t may be 679 * byteswapped by lower layers and so we must "undo" that byteswap here upon 680 * decoding and encoding in a non-native byteorder. These functions require 681 * that the byteorder bit is correct before being called. 682 */ 683 void 684 zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) 685 { 686 uint64_t val64; 687 uint32_t val32; 688 689 ASSERT(BP_IS_ENCRYPTED(bp)); 690 691 if (!BP_SHOULD_BYTESWAP(bp)) { 692 bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); 693 bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); 694 bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); 695 BP_SET_IV2(bp, val32); 696 } else { 697 bcopy(salt, &val64, sizeof (uint64_t)); 698 bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); 699 700 bcopy(iv, &val64, sizeof (uint64_t)); 701 bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); 702 703 bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); 704 BP_SET_IV2(bp, BSWAP_32(val32)); 705 } 706 } 707 708 void 709 zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) 710 { 711 uint64_t val64; 712 uint32_t val32; 713 714 ASSERT(BP_IS_PROTECTED(bp)); 715 716 /* for convenience, so callers don't need to check */ 717 if (BP_IS_AUTHENTICATED(bp)) { 718 bzero(salt, ZIO_DATA_SALT_LEN); 719 bzero(iv, ZIO_DATA_IV_LEN); 720 return; 721 } 722 723 if (!BP_SHOULD_BYTESWAP(bp)) { 724 bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); 725 bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); 726 727 val32 = (uint32_t)BP_GET_IV2(bp); 728 bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); 729 } else { 730 val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); 731 bcopy(&val64, salt, sizeof (uint64_t)); 732 733 val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); 734 bcopy(&val64, iv, sizeof (uint64_t)); 735 736 val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); 737 bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); 738 } 739 } 740 741 void 742 zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) 743 { 744 uint64_t val64; 745 746 ASSERT(BP_USES_CRYPT(bp)); 747 ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); 748 749 if (!BP_SHOULD_BYTESWAP(bp)) { 750 bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); 751 bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], 752 sizeof (uint64_t)); 753 } else { 754 bcopy(mac, &val64, sizeof (uint64_t)); 755 bp->blk_cksum.zc_word[2] = BSWAP_64(val64); 756 757 bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); 758 bp->blk_cksum.zc_word[3] = BSWAP_64(val64); 759 } 760 } 761 762 void 763 zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) 764 { 765 uint64_t val64; 766 767 ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); 768 769 /* for convenience, so callers don't need to check */ 770 if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 771 bzero(mac, ZIO_DATA_MAC_LEN); 772 return; 773 } 774 775 if (!BP_SHOULD_BYTESWAP(bp)) { 776 bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); 777 bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), 778 sizeof (uint64_t)); 779 } else { 780 val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); 781 bcopy(&val64, mac, sizeof (uint64_t)); 782 783 val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); 784 bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); 785 } 786 } 787 788 void 789 zio_crypt_encode_mac_zil(void *data, uint8_t *mac) 790 { 791 zil_chain_t *zilc = data; 792 793 bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); 794 bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], 795 sizeof (uint64_t)); 796 } 797 798 void 799 zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) 800 { 801 /* 802 * The ZIL MAC is embedded in the block it protects, which will 803 * not have been byteswapped by the time this function has been called. 804 * As a result, we don't need to worry about byteswapping the MAC. 805 */ 806 const zil_chain_t *zilc = data; 807 808 bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); 809 bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), 810 sizeof (uint64_t)); 811 } 812 813 /* 814 * This routine takes a block of dnodes (src_abd) and copies only the bonus 815 * buffers to the same offsets in the dst buffer. datalen should be the size 816 * of both the src_abd and the dst buffer (not just the length of the bonus 817 * buffers). 818 */ 819 void 820 zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) 821 { 822 uint_t i, max_dnp = datalen >> DNODE_SHIFT; 823 uint8_t *src; 824 dnode_phys_t *dnp, *sdnp, *ddnp; 825 826 src = abd_borrow_buf_copy(src_abd, datalen); 827 828 sdnp = (dnode_phys_t *)src; 829 ddnp = (dnode_phys_t *)dst; 830 831 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 832 dnp = &sdnp[i]; 833 if (dnp->dn_type != DMU_OT_NONE && 834 DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && 835 dnp->dn_bonuslen != 0) { 836 bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), 837 DN_MAX_BONUS_LEN(dnp)); 838 } 839 } 840 841 abd_return_buf(src_abd, src, datalen); 842 } 843 844 /* 845 * This function decides what fields from blk_prop are included in 846 * the on-disk various MAC algorithms. 847 */ 848 static void 849 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) 850 { 851 int avoidlint = SPA_MINBLOCKSIZE; 852 /* 853 * Version 0 did not properly zero out all non-portable fields 854 * as it should have done. We maintain this code so that we can 855 * do read-only imports of pools on this version. 856 */ 857 if (version == 0) { 858 BP_SET_DEDUP(bp, 0); 859 BP_SET_CHECKSUM(bp, 0); 860 BP_SET_PSIZE(bp, avoidlint); 861 return; 862 } 863 864 ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 865 866 /* 867 * The hole_birth feature might set these fields even if this bp 868 * is a hole. We zero them out here to guarantee that raw sends 869 * will function with or without the feature. 870 */ 871 if (BP_IS_HOLE(bp)) { 872 bp->blk_prop = 0ULL; 873 return; 874 } 875 876 /* 877 * At L0 we want to verify these fields to ensure that data blocks 878 * can not be reinterpreted. For instance, we do not want an attacker 879 * to trick us into returning raw lz4 compressed data to the user 880 * by modifying the compression bits. At higher levels, we cannot 881 * enforce this policy since raw sends do not convey any information 882 * about indirect blocks, so these values might be different on the 883 * receive side. Fortunately, this does not open any new attack 884 * vectors, since any alterations that can be made to a higher level 885 * bp must still verify the correct order of the layer below it. 886 */ 887 if (BP_GET_LEVEL(bp) != 0) { 888 BP_SET_BYTEORDER(bp, 0); 889 BP_SET_COMPRESS(bp, 0); 890 891 /* 892 * psize cannot be set to zero or it will trigger 893 * asserts, but the value doesn't really matter as 894 * long as it is constant. 895 */ 896 BP_SET_PSIZE(bp, avoidlint); 897 } 898 899 BP_SET_DEDUP(bp, 0); 900 BP_SET_CHECKSUM(bp, 0); 901 } 902 903 static void 904 zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, 905 blkptr_auth_buf_t *bab, uint_t *bab_len) 906 { 907 blkptr_t tmpbp = *bp; 908 909 if (should_bswap) 910 byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); 911 912 ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); 913 ASSERT0(BP_IS_EMBEDDED(&tmpbp)); 914 915 zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); 916 917 /* 918 * We always MAC blk_prop in LE to ensure portability. This 919 * must be done after decoding the mac, since the endianness 920 * will get zero'd out here. 921 */ 922 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); 923 bab->bab_prop = LE_64(tmpbp.blk_prop); 924 bab->bab_pad = 0ULL; 925 926 /* version 0 did not include the padding */ 927 *bab_len = sizeof (blkptr_auth_buf_t); 928 if (version == 0) 929 *bab_len -= sizeof (uint64_t); 930 } 931 932 static int 933 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, 934 boolean_t should_bswap, blkptr_t *bp) 935 { 936 uint_t bab_len; 937 blkptr_auth_buf_t bab; 938 939 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 940 crypto_mac_update(ctx, &bab, bab_len); 941 942 return (0); 943 } 944 945 static void 946 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, 947 boolean_t should_bswap, blkptr_t *bp) 948 { 949 uint_t bab_len; 950 blkptr_auth_buf_t bab; 951 952 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 953 SHA2Update(ctx, &bab, bab_len); 954 } 955 956 static void 957 zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, 958 boolean_t should_bswap, blkptr_t *bp) 959 { 960 uint_t bab_len; 961 blkptr_auth_buf_t bab; 962 963 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 964 bcopy(&bab, *aadp, bab_len); 965 *aadp += bab_len; 966 *aad_len += bab_len; 967 } 968 969 static int 970 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, 971 boolean_t should_bswap, dnode_phys_t *dnp) 972 { 973 int ret, i; 974 dnode_phys_t *adnp; 975 boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); 976 uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; 977 978 /* authenticate the core dnode (masking out non-portable bits) */ 979 bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); 980 adnp = (dnode_phys_t *)tmp_dncore; 981 if (le_bswap) { 982 adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); 983 adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); 984 adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); 985 adnp->dn_used = BSWAP_64(adnp->dn_used); 986 } 987 adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; 988 adnp->dn_used = 0; 989 990 crypto_mac_update(ctx, adnp, sizeof (tmp_dncore)); 991 992 for (i = 0; i < dnp->dn_nblkptr; i++) { 993 ret = zio_crypt_bp_do_hmac_updates(ctx, version, 994 should_bswap, &dnp->dn_blkptr[i]); 995 if (ret != 0) 996 goto error; 997 } 998 999 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1000 ret = zio_crypt_bp_do_hmac_updates(ctx, version, 1001 should_bswap, DN_SPILL_BLKPTR(dnp)); 1002 if (ret != 0) 1003 goto error; 1004 } 1005 1006 return (0); 1007 1008 error: 1009 return (ret); 1010 } 1011 1012 /* 1013 * objset_phys_t blocks introduce a number of exceptions to the normal 1014 * authentication process. objset_phys_t's contain 2 separate HMACS for 1015 * protecting the integrity of their data. The portable_mac protects the 1016 * metadnode. This MAC can be sent with a raw send and protects against 1017 * reordering of data within the metadnode. The local_mac protects the user 1018 * accounting objects which are not sent from one system to another. 1019 * 1020 * In addition, objset blocks are the only blocks that can be modified and 1021 * written to disk without the key loaded under certain circumstances. During 1022 * zil_claim() we need to be able to update the zil_header_t to complete 1023 * claiming log blocks and during raw receives we need to write out the 1024 * portable_mac from the send file. Both of these actions are possible 1025 * because these fields are not protected by either MAC so neither one will 1026 * need to modify the MACs without the key. However, when the modified blocks 1027 * are written out they will be byteswapped into the host machine's native 1028 * endianness which will modify fields protected by the MAC. As a result, MAC 1029 * calculation for objset blocks works slightly differently from other block 1030 * types. Where other block types MAC the data in whatever endianness is 1031 * written to disk, objset blocks always MAC little endian version of their 1032 * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() 1033 * and le_bswap indicates whether a byteswap is needed to get this block 1034 * into little endian format. 1035 */ 1036 /* ARGSUSED */ 1037 int 1038 zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, 1039 boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) 1040 { 1041 int ret; 1042 struct hmac_ctx hash_ctx; 1043 struct hmac_ctx *ctx = &hash_ctx; 1044 objset_phys_t *osp = data; 1045 uint64_t intval; 1046 boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); 1047 uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; 1048 uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; 1049 1050 1051 /* calculate the portable MAC from the portable fields and metadnode */ 1052 crypto_mac_init(ctx, &key->zk_hmac_key); 1053 1054 /* add in the os_type */ 1055 intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); 1056 crypto_mac_update(ctx, &intval, sizeof (uint64_t)); 1057 1058 /* add in the portable os_flags */ 1059 intval = osp->os_flags; 1060 if (should_bswap) 1061 intval = BSWAP_64(intval); 1062 intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; 1063 if (!ZFS_HOST_BYTEORDER) 1064 intval = BSWAP_64(intval); 1065 1066 crypto_mac_update(ctx, &intval, sizeof (uint64_t)); 1067 1068 /* add in fields from the metadnode */ 1069 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1070 should_bswap, &osp->os_meta_dnode); 1071 if (ret) 1072 goto error; 1073 1074 crypto_mac_final(ctx, raw_portable_mac, SHA512_DIGEST_LENGTH); 1075 1076 bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); 1077 1078 /* 1079 * The local MAC protects the user, group and project accounting. 1080 * If these objects are not present, the local MAC is zeroed out. 1081 */ 1082 if ((datalen >= OBJSET_PHYS_SIZE_V3 && 1083 osp->os_userused_dnode.dn_type == DMU_OT_NONE && 1084 osp->os_groupused_dnode.dn_type == DMU_OT_NONE && 1085 osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || 1086 (datalen >= OBJSET_PHYS_SIZE_V2 && 1087 osp->os_userused_dnode.dn_type == DMU_OT_NONE && 1088 osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || 1089 (datalen <= OBJSET_PHYS_SIZE_V1)) { 1090 bzero(local_mac, ZIO_OBJSET_MAC_LEN); 1091 return (0); 1092 } 1093 1094 /* calculate the local MAC from the userused and groupused dnodes */ 1095 crypto_mac_init(ctx, &key->zk_hmac_key); 1096 1097 /* add in the non-portable os_flags */ 1098 intval = osp->os_flags; 1099 if (should_bswap) 1100 intval = BSWAP_64(intval); 1101 intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; 1102 if (!ZFS_HOST_BYTEORDER) 1103 intval = BSWAP_64(intval); 1104 1105 crypto_mac_update(ctx, &intval, sizeof (uint64_t)); 1106 1107 /* XXX check dnode type ... */ 1108 /* add in fields from the user accounting dnodes */ 1109 if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { 1110 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1111 should_bswap, &osp->os_userused_dnode); 1112 if (ret) 1113 goto error; 1114 } 1115 1116 if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { 1117 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1118 should_bswap, &osp->os_groupused_dnode); 1119 if (ret) 1120 goto error; 1121 } 1122 1123 if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && 1124 datalen >= OBJSET_PHYS_SIZE_V3) { 1125 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1126 should_bswap, &osp->os_projectused_dnode); 1127 if (ret) 1128 goto error; 1129 } 1130 1131 crypto_mac_final(ctx, raw_local_mac, SHA512_DIGEST_LENGTH); 1132 1133 bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); 1134 1135 return (0); 1136 1137 error: 1138 bzero(portable_mac, ZIO_OBJSET_MAC_LEN); 1139 bzero(local_mac, ZIO_OBJSET_MAC_LEN); 1140 return (ret); 1141 } 1142 1143 static void 1144 zio_crypt_destroy_uio(zfs_uio_t *uio) 1145 { 1146 if (GET_UIO_STRUCT(uio)->uio_iov) 1147 kmem_free(GET_UIO_STRUCT(uio)->uio_iov, 1148 zfs_uio_iovcnt(uio) * sizeof (iovec_t)); 1149 } 1150 1151 /* 1152 * This function parses an uncompressed indirect block and returns a checksum 1153 * of all the portable fields from all of the contained bps. The portable 1154 * fields are the MAC and all of the fields from blk_prop except for the dedup, 1155 * checksum, and psize bits. For an explanation of the purpose of this, see 1156 * the comment block on object set authentication. 1157 */ 1158 static int 1159 zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, 1160 uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) 1161 { 1162 blkptr_t *bp; 1163 int i, epb = datalen >> SPA_BLKPTRSHIFT; 1164 SHA2_CTX ctx; 1165 uint8_t digestbuf[SHA512_DIGEST_LENGTH]; 1166 1167 /* checksum all of the MACs from the layer below */ 1168 SHA2Init(SHA512, &ctx); 1169 for (i = 0, bp = buf; i < epb; i++, bp++) { 1170 zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, 1171 byteswap, bp); 1172 } 1173 SHA2Final(digestbuf, &ctx); 1174 1175 if (generate) { 1176 bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); 1177 return (0); 1178 } 1179 1180 if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) { 1181 #ifdef FCRYPTO_DEBUG 1182 printf("%s(%d): Setting ECKSUM\n", __FUNCTION__, __LINE__); 1183 #endif 1184 return (SET_ERROR(ECKSUM)); 1185 } 1186 return (0); 1187 } 1188 1189 int 1190 zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, 1191 uint_t datalen, boolean_t byteswap, uint8_t *cksum) 1192 { 1193 int ret; 1194 1195 /* 1196 * Unfortunately, callers of this function will not always have 1197 * easy access to the on-disk format version. This info is 1198 * normally found in the DSL Crypto Key, but the checksum-of-MACs 1199 * is expected to be verifiable even when the key isn't loaded. 1200 * Here, instead of doing a ZAP lookup for the version for each 1201 * zio, we simply try both existing formats. 1202 */ 1203 ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, 1204 datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); 1205 if (ret == ECKSUM) { 1206 ASSERT(!generate); 1207 ret = zio_crypt_do_indirect_mac_checksum_impl(generate, 1208 buf, datalen, 0, byteswap, cksum); 1209 } 1210 1211 return (ret); 1212 } 1213 1214 int 1215 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, 1216 uint_t datalen, boolean_t byteswap, uint8_t *cksum) 1217 { 1218 int ret; 1219 void *buf; 1220 1221 buf = abd_borrow_buf_copy(abd, datalen); 1222 ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, 1223 byteswap, cksum); 1224 abd_return_buf(abd, buf, datalen); 1225 1226 return (ret); 1227 } 1228 1229 /* 1230 * Special case handling routine for encrypting / decrypting ZIL blocks. 1231 * We do not check for the older ZIL chain because the encryption feature 1232 * was not available before the newer ZIL chain was introduced. The goal 1233 * here is to encrypt everything except the blkptr_t of a lr_write_t and 1234 * the zil_chain_t header. Everything that is not encrypted is authenticated. 1235 */ 1236 /* 1237 * The OpenCrypto used in FreeBSD does not use separate source and 1238 * destination buffers; instead, the same buffer is used. Further, to 1239 * accommodate some of the drivers, the authbuf needs to be logically before 1240 * the data. This means that we need to copy the source to the destination, 1241 * and set up an extra iovec_t at the beginning to handle the authbuf. 1242 * It also means we'll only return one zfs_uio_t. 1243 */ 1244 1245 /* ARGSUSED */ 1246 static int 1247 zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, 1248 uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, zfs_uio_t *puio, 1249 zfs_uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, 1250 boolean_t *no_crypt) 1251 { 1252 uint8_t *aadbuf = zio_buf_alloc(datalen); 1253 uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; 1254 iovec_t *dst_iovecs; 1255 zil_chain_t *zilc; 1256 lr_t *lr; 1257 uint64_t txtype, lr_len; 1258 uint_t crypt_len, nr_iovecs, vec; 1259 uint_t aad_len = 0, total_len = 0; 1260 1261 if (encrypt) { 1262 src = plainbuf; 1263 dst = cipherbuf; 1264 } else { 1265 src = cipherbuf; 1266 dst = plainbuf; 1267 } 1268 bcopy(src, dst, datalen); 1269 1270 /* Find the start and end record of the log block. */ 1271 zilc = (zil_chain_t *)src; 1272 slrp = src + sizeof (zil_chain_t); 1273 aadp = aadbuf; 1274 blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); 1275 1276 /* 1277 * Calculate the number of encrypted iovecs we will need. 1278 */ 1279 1280 /* We need at least two iovecs -- one for the AAD, one for the MAC. */ 1281 nr_iovecs = 2; 1282 1283 for (; slrp < blkend; slrp += lr_len) { 1284 lr = (lr_t *)slrp; 1285 1286 if (byteswap) { 1287 txtype = BSWAP_64(lr->lrc_txtype); 1288 lr_len = BSWAP_64(lr->lrc_reclen); 1289 } else { 1290 txtype = lr->lrc_txtype; 1291 lr_len = lr->lrc_reclen; 1292 } 1293 1294 nr_iovecs++; 1295 if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) 1296 nr_iovecs++; 1297 } 1298 1299 dst_iovecs = kmem_alloc(nr_iovecs * sizeof (iovec_t), KM_SLEEP); 1300 1301 /* 1302 * Copy the plain zil header over and authenticate everything except 1303 * the checksum that will store our MAC. If we are writing the data 1304 * the embedded checksum will not have been calculated yet, so we don't 1305 * authenticate that. 1306 */ 1307 bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); 1308 aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); 1309 aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); 1310 1311 slrp = src + sizeof (zil_chain_t); 1312 dlrp = dst + sizeof (zil_chain_t); 1313 1314 /* 1315 * Loop over records again, filling in iovecs. 1316 */ 1317 1318 /* The first iovec will contain the authbuf. */ 1319 vec = 1; 1320 1321 for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { 1322 lr = (lr_t *)slrp; 1323 1324 if (!byteswap) { 1325 txtype = lr->lrc_txtype; 1326 lr_len = lr->lrc_reclen; 1327 } else { 1328 txtype = BSWAP_64(lr->lrc_txtype); 1329 lr_len = BSWAP_64(lr->lrc_reclen); 1330 } 1331 1332 /* copy the common lr_t */ 1333 bcopy(slrp, dlrp, sizeof (lr_t)); 1334 bcopy(slrp, aadp, sizeof (lr_t)); 1335 aadp += sizeof (lr_t); 1336 aad_len += sizeof (lr_t); 1337 1338 /* 1339 * If this is a TX_WRITE record we want to encrypt everything 1340 * except the bp if exists. If the bp does exist we want to 1341 * authenticate it. 1342 */ 1343 if (txtype == TX_WRITE) { 1344 crypt_len = sizeof (lr_write_t) - 1345 sizeof (lr_t) - sizeof (blkptr_t); 1346 dst_iovecs[vec].iov_base = (char *)dlrp + 1347 sizeof (lr_t); 1348 dst_iovecs[vec].iov_len = crypt_len; 1349 1350 /* copy the bp now since it will not be encrypted */ 1351 bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1352 dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1353 sizeof (blkptr_t)); 1354 bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1355 aadp, sizeof (blkptr_t)); 1356 aadp += sizeof (blkptr_t); 1357 aad_len += sizeof (blkptr_t); 1358 vec++; 1359 total_len += crypt_len; 1360 1361 if (lr_len != sizeof (lr_write_t)) { 1362 crypt_len = lr_len - sizeof (lr_write_t); 1363 dst_iovecs[vec].iov_base = (char *) 1364 dlrp + sizeof (lr_write_t); 1365 dst_iovecs[vec].iov_len = crypt_len; 1366 vec++; 1367 total_len += crypt_len; 1368 } 1369 } else { 1370 crypt_len = lr_len - sizeof (lr_t); 1371 dst_iovecs[vec].iov_base = (char *)dlrp + 1372 sizeof (lr_t); 1373 dst_iovecs[vec].iov_len = crypt_len; 1374 vec++; 1375 total_len += crypt_len; 1376 } 1377 } 1378 1379 /* The last iovec will contain the MAC. */ 1380 ASSERT3U(vec, ==, nr_iovecs - 1); 1381 1382 /* AAD */ 1383 dst_iovecs[0].iov_base = aadbuf; 1384 dst_iovecs[0].iov_len = aad_len; 1385 /* MAC */ 1386 dst_iovecs[vec].iov_base = 0; 1387 dst_iovecs[vec].iov_len = 0; 1388 1389 *no_crypt = (vec == 1); 1390 *enc_len = total_len; 1391 *authbuf = aadbuf; 1392 *auth_len = aad_len; 1393 GET_UIO_STRUCT(out_uio)->uio_iov = dst_iovecs; 1394 zfs_uio_iovcnt(out_uio) = nr_iovecs; 1395 1396 return (0); 1397 } 1398 1399 /* 1400 * Special case handling routine for encrypting / decrypting dnode blocks. 1401 */ 1402 static int 1403 zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, 1404 uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, 1405 zfs_uio_t *puio, zfs_uio_t *out_uio, uint_t *enc_len, uint8_t **authbuf, 1406 uint_t *auth_len, boolean_t *no_crypt) 1407 { 1408 uint8_t *aadbuf = zio_buf_alloc(datalen); 1409 uint8_t *src, *dst, *aadp; 1410 dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; 1411 iovec_t *dst_iovecs; 1412 uint_t nr_iovecs, crypt_len, vec; 1413 uint_t aad_len = 0, total_len = 0; 1414 uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; 1415 1416 if (encrypt) { 1417 src = plainbuf; 1418 dst = cipherbuf; 1419 } else { 1420 src = cipherbuf; 1421 dst = plainbuf; 1422 } 1423 bcopy(src, dst, datalen); 1424 1425 sdnp = (dnode_phys_t *)src; 1426 ddnp = (dnode_phys_t *)dst; 1427 aadp = aadbuf; 1428 1429 /* 1430 * Count the number of iovecs we will need to do the encryption by 1431 * counting the number of bonus buffers that need to be encrypted. 1432 */ 1433 1434 /* We need at least two iovecs -- one for the AAD, one for the MAC. */ 1435 nr_iovecs = 2; 1436 1437 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 1438 /* 1439 * This block may still be byteswapped. However, all of the 1440 * values we use are either uint8_t's (for which byteswapping 1441 * is a noop) or a * != 0 check, which will work regardless 1442 * of whether or not we byteswap. 1443 */ 1444 if (sdnp[i].dn_type != DMU_OT_NONE && 1445 DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && 1446 sdnp[i].dn_bonuslen != 0) { 1447 nr_iovecs++; 1448 } 1449 } 1450 1451 dst_iovecs = kmem_alloc(nr_iovecs * sizeof (iovec_t), KM_SLEEP); 1452 1453 /* 1454 * Iterate through the dnodes again, this time filling in the uios 1455 * we allocated earlier. We also concatenate any data we want to 1456 * authenticate onto aadbuf. 1457 */ 1458 1459 /* The first iovec will contain the authbuf. */ 1460 vec = 1; 1461 1462 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 1463 dnp = &sdnp[i]; 1464 1465 /* copy over the core fields and blkptrs (kept as plaintext) */ 1466 bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); 1467 1468 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1469 bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), 1470 sizeof (blkptr_t)); 1471 } 1472 1473 /* 1474 * Handle authenticated data. We authenticate everything in 1475 * the dnode that can be brought over when we do a raw send. 1476 * This includes all of the core fields as well as the MACs 1477 * stored in the bp checksums and all of the portable bits 1478 * from blk_prop. We include the dnode padding here in case it 1479 * ever gets used in the future. Some dn_flags and dn_used are 1480 * not portable so we mask those out values out of the 1481 * authenticated data. 1482 */ 1483 crypt_len = offsetof(dnode_phys_t, dn_blkptr); 1484 bcopy(dnp, aadp, crypt_len); 1485 adnp = (dnode_phys_t *)aadp; 1486 adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; 1487 adnp->dn_used = 0; 1488 aadp += crypt_len; 1489 aad_len += crypt_len; 1490 1491 for (j = 0; j < dnp->dn_nblkptr; j++) { 1492 zio_crypt_bp_do_aad_updates(&aadp, &aad_len, 1493 version, byteswap, &dnp->dn_blkptr[j]); 1494 } 1495 1496 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1497 zio_crypt_bp_do_aad_updates(&aadp, &aad_len, 1498 version, byteswap, DN_SPILL_BLKPTR(dnp)); 1499 } 1500 1501 /* 1502 * If this bonus buffer needs to be encrypted, we prepare an 1503 * iovec_t. The encryption / decryption functions will fill 1504 * this in for us with the encrypted or decrypted data. 1505 * Otherwise we add the bonus buffer to the authenticated 1506 * data buffer and copy it over to the destination. The 1507 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that 1508 * we can guarantee alignment with the AES block size 1509 * (128 bits). 1510 */ 1511 crypt_len = DN_MAX_BONUS_LEN(dnp); 1512 if (dnp->dn_type != DMU_OT_NONE && 1513 DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && 1514 dnp->dn_bonuslen != 0) { 1515 dst_iovecs[vec].iov_base = DN_BONUS(&ddnp[i]); 1516 dst_iovecs[vec].iov_len = crypt_len; 1517 1518 vec++; 1519 total_len += crypt_len; 1520 } else { 1521 bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); 1522 bcopy(DN_BONUS(dnp), aadp, crypt_len); 1523 aadp += crypt_len; 1524 aad_len += crypt_len; 1525 } 1526 } 1527 1528 /* The last iovec will contain the MAC. */ 1529 ASSERT3U(vec, ==, nr_iovecs - 1); 1530 1531 /* AAD */ 1532 dst_iovecs[0].iov_base = aadbuf; 1533 dst_iovecs[0].iov_len = aad_len; 1534 /* MAC */ 1535 dst_iovecs[vec].iov_base = 0; 1536 dst_iovecs[vec].iov_len = 0; 1537 1538 *no_crypt = (vec == 1); 1539 *enc_len = total_len; 1540 *authbuf = aadbuf; 1541 *auth_len = aad_len; 1542 GET_UIO_STRUCT(out_uio)->uio_iov = dst_iovecs; 1543 zfs_uio_iovcnt(out_uio) = nr_iovecs; 1544 1545 return (0); 1546 } 1547 1548 /* ARGSUSED */ 1549 static int 1550 zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, 1551 uint8_t *cipherbuf, uint_t datalen, zfs_uio_t *puio, zfs_uio_t *out_uio, 1552 uint_t *enc_len) 1553 { 1554 int ret; 1555 uint_t nr_plain = 1, nr_cipher = 2; 1556 iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; 1557 void *src, *dst; 1558 1559 cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), 1560 KM_SLEEP); 1561 if (!cipher_iovecs) { 1562 ret = SET_ERROR(ENOMEM); 1563 goto error; 1564 } 1565 bzero(cipher_iovecs, nr_cipher * sizeof (iovec_t)); 1566 1567 if (encrypt) { 1568 src = plainbuf; 1569 dst = cipherbuf; 1570 } else { 1571 src = cipherbuf; 1572 dst = plainbuf; 1573 } 1574 bcopy(src, dst, datalen); 1575 cipher_iovecs[0].iov_base = dst; 1576 cipher_iovecs[0].iov_len = datalen; 1577 1578 *enc_len = datalen; 1579 GET_UIO_STRUCT(out_uio)->uio_iov = cipher_iovecs; 1580 zfs_uio_iovcnt(out_uio) = nr_cipher; 1581 1582 return (0); 1583 1584 error: 1585 if (plain_iovecs != NULL) 1586 kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); 1587 if (cipher_iovecs != NULL) 1588 kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); 1589 1590 *enc_len = 0; 1591 GET_UIO_STRUCT(out_uio)->uio_iov = NULL; 1592 zfs_uio_iovcnt(out_uio) = 0; 1593 1594 return (ret); 1595 } 1596 1597 /* 1598 * This function builds up the plaintext (puio) and ciphertext (cuio) uios so 1599 * that they can be used for encryption and decryption by zio_do_crypt_uio(). 1600 * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks 1601 * requiring special handling to parse out pieces that are to be encrypted. The 1602 * authbuf is used by these special cases to store additional authenticated 1603 * data (AAD) for the encryption modes. 1604 */ 1605 static int 1606 zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, 1607 uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, 1608 uint8_t *mac, zfs_uio_t *puio, zfs_uio_t *cuio, uint_t *enc_len, 1609 uint8_t **authbuf, uint_t *auth_len, boolean_t *no_crypt) 1610 { 1611 int ret; 1612 iovec_t *mac_iov; 1613 1614 ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); 1615 1616 /* route to handler */ 1617 switch (ot) { 1618 case DMU_OT_INTENT_LOG: 1619 ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, 1620 datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, 1621 no_crypt); 1622 break; 1623 case DMU_OT_DNODE: 1624 ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, 1625 cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, 1626 auth_len, no_crypt); 1627 break; 1628 default: 1629 ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, 1630 datalen, puio, cuio, enc_len); 1631 *authbuf = NULL; 1632 *auth_len = 0; 1633 *no_crypt = B_FALSE; 1634 break; 1635 } 1636 1637 if (ret != 0) 1638 goto error; 1639 1640 /* populate the uios */ 1641 zfs_uio_segflg(cuio) = UIO_SYSSPACE; 1642 1643 mac_iov = 1644 ((iovec_t *)&(GET_UIO_STRUCT(cuio)-> 1645 uio_iov[zfs_uio_iovcnt(cuio) - 1])); 1646 mac_iov->iov_base = (void *)mac; 1647 mac_iov->iov_len = ZIO_DATA_MAC_LEN; 1648 1649 return (0); 1650 1651 error: 1652 return (ret); 1653 } 1654 1655 void *failed_decrypt_buf; 1656 int faile_decrypt_size; 1657 1658 /* 1659 * Primary encryption / decryption entrypoint for zio data. 1660 */ 1661 int 1662 zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, 1663 dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, 1664 uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, 1665 boolean_t *no_crypt) 1666 { 1667 int ret; 1668 boolean_t locked = B_FALSE; 1669 uint64_t crypt = key->zk_crypt; 1670 uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; 1671 uint_t enc_len, auth_len; 1672 zfs_uio_t puio, cuio; 1673 struct uio puio_s, cuio_s; 1674 uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; 1675 crypto_key_t tmp_ckey, *ckey = NULL; 1676 freebsd_crypt_session_t *tmpl = NULL; 1677 uint8_t *authbuf = NULL; 1678 1679 1680 zfs_uio_init(&puio, &puio_s); 1681 zfs_uio_init(&cuio, &cuio_s); 1682 bzero(GET_UIO_STRUCT(&puio), sizeof (struct uio)); 1683 bzero(GET_UIO_STRUCT(&cuio), sizeof (struct uio)); 1684 1685 #ifdef FCRYPTO_DEBUG 1686 printf("%s(%s, %p, %p, %d, %p, %p, %u, %s, %p, %p, %p)\n", 1687 __FUNCTION__, 1688 encrypt ? "encrypt" : "decrypt", 1689 key, salt, ot, iv, mac, datalen, 1690 byteswap ? "byteswap" : "native_endian", plainbuf, 1691 cipherbuf, no_crypt); 1692 1693 printf("\tkey = {"); 1694 for (int i = 0; i < key->zk_current_key.ck_length/8; i++) 1695 printf("%02x ", ((uint8_t *)key->zk_current_key.ck_data)[i]); 1696 printf("}\n"); 1697 #endif 1698 /* create uios for encryption */ 1699 ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, 1700 cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, 1701 &authbuf, &auth_len, no_crypt); 1702 if (ret != 0) 1703 return (ret); 1704 1705 /* 1706 * If the needed key is the current one, just use it. Otherwise we 1707 * need to generate a temporary one from the given salt + master key. 1708 * If we are encrypting, we must return a copy of the current salt 1709 * so that it can be stored in the blkptr_t. 1710 */ 1711 rw_enter(&key->zk_salt_lock, RW_READER); 1712 locked = B_TRUE; 1713 1714 if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { 1715 ckey = &key->zk_current_key; 1716 tmpl = &key->zk_session; 1717 } else { 1718 rw_exit(&key->zk_salt_lock); 1719 locked = B_FALSE; 1720 1721 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 1722 salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); 1723 if (ret != 0) 1724 goto error; 1725 tmp_ckey.ck_format = CRYPTO_KEY_RAW; 1726 tmp_ckey.ck_data = enc_keydata; 1727 tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); 1728 1729 ckey = &tmp_ckey; 1730 tmpl = NULL; 1731 } 1732 1733 /* perform the encryption / decryption */ 1734 ret = zio_do_crypt_uio_opencrypto(encrypt, tmpl, key->zk_crypt, 1735 ckey, iv, enc_len, &cuio, auth_len); 1736 if (ret != 0) 1737 goto error; 1738 if (locked) { 1739 rw_exit(&key->zk_salt_lock); 1740 locked = B_FALSE; 1741 } 1742 1743 if (authbuf != NULL) 1744 zio_buf_free(authbuf, datalen); 1745 if (ckey == &tmp_ckey) 1746 bzero(enc_keydata, keydata_len); 1747 zio_crypt_destroy_uio(&puio); 1748 zio_crypt_destroy_uio(&cuio); 1749 1750 return (0); 1751 1752 error: 1753 if (!encrypt) { 1754 if (failed_decrypt_buf != NULL) 1755 kmem_free(failed_decrypt_buf, failed_decrypt_size); 1756 failed_decrypt_buf = kmem_alloc(datalen, KM_SLEEP); 1757 failed_decrypt_size = datalen; 1758 bcopy(cipherbuf, failed_decrypt_buf, datalen); 1759 } 1760 if (locked) 1761 rw_exit(&key->zk_salt_lock); 1762 if (authbuf != NULL) 1763 zio_buf_free(authbuf, datalen); 1764 if (ckey == &tmp_ckey) 1765 bzero(enc_keydata, keydata_len); 1766 zio_crypt_destroy_uio(&puio); 1767 zio_crypt_destroy_uio(&cuio); 1768 return (SET_ERROR(ret)); 1769 } 1770 1771 /* 1772 * Simple wrapper around zio_do_crypt_data() to work with abd's instead of 1773 * linear buffers. 1774 */ 1775 int 1776 zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, 1777 boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, 1778 uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) 1779 { 1780 int ret; 1781 void *ptmp, *ctmp; 1782 1783 if (encrypt) { 1784 ptmp = abd_borrow_buf_copy(pabd, datalen); 1785 ctmp = abd_borrow_buf(cabd, datalen); 1786 } else { 1787 ptmp = abd_borrow_buf(pabd, datalen); 1788 ctmp = abd_borrow_buf_copy(cabd, datalen); 1789 } 1790 1791 ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, 1792 datalen, ptmp, ctmp, no_crypt); 1793 if (ret != 0) 1794 goto error; 1795 1796 if (encrypt) { 1797 abd_return_buf(pabd, ptmp, datalen); 1798 abd_return_buf_copy(cabd, ctmp, datalen); 1799 } else { 1800 abd_return_buf_copy(pabd, ptmp, datalen); 1801 abd_return_buf(cabd, ctmp, datalen); 1802 } 1803 1804 return (0); 1805 1806 error: 1807 if (encrypt) { 1808 abd_return_buf(pabd, ptmp, datalen); 1809 abd_return_buf_copy(cabd, ctmp, datalen); 1810 } else { 1811 abd_return_buf_copy(pabd, ptmp, datalen); 1812 abd_return_buf(cabd, ctmp, datalen); 1813 } 1814 1815 return (SET_ERROR(ret)); 1816 } 1817 1818 #if defined(_KERNEL) && defined(HAVE_SPL) 1819 /* BEGIN CSTYLED */ 1820 module_param(zfs_key_max_salt_uses, ulong, 0644); 1821 MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " 1822 "can be used for generating encryption keys before it is rotated"); 1823 /* END CSTYLED */ 1824 #endif 1825