1 /* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16 /* 17 * Copyright (c) 2017, Datto, Inc. All rights reserved. 18 */ 19 20 #include <sys/zio_crypt.h> 21 #include <sys/dmu.h> 22 #include <sys/dmu_objset.h> 23 #include <sys/dnode.h> 24 #include <sys/fs/zfs.h> 25 #include <sys/zio.h> 26 #include <sys/zil.h> 27 #include <sys/sha2.h> 28 #include <sys/hkdf.h> 29 #include <sys/qat.h> 30 31 /* 32 * This file is responsible for handling all of the details of generating 33 * encryption parameters and performing encryption and authentication. 34 * 35 * BLOCK ENCRYPTION PARAMETERS: 36 * Encryption /Authentication Algorithm Suite (crypt): 37 * The encryption algorithm, mode, and key length we are going to use. We 38 * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit 39 * keys. All authentication is currently done with SHA512-HMAC. 40 * 41 * Plaintext: 42 * The unencrypted data that we want to encrypt. 43 * 44 * Initialization Vector (IV): 45 * An initialization vector for the encryption algorithms. This is used to 46 * "tweak" the encryption algorithms so that two blocks of the same data are 47 * encrypted into different ciphertext outputs, thus obfuscating block patterns. 48 * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is 49 * never reused with the same encryption key. This value is stored unencrypted 50 * and must simply be provided to the decryption function. We use a 96 bit IV 51 * (as recommended by NIST) for all block encryption. For non-dedup blocks we 52 * derive the IV randomly. The first 64 bits of the IV are stored in the second 53 * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of 54 * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits 55 * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count 56 * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of 57 * level 0 blocks is the number of allocated dnodes in that block. The on-disk 58 * format supports at most 2^15 slots per L0 dnode block, because the maximum 59 * block size is 16MB (2^24). In either case, for level 0 blocks this number 60 * will still be smaller than UINT32_MAX so it is safe to store the IV in the 61 * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count 62 * for the dnode code. 63 * 64 * Master key: 65 * This is the most important secret data of an encrypted dataset. It is used 66 * along with the salt to generate that actual encryption keys via HKDF. We 67 * do not use the master key to directly encrypt any data because there are 68 * theoretical limits on how much data can actually be safely encrypted with 69 * any encryption mode. The master key is stored encrypted on disk with the 70 * user's wrapping key. Its length is determined by the encryption algorithm. 71 * For details on how this is stored see the block comment in dsl_crypt.c 72 * 73 * Salt: 74 * Used as an input to the HKDF function, along with the master key. We use a 75 * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt 76 * can be used for encrypting many blocks, so we cache the current salt and the 77 * associated derived key in zio_crypt_t so we do not need to derive it again 78 * needlessly. 79 * 80 * Encryption Key: 81 * A secret binary key, generated from an HKDF function used to encrypt and 82 * decrypt data. 83 * 84 * Message Authentication Code (MAC) 85 * The MAC is an output of authenticated encryption modes such as AES-GCM and 86 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted 87 * data on disk and return garbage to the application. Effectively, it is a 88 * checksum that can not be reproduced by an attacker. We store the MAC in the 89 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated 90 * regular checksum of the ciphertext which can be used for scrubbing. 91 * 92 * OBJECT AUTHENTICATION: 93 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because 94 * they contain some info that always needs to be readable. To prevent this 95 * data from being altered, we authenticate this data using SHA512-HMAC. This 96 * will produce a MAC (similar to the one produced via encryption) which can 97 * be used to verify the object was not modified. HMACs do not require key 98 * rotation or IVs, so we can keep up to the full 3 copies of authenticated 99 * data. 100 * 101 * ZIL ENCRYPTION: 102 * ZIL blocks have their bp written to disk ahead of the associated data, so we 103 * cannot store the MAC there as we normally do. For these blocks the MAC is 104 * stored in the embedded checksum within the zil_chain_t header. The salt and 105 * IV are generated for the block on bp allocation instead of at encryption 106 * time. In addition, ZIL blocks have some pieces that must be left in plaintext 107 * for claiming even though all of the sensitive user data still needs to be 108 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which 109 * pieces of the block need to be encrypted. All data that is not encrypted is 110 * authenticated using the AAD mechanisms that the supported encryption modes 111 * provide for. In order to preserve the semantics of the ZIL for encrypted 112 * datasets, the ZIL is not protected at the objset level as described below. 113 * 114 * DNODE ENCRYPTION: 115 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left 116 * in plaintext for scrubbing and claiming, but the bonus buffers might contain 117 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing 118 * which which pieces of the block need to be encrypted. For more details about 119 * dnode authentication and encryption, see zio_crypt_init_uios_dnode(). 120 * 121 * OBJECT SET AUTHENTICATION: 122 * Up to this point, everything we have encrypted and authenticated has been 123 * at level 0 (or -2 for the ZIL). If we did not do any further work the 124 * on-disk format would be susceptible to attacks that deleted or rearranged 125 * the order of level 0 blocks. Ideally, the cleanest solution would be to 126 * maintain a tree of authentication MACs going up the bp tree. However, this 127 * presents a problem for raw sends. Send files do not send information about 128 * indirect blocks so there would be no convenient way to transfer the MACs and 129 * they cannot be recalculated on the receive side without the master key which 130 * would defeat one of the purposes of raw sends in the first place. Instead, 131 * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs 132 * from the level below. We also include some portable fields from blk_prop such 133 * as the lsize and compression algorithm to prevent the data from being 134 * misinterpreted. 135 * 136 * At the objset level, we maintain 2 separate 256 bit MACs in the 137 * objset_phys_t. The first one is "portable" and is the logical root of the 138 * MAC tree maintained in the metadnode's bps. The second, is "local" and is 139 * used as the root MAC for the user accounting objects, which are also not 140 * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload 141 * of the send file. The useraccounting code ensures that the useraccounting 142 * info is not present upon a receive, so the local MAC can simply be cleared 143 * out at that time. For more info about objset_phys_t authentication, see 144 * zio_crypt_do_objset_hmacs(). 145 * 146 * CONSIDERATIONS FOR DEDUP: 147 * In order for dedup to work, blocks that we want to dedup with one another 148 * need to use the same IV and encryption key, so that they will have the same 149 * ciphertext. Normally, one should never reuse an IV with the same encryption 150 * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both 151 * blocks. In this case, however, since we are using the same plaintext as 152 * well all that we end up with is a duplicate of the original ciphertext we 153 * already had. As a result, an attacker with read access to the raw disk will 154 * be able to tell which blocks are the same but this information is given away 155 * by dedup anyway. In order to get the same IVs and encryption keys for 156 * equivalent blocks of data we use an HMAC of the plaintext. We use an HMAC 157 * here so that a reproducible checksum of the plaintext is never available to 158 * the attacker. The HMAC key is kept alongside the master key, encrypted on 159 * disk. The first 64 bits of the HMAC are used in place of the random salt, and 160 * the next 96 bits are used as the IV. As a result of this mechanism, dedup 161 * will only work within a clone family since encrypted dedup requires use of 162 * the same master and HMAC keys. 163 */ 164 165 /* 166 * After encrypting many blocks with the same key we may start to run up 167 * against the theoretical limits of how much data can securely be encrypted 168 * with a single key using the supported encryption modes. The most obvious 169 * limitation is that our risk of generating 2 equivalent 96 bit IVs increases 170 * the more IVs we generate (which both GCM and CCM modes strictly forbid). 171 * This risk actually grows surprisingly quickly over time according to the 172 * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have 173 * generated n IVs with a cryptographically secure RNG, the approximate 174 * probability p(n) of a collision is given as: 175 * 176 * p(n) ~= e^(-n*(n-1)/(2*(2^96))) 177 * 178 * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html] 179 * 180 * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion 181 * we must not write more than 398,065,730 blocks with the same encryption key. 182 * Therefore, we rotate our keys after 400,000,000 blocks have been written by 183 * generating a new random 64 bit salt for our HKDF encryption key generation 184 * function. 185 */ 186 #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000 187 #define ZFS_CURRENT_MAX_SALT_USES \ 188 (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT)) 189 unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT; 190 191 typedef struct blkptr_auth_buf { 192 uint64_t bab_prop; /* blk_prop - portable mask */ 193 uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */ 194 uint64_t bab_pad; /* reserved for future use */ 195 } blkptr_auth_buf_t; 196 197 zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = { 198 {"", ZC_TYPE_NONE, 0, "inherit"}, 199 {"", ZC_TYPE_NONE, 0, "on"}, 200 {"", ZC_TYPE_NONE, 0, "off"}, 201 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"}, 202 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"}, 203 {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"}, 204 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"}, 205 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"}, 206 {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"} 207 }; 208 209 void 210 zio_crypt_key_destroy(zio_crypt_key_t *key) 211 { 212 rw_destroy(&key->zk_salt_lock); 213 214 /* free crypto templates */ 215 crypto_destroy_ctx_template(key->zk_current_tmpl); 216 crypto_destroy_ctx_template(key->zk_hmac_tmpl); 217 218 /* zero out sensitive data */ 219 bzero(key, sizeof (zio_crypt_key_t)); 220 } 221 222 int 223 zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key) 224 { 225 int ret; 226 crypto_mechanism_t mech; 227 uint_t keydata_len; 228 229 ASSERT(key != NULL); 230 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 231 232 keydata_len = zio_crypt_table[crypt].ci_keylen; 233 bzero(key, sizeof (zio_crypt_key_t)); 234 235 /* fill keydata buffers and salt with random data */ 236 ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t)); 237 if (ret != 0) 238 goto error; 239 240 ret = random_get_bytes(key->zk_master_keydata, keydata_len); 241 if (ret != 0) 242 goto error; 243 244 ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN); 245 if (ret != 0) 246 goto error; 247 248 ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); 249 if (ret != 0) 250 goto error; 251 252 /* derive the current key from the master key */ 253 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 254 key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, 255 keydata_len); 256 if (ret != 0) 257 goto error; 258 259 /* initialize keys for the ICP */ 260 key->zk_current_key.ck_format = CRYPTO_KEY_RAW; 261 key->zk_current_key.ck_data = key->zk_current_keydata; 262 key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); 263 264 key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; 265 key->zk_hmac_key.ck_data = &key->zk_hmac_key; 266 key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); 267 268 /* 269 * Initialize the crypto templates. It's ok if this fails because 270 * this is just an optimization. 271 */ 272 mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); 273 ret = crypto_create_ctx_template(&mech, &key->zk_current_key, 274 &key->zk_current_tmpl, KM_SLEEP); 275 if (ret != CRYPTO_SUCCESS) 276 key->zk_current_tmpl = NULL; 277 278 mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); 279 ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, 280 &key->zk_hmac_tmpl, KM_SLEEP); 281 if (ret != CRYPTO_SUCCESS) 282 key->zk_hmac_tmpl = NULL; 283 284 key->zk_crypt = crypt; 285 key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION; 286 key->zk_salt_count = 0; 287 rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); 288 289 return (0); 290 291 error: 292 zio_crypt_key_destroy(key); 293 return (ret); 294 } 295 296 static int 297 zio_crypt_key_change_salt(zio_crypt_key_t *key) 298 { 299 int ret = 0; 300 uint8_t salt[ZIO_DATA_SALT_LEN]; 301 crypto_mechanism_t mech; 302 uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen; 303 304 /* generate a new salt */ 305 ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN); 306 if (ret != 0) 307 goto error; 308 309 rw_enter(&key->zk_salt_lock, RW_WRITER); 310 311 /* someone beat us to the salt rotation, just unlock and return */ 312 if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES) 313 goto out_unlock; 314 315 /* derive the current key from the master key and the new salt */ 316 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 317 salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len); 318 if (ret != 0) 319 goto out_unlock; 320 321 /* assign the salt and reset the usage count */ 322 bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN); 323 key->zk_salt_count = 0; 324 325 /* destroy the old context template and create the new one */ 326 crypto_destroy_ctx_template(key->zk_current_tmpl); 327 ret = crypto_create_ctx_template(&mech, &key->zk_current_key, 328 &key->zk_current_tmpl, KM_SLEEP); 329 if (ret != CRYPTO_SUCCESS) 330 key->zk_current_tmpl = NULL; 331 332 rw_exit(&key->zk_salt_lock); 333 334 return (0); 335 336 out_unlock: 337 rw_exit(&key->zk_salt_lock); 338 error: 339 return (ret); 340 } 341 342 /* See comment above zfs_key_max_salt_uses definition for details */ 343 int 344 zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt) 345 { 346 int ret; 347 boolean_t salt_change; 348 349 rw_enter(&key->zk_salt_lock, RW_READER); 350 351 bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN); 352 salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >= 353 ZFS_CURRENT_MAX_SALT_USES); 354 355 rw_exit(&key->zk_salt_lock); 356 357 if (salt_change) { 358 ret = zio_crypt_key_change_salt(key); 359 if (ret != 0) 360 goto error; 361 } 362 363 return (0); 364 365 error: 366 return (ret); 367 } 368 369 /* 370 * This function handles all encryption and decryption in zfs. When 371 * encrypting it expects puio to reference the plaintext and cuio to 372 * reference the ciphertext. cuio must have enough space for the 373 * ciphertext + room for a MAC. datalen should be the length of the 374 * plaintext / ciphertext alone. 375 */ 376 static int 377 zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key, 378 crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen, 379 uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len) 380 { 381 int ret; 382 crypto_data_t plaindata, cipherdata; 383 CK_AES_CCM_PARAMS ccmp; 384 CK_AES_GCM_PARAMS gcmp; 385 crypto_mechanism_t mech; 386 zio_crypt_info_t crypt_info; 387 uint_t plain_full_len, maclen; 388 389 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 390 ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW); 391 392 /* lookup the encryption info */ 393 crypt_info = zio_crypt_table[crypt]; 394 395 /* the mac will always be the last iovec_t in the cipher uio */ 396 maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len; 397 398 ASSERT(maclen <= ZIO_DATA_MAC_LEN); 399 400 /* setup encryption mechanism (same as crypt) */ 401 mech.cm_type = crypto_mech2id(crypt_info.ci_mechname); 402 403 /* 404 * Strangely, the ICP requires that plain_full_len must include 405 * the MAC length when decrypting, even though the UIO does not 406 * need to have the extra space allocated. 407 */ 408 if (encrypt) { 409 plain_full_len = datalen; 410 } else { 411 plain_full_len = datalen + maclen; 412 } 413 414 /* 415 * setup encryption params (currently only AES CCM and AES GCM 416 * are supported) 417 */ 418 if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) { 419 ccmp.ulNonceSize = ZIO_DATA_IV_LEN; 420 ccmp.ulAuthDataSize = auth_len; 421 ccmp.authData = authbuf; 422 ccmp.ulMACSize = maclen; 423 ccmp.nonce = ivbuf; 424 ccmp.ulDataSize = plain_full_len; 425 426 mech.cm_param = (char *)(&ccmp); 427 mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS); 428 } else { 429 gcmp.ulIvLen = ZIO_DATA_IV_LEN; 430 gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN); 431 gcmp.ulAADLen = auth_len; 432 gcmp.pAAD = authbuf; 433 gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen); 434 gcmp.pIv = ivbuf; 435 436 mech.cm_param = (char *)(&gcmp); 437 mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); 438 } 439 440 /* populate the cipher and plain data structs. */ 441 plaindata.cd_format = CRYPTO_DATA_UIO; 442 plaindata.cd_offset = 0; 443 plaindata.cd_uio = puio; 444 plaindata.cd_miscdata = NULL; 445 plaindata.cd_length = plain_full_len; 446 447 cipherdata.cd_format = CRYPTO_DATA_UIO; 448 cipherdata.cd_offset = 0; 449 cipherdata.cd_uio = cuio; 450 cipherdata.cd_miscdata = NULL; 451 cipherdata.cd_length = datalen + maclen; 452 453 /* perform the actual encryption */ 454 if (encrypt) { 455 ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata, 456 NULL); 457 if (ret != CRYPTO_SUCCESS) { 458 ret = SET_ERROR(EIO); 459 goto error; 460 } 461 } else { 462 ret = crypto_decrypt(&mech, &cipherdata, key, tmpl, &plaindata, 463 NULL); 464 if (ret != CRYPTO_SUCCESS) { 465 ASSERT3U(ret, ==, CRYPTO_INVALID_MAC); 466 ret = SET_ERROR(ECKSUM); 467 goto error; 468 } 469 } 470 471 return (0); 472 473 error: 474 return (ret); 475 } 476 477 int 478 zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv, 479 uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out) 480 { 481 int ret; 482 uio_t puio, cuio; 483 uint64_t aad[3]; 484 iovec_t plain_iovecs[2], cipher_iovecs[3]; 485 uint64_t crypt = key->zk_crypt; 486 uint_t enc_len, keydata_len, aad_len; 487 488 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 489 ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); 490 491 keydata_len = zio_crypt_table[crypt].ci_keylen; 492 493 /* generate iv for wrapping the master and hmac key */ 494 ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN); 495 if (ret != 0) 496 goto error; 497 498 /* initialize uio_ts */ 499 plain_iovecs[0].iov_base = key->zk_master_keydata; 500 plain_iovecs[0].iov_len = keydata_len; 501 plain_iovecs[1].iov_base = key->zk_hmac_keydata; 502 plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; 503 504 cipher_iovecs[0].iov_base = keydata_out; 505 cipher_iovecs[0].iov_len = keydata_len; 506 cipher_iovecs[1].iov_base = hmac_keydata_out; 507 cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; 508 cipher_iovecs[2].iov_base = mac; 509 cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; 510 511 /* 512 * Although we don't support writing to the old format, we do 513 * support rewrapping the key so that the user can move and 514 * quarantine datasets on the old format. 515 */ 516 if (key->zk_version == 0) { 517 aad_len = sizeof (uint64_t); 518 aad[0] = LE_64(key->zk_guid); 519 } else { 520 ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 521 aad_len = sizeof (uint64_t) * 3; 522 aad[0] = LE_64(key->zk_guid); 523 aad[1] = LE_64(crypt); 524 aad[2] = LE_64(key->zk_version); 525 } 526 527 enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN; 528 puio.uio_iov = plain_iovecs; 529 puio.uio_iovcnt = 2; 530 puio.uio_segflg = UIO_SYSSPACE; 531 cuio.uio_iov = cipher_iovecs; 532 cuio.uio_iovcnt = 3; 533 cuio.uio_segflg = UIO_SYSSPACE; 534 535 /* encrypt the keys and store the resulting ciphertext and mac */ 536 ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len, 537 &puio, &cuio, (uint8_t *)aad, aad_len); 538 if (ret != 0) 539 goto error; 540 541 return (0); 542 543 error: 544 return (ret); 545 } 546 547 int 548 zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version, 549 uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv, 550 uint8_t *mac, zio_crypt_key_t *key) 551 { 552 crypto_mechanism_t mech; 553 uio_t puio, cuio; 554 uint64_t aad[3]; 555 iovec_t plain_iovecs[2], cipher_iovecs[3]; 556 uint_t enc_len, keydata_len, aad_len; 557 int ret; 558 559 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 560 ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW); 561 562 rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL); 563 564 keydata_len = zio_crypt_table[crypt].ci_keylen; 565 566 /* initialize uio_ts */ 567 plain_iovecs[0].iov_base = key->zk_master_keydata; 568 plain_iovecs[0].iov_len = keydata_len; 569 plain_iovecs[1].iov_base = key->zk_hmac_keydata; 570 plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; 571 572 cipher_iovecs[0].iov_base = keydata; 573 cipher_iovecs[0].iov_len = keydata_len; 574 cipher_iovecs[1].iov_base = hmac_keydata; 575 cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN; 576 cipher_iovecs[2].iov_base = mac; 577 cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN; 578 579 if (version == 0) { 580 aad_len = sizeof (uint64_t); 581 aad[0] = LE_64(guid); 582 } else { 583 ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 584 aad_len = sizeof (uint64_t) * 3; 585 aad[0] = LE_64(guid); 586 aad[1] = LE_64(crypt); 587 aad[2] = LE_64(version); 588 } 589 590 enc_len = keydata_len + SHA512_HMAC_KEYLEN; 591 puio.uio_iov = plain_iovecs; 592 puio.uio_segflg = UIO_SYSSPACE; 593 puio.uio_iovcnt = 2; 594 cuio.uio_iov = cipher_iovecs; 595 cuio.uio_iovcnt = 3; 596 cuio.uio_segflg = UIO_SYSSPACE; 597 598 /* decrypt the keys and store the result in the output buffers */ 599 ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len, 600 &puio, &cuio, (uint8_t *)aad, aad_len); 601 if (ret != 0) 602 goto error; 603 604 /* generate a fresh salt */ 605 ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN); 606 if (ret != 0) 607 goto error; 608 609 /* derive the current key from the master key */ 610 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 611 key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, 612 keydata_len); 613 if (ret != 0) 614 goto error; 615 616 /* initialize keys for ICP */ 617 key->zk_current_key.ck_format = CRYPTO_KEY_RAW; 618 key->zk_current_key.ck_data = key->zk_current_keydata; 619 key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len); 620 621 key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW; 622 key->zk_hmac_key.ck_data = key->zk_hmac_keydata; 623 key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN); 624 625 /* 626 * Initialize the crypto templates. It's ok if this fails because 627 * this is just an optimization. 628 */ 629 mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname); 630 ret = crypto_create_ctx_template(&mech, &key->zk_current_key, 631 &key->zk_current_tmpl, KM_SLEEP); 632 if (ret != CRYPTO_SUCCESS) 633 key->zk_current_tmpl = NULL; 634 635 mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); 636 ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key, 637 &key->zk_hmac_tmpl, KM_SLEEP); 638 if (ret != CRYPTO_SUCCESS) 639 key->zk_hmac_tmpl = NULL; 640 641 key->zk_crypt = crypt; 642 key->zk_version = version; 643 key->zk_guid = guid; 644 key->zk_salt_count = 0; 645 646 return (0); 647 648 error: 649 zio_crypt_key_destroy(key); 650 return (ret); 651 } 652 653 int 654 zio_crypt_generate_iv(uint8_t *ivbuf) 655 { 656 int ret; 657 658 /* randomly generate the IV */ 659 ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN); 660 if (ret != 0) 661 goto error; 662 663 return (0); 664 665 error: 666 bzero(ivbuf, ZIO_DATA_IV_LEN); 667 return (ret); 668 } 669 670 int 671 zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen, 672 uint8_t *digestbuf, uint_t digestlen) 673 { 674 int ret; 675 crypto_mechanism_t mech; 676 crypto_data_t in_data, digest_data; 677 uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH]; 678 679 ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH); 680 681 /* initialize sha512-hmac mechanism and crypto data */ 682 mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); 683 mech.cm_param = NULL; 684 mech.cm_param_len = 0; 685 686 /* initialize the crypto data */ 687 in_data.cd_format = CRYPTO_DATA_RAW; 688 in_data.cd_offset = 0; 689 in_data.cd_length = datalen; 690 in_data.cd_raw.iov_base = (char *)data; 691 in_data.cd_raw.iov_len = in_data.cd_length; 692 693 digest_data.cd_format = CRYPTO_DATA_RAW; 694 digest_data.cd_offset = 0; 695 digest_data.cd_length = SHA512_DIGEST_LENGTH; 696 digest_data.cd_raw.iov_base = (char *)raw_digestbuf; 697 digest_data.cd_raw.iov_len = digest_data.cd_length; 698 699 /* generate the hmac */ 700 ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl, 701 &digest_data, NULL); 702 if (ret != CRYPTO_SUCCESS) { 703 ret = SET_ERROR(EIO); 704 goto error; 705 } 706 707 bcopy(raw_digestbuf, digestbuf, digestlen); 708 709 return (0); 710 711 error: 712 bzero(digestbuf, digestlen); 713 return (ret); 714 } 715 716 int 717 zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data, 718 uint_t datalen, uint8_t *ivbuf, uint8_t *salt) 719 { 720 int ret; 721 uint8_t digestbuf[SHA512_DIGEST_LENGTH]; 722 723 ret = zio_crypt_do_hmac(key, data, datalen, 724 digestbuf, SHA512_DIGEST_LENGTH); 725 if (ret != 0) 726 return (ret); 727 728 bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN); 729 bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN); 730 731 return (0); 732 } 733 734 /* 735 * The following functions are used to encode and decode encryption parameters 736 * into blkptr_t and zil_header_t. The ICP wants to use these parameters as 737 * byte strings, which normally means that these strings would not need to deal 738 * with byteswapping at all. However, both blkptr_t and zil_header_t may be 739 * byteswapped by lower layers and so we must "undo" that byteswap here upon 740 * decoding and encoding in a non-native byteorder. These functions require 741 * that the byteorder bit is correct before being called. 742 */ 743 void 744 zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv) 745 { 746 uint64_t val64; 747 uint32_t val32; 748 749 ASSERT(BP_IS_ENCRYPTED(bp)); 750 751 if (!BP_SHOULD_BYTESWAP(bp)) { 752 bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t)); 753 bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t)); 754 bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); 755 BP_SET_IV2(bp, val32); 756 } else { 757 bcopy(salt, &val64, sizeof (uint64_t)); 758 bp->blk_dva[2].dva_word[0] = BSWAP_64(val64); 759 760 bcopy(iv, &val64, sizeof (uint64_t)); 761 bp->blk_dva[2].dva_word[1] = BSWAP_64(val64); 762 763 bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t)); 764 BP_SET_IV2(bp, BSWAP_32(val32)); 765 } 766 } 767 768 void 769 zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv) 770 { 771 uint64_t val64; 772 uint32_t val32; 773 774 ASSERT(BP_IS_PROTECTED(bp)); 775 776 /* for convenience, so callers don't need to check */ 777 if (BP_IS_AUTHENTICATED(bp)) { 778 bzero(salt, ZIO_DATA_SALT_LEN); 779 bzero(iv, ZIO_DATA_IV_LEN); 780 return; 781 } 782 783 if (!BP_SHOULD_BYTESWAP(bp)) { 784 bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t)); 785 bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t)); 786 787 val32 = (uint32_t)BP_GET_IV2(bp); 788 bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); 789 } else { 790 val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]); 791 bcopy(&val64, salt, sizeof (uint64_t)); 792 793 val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]); 794 bcopy(&val64, iv, sizeof (uint64_t)); 795 796 val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp)); 797 bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t)); 798 } 799 } 800 801 void 802 zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac) 803 { 804 uint64_t val64; 805 806 ASSERT(BP_USES_CRYPT(bp)); 807 ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET); 808 809 if (!BP_SHOULD_BYTESWAP(bp)) { 810 bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t)); 811 bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3], 812 sizeof (uint64_t)); 813 } else { 814 bcopy(mac, &val64, sizeof (uint64_t)); 815 bp->blk_cksum.zc_word[2] = BSWAP_64(val64); 816 817 bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t)); 818 bp->blk_cksum.zc_word[3] = BSWAP_64(val64); 819 } 820 } 821 822 void 823 zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac) 824 { 825 uint64_t val64; 826 827 ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp)); 828 829 /* for convenience, so callers don't need to check */ 830 if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 831 bzero(mac, ZIO_DATA_MAC_LEN); 832 return; 833 } 834 835 if (!BP_SHOULD_BYTESWAP(bp)) { 836 bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t)); 837 bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t), 838 sizeof (uint64_t)); 839 } else { 840 val64 = BSWAP_64(bp->blk_cksum.zc_word[2]); 841 bcopy(&val64, mac, sizeof (uint64_t)); 842 843 val64 = BSWAP_64(bp->blk_cksum.zc_word[3]); 844 bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t)); 845 } 846 } 847 848 void 849 zio_crypt_encode_mac_zil(void *data, uint8_t *mac) 850 { 851 zil_chain_t *zilc = data; 852 853 bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t)); 854 bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3], 855 sizeof (uint64_t)); 856 } 857 858 void 859 zio_crypt_decode_mac_zil(const void *data, uint8_t *mac) 860 { 861 /* 862 * The ZIL MAC is embedded in the block it protects, which will 863 * not have been byteswapped by the time this function has been called. 864 * As a result, we don't need to worry about byteswapping the MAC. 865 */ 866 const zil_chain_t *zilc = data; 867 868 bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t)); 869 bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t), 870 sizeof (uint64_t)); 871 } 872 873 /* 874 * This routine takes a block of dnodes (src_abd) and copies only the bonus 875 * buffers to the same offsets in the dst buffer. datalen should be the size 876 * of both the src_abd and the dst buffer (not just the length of the bonus 877 * buffers). 878 */ 879 void 880 zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen) 881 { 882 uint_t i, max_dnp = datalen >> DNODE_SHIFT; 883 uint8_t *src; 884 dnode_phys_t *dnp, *sdnp, *ddnp; 885 886 src = abd_borrow_buf_copy(src_abd, datalen); 887 888 sdnp = (dnode_phys_t *)src; 889 ddnp = (dnode_phys_t *)dst; 890 891 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 892 dnp = &sdnp[i]; 893 if (dnp->dn_type != DMU_OT_NONE && 894 DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && 895 dnp->dn_bonuslen != 0) { 896 bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), 897 DN_MAX_BONUS_LEN(dnp)); 898 } 899 } 900 901 abd_return_buf(src_abd, src, datalen); 902 } 903 904 /* 905 * This function decides what fields from blk_prop are included in 906 * the on-disk various MAC algorithms. 907 */ 908 static void 909 zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version) 910 { 911 /* 912 * Version 0 did not properly zero out all non-portable fields 913 * as it should have done. We maintain this code so that we can 914 * do read-only imports of pools on this version. 915 */ 916 if (version == 0) { 917 BP_SET_DEDUP(bp, 0); 918 BP_SET_CHECKSUM(bp, 0); 919 BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); 920 return; 921 } 922 923 ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION); 924 925 /* 926 * The hole_birth feature might set these fields even if this bp 927 * is a hole. We zero them out here to guarantee that raw sends 928 * will function with or without the feature. 929 */ 930 if (BP_IS_HOLE(bp)) { 931 bp->blk_prop = 0ULL; 932 return; 933 } 934 935 /* 936 * At L0 we want to verify these fields to ensure that data blocks 937 * can not be reinterpreted. For instance, we do not want an attacker 938 * to trick us into returning raw lz4 compressed data to the user 939 * by modifying the compression bits. At higher levels, we cannot 940 * enforce this policy since raw sends do not convey any information 941 * about indirect blocks, so these values might be different on the 942 * receive side. Fortunately, this does not open any new attack 943 * vectors, since any alterations that can be made to a higher level 944 * bp must still verify the correct order of the layer below it. 945 */ 946 if (BP_GET_LEVEL(bp) != 0) { 947 BP_SET_BYTEORDER(bp, 0); 948 BP_SET_COMPRESS(bp, 0); 949 950 /* 951 * psize cannot be set to zero or it will trigger 952 * asserts, but the value doesn't really matter as 953 * long as it is constant. 954 */ 955 BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE); 956 } 957 958 BP_SET_DEDUP(bp, 0); 959 BP_SET_CHECKSUM(bp, 0); 960 } 961 962 static void 963 zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp, 964 blkptr_auth_buf_t *bab, uint_t *bab_len) 965 { 966 blkptr_t tmpbp = *bp; 967 968 if (should_bswap) 969 byteswap_uint64_array(&tmpbp, sizeof (blkptr_t)); 970 971 ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp)); 972 ASSERT0(BP_IS_EMBEDDED(&tmpbp)); 973 974 zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac); 975 976 /* 977 * We always MAC blk_prop in LE to ensure portability. This 978 * must be done after decoding the mac, since the endianness 979 * will get zero'd out here. 980 */ 981 zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version); 982 bab->bab_prop = LE_64(tmpbp.blk_prop); 983 bab->bab_pad = 0ULL; 984 985 /* version 0 did not include the padding */ 986 *bab_len = sizeof (blkptr_auth_buf_t); 987 if (version == 0) 988 *bab_len -= sizeof (uint64_t); 989 } 990 991 static int 992 zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version, 993 boolean_t should_bswap, blkptr_t *bp) 994 { 995 int ret; 996 uint_t bab_len; 997 blkptr_auth_buf_t bab; 998 crypto_data_t cd; 999 1000 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 1001 cd.cd_format = CRYPTO_DATA_RAW; 1002 cd.cd_offset = 0; 1003 cd.cd_length = bab_len; 1004 cd.cd_raw.iov_base = (char *)&bab; 1005 cd.cd_raw.iov_len = cd.cd_length; 1006 1007 ret = crypto_mac_update(ctx, &cd, NULL); 1008 if (ret != CRYPTO_SUCCESS) { 1009 ret = SET_ERROR(EIO); 1010 goto error; 1011 } 1012 1013 return (0); 1014 1015 error: 1016 return (ret); 1017 } 1018 1019 static void 1020 zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version, 1021 boolean_t should_bswap, blkptr_t *bp) 1022 { 1023 uint_t bab_len; 1024 blkptr_auth_buf_t bab; 1025 1026 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 1027 SHA2Update(ctx, &bab, bab_len); 1028 } 1029 1030 static void 1031 zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version, 1032 boolean_t should_bswap, blkptr_t *bp) 1033 { 1034 uint_t bab_len; 1035 blkptr_auth_buf_t bab; 1036 1037 zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len); 1038 bcopy(&bab, *aadp, bab_len); 1039 *aadp += bab_len; 1040 *aad_len += bab_len; 1041 } 1042 1043 static int 1044 zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version, 1045 boolean_t should_bswap, dnode_phys_t *dnp) 1046 { 1047 int ret, i; 1048 dnode_phys_t *adnp; 1049 boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); 1050 crypto_data_t cd; 1051 uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)]; 1052 1053 cd.cd_format = CRYPTO_DATA_RAW; 1054 cd.cd_offset = 0; 1055 1056 /* authenticate the core dnode (masking out non-portable bits) */ 1057 bcopy(dnp, tmp_dncore, sizeof (tmp_dncore)); 1058 adnp = (dnode_phys_t *)tmp_dncore; 1059 if (le_bswap) { 1060 adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec); 1061 adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen); 1062 adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid); 1063 adnp->dn_used = BSWAP_64(adnp->dn_used); 1064 } 1065 adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; 1066 adnp->dn_used = 0; 1067 1068 cd.cd_length = sizeof (tmp_dncore); 1069 cd.cd_raw.iov_base = (char *)adnp; 1070 cd.cd_raw.iov_len = cd.cd_length; 1071 1072 ret = crypto_mac_update(ctx, &cd, NULL); 1073 if (ret != CRYPTO_SUCCESS) { 1074 ret = SET_ERROR(EIO); 1075 goto error; 1076 } 1077 1078 for (i = 0; i < dnp->dn_nblkptr; i++) { 1079 ret = zio_crypt_bp_do_hmac_updates(ctx, version, 1080 should_bswap, &dnp->dn_blkptr[i]); 1081 if (ret != 0) 1082 goto error; 1083 } 1084 1085 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1086 ret = zio_crypt_bp_do_hmac_updates(ctx, version, 1087 should_bswap, DN_SPILL_BLKPTR(dnp)); 1088 if (ret != 0) 1089 goto error; 1090 } 1091 1092 return (0); 1093 1094 error: 1095 return (ret); 1096 } 1097 1098 /* 1099 * objset_phys_t blocks introduce a number of exceptions to the normal 1100 * authentication process. objset_phys_t's contain 2 separate HMACS for 1101 * protecting the integrity of their data. The portable_mac protects the 1102 * metadnode. This MAC can be sent with a raw send and protects against 1103 * reordering of data within the metadnode. The local_mac protects the user 1104 * accounting objects which are not sent from one system to another. 1105 * 1106 * In addition, objset blocks are the only blocks that can be modified and 1107 * written to disk without the key loaded under certain circumstances. During 1108 * zil_claim() we need to be able to update the zil_header_t to complete 1109 * claiming log blocks and during raw receives we need to write out the 1110 * portable_mac from the send file. Both of these actions are possible 1111 * because these fields are not protected by either MAC so neither one will 1112 * need to modify the MACs without the key. However, when the modified blocks 1113 * are written out they will be byteswapped into the host machine's native 1114 * endianness which will modify fields protected by the MAC. As a result, MAC 1115 * calculation for objset blocks works slightly differently from other block 1116 * types. Where other block types MAC the data in whatever endianness is 1117 * written to disk, objset blocks always MAC little endian version of their 1118 * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP() 1119 * and le_bswap indicates whether a byteswap is needed to get this block 1120 * into little endian format. 1121 */ 1122 int 1123 zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen, 1124 boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac) 1125 { 1126 int ret; 1127 crypto_mechanism_t mech; 1128 crypto_context_t ctx; 1129 crypto_data_t cd; 1130 objset_phys_t *osp = data; 1131 uint64_t intval; 1132 boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER); 1133 uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH]; 1134 uint8_t raw_local_mac[SHA512_DIGEST_LENGTH]; 1135 1136 /* initialize HMAC mechanism */ 1137 mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC); 1138 mech.cm_param = NULL; 1139 mech.cm_param_len = 0; 1140 1141 cd.cd_format = CRYPTO_DATA_RAW; 1142 cd.cd_offset = 0; 1143 1144 /* calculate the portable MAC from the portable fields and metadnode */ 1145 ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); 1146 if (ret != CRYPTO_SUCCESS) { 1147 ret = SET_ERROR(EIO); 1148 goto error; 1149 } 1150 1151 /* add in the os_type */ 1152 intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type); 1153 cd.cd_length = sizeof (uint64_t); 1154 cd.cd_raw.iov_base = (char *)&intval; 1155 cd.cd_raw.iov_len = cd.cd_length; 1156 1157 ret = crypto_mac_update(ctx, &cd, NULL); 1158 if (ret != CRYPTO_SUCCESS) { 1159 ret = SET_ERROR(EIO); 1160 goto error; 1161 } 1162 1163 /* add in the portable os_flags */ 1164 intval = osp->os_flags; 1165 if (should_bswap) 1166 intval = BSWAP_64(intval); 1167 intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK; 1168 if (!ZFS_HOST_BYTEORDER) 1169 intval = BSWAP_64(intval); 1170 1171 cd.cd_length = sizeof (uint64_t); 1172 cd.cd_raw.iov_base = (char *)&intval; 1173 cd.cd_raw.iov_len = cd.cd_length; 1174 1175 ret = crypto_mac_update(ctx, &cd, NULL); 1176 if (ret != CRYPTO_SUCCESS) { 1177 ret = SET_ERROR(EIO); 1178 goto error; 1179 } 1180 1181 /* add in fields from the metadnode */ 1182 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1183 should_bswap, &osp->os_meta_dnode); 1184 if (ret) 1185 goto error; 1186 1187 /* store the final digest in a temporary buffer and copy what we need */ 1188 cd.cd_length = SHA512_DIGEST_LENGTH; 1189 cd.cd_raw.iov_base = (char *)raw_portable_mac; 1190 cd.cd_raw.iov_len = cd.cd_length; 1191 1192 ret = crypto_mac_final(ctx, &cd, NULL); 1193 if (ret != CRYPTO_SUCCESS) { 1194 ret = SET_ERROR(EIO); 1195 goto error; 1196 } 1197 1198 bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); 1199 1200 /* 1201 * The local MAC protects the user, group and project accounting. 1202 * If these objects are not present, the local MAC is zeroed out. 1203 */ 1204 if ((datalen >= OBJSET_PHYS_SIZE_V3 && 1205 osp->os_userused_dnode.dn_type == DMU_OT_NONE && 1206 osp->os_groupused_dnode.dn_type == DMU_OT_NONE && 1207 osp->os_projectused_dnode.dn_type == DMU_OT_NONE) || 1208 (datalen >= OBJSET_PHYS_SIZE_V2 && 1209 osp->os_userused_dnode.dn_type == DMU_OT_NONE && 1210 osp->os_groupused_dnode.dn_type == DMU_OT_NONE) || 1211 (datalen <= OBJSET_PHYS_SIZE_V1)) { 1212 bzero(local_mac, ZIO_OBJSET_MAC_LEN); 1213 return (0); 1214 } 1215 1216 /* calculate the local MAC from the userused and groupused dnodes */ 1217 ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL); 1218 if (ret != CRYPTO_SUCCESS) { 1219 ret = SET_ERROR(EIO); 1220 goto error; 1221 } 1222 1223 /* add in the non-portable os_flags */ 1224 intval = osp->os_flags; 1225 if (should_bswap) 1226 intval = BSWAP_64(intval); 1227 intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK; 1228 if (!ZFS_HOST_BYTEORDER) 1229 intval = BSWAP_64(intval); 1230 1231 cd.cd_length = sizeof (uint64_t); 1232 cd.cd_raw.iov_base = (char *)&intval; 1233 cd.cd_raw.iov_len = cd.cd_length; 1234 1235 ret = crypto_mac_update(ctx, &cd, NULL); 1236 if (ret != CRYPTO_SUCCESS) { 1237 ret = SET_ERROR(EIO); 1238 goto error; 1239 } 1240 1241 /* add in fields from the user accounting dnodes */ 1242 if (osp->os_userused_dnode.dn_type != DMU_OT_NONE) { 1243 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1244 should_bswap, &osp->os_userused_dnode); 1245 if (ret) 1246 goto error; 1247 } 1248 1249 if (osp->os_groupused_dnode.dn_type != DMU_OT_NONE) { 1250 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1251 should_bswap, &osp->os_groupused_dnode); 1252 if (ret) 1253 goto error; 1254 } 1255 1256 if (osp->os_projectused_dnode.dn_type != DMU_OT_NONE && 1257 datalen >= OBJSET_PHYS_SIZE_V3) { 1258 ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version, 1259 should_bswap, &osp->os_projectused_dnode); 1260 if (ret) 1261 goto error; 1262 } 1263 1264 /* store the final digest in a temporary buffer and copy what we need */ 1265 cd.cd_length = SHA512_DIGEST_LENGTH; 1266 cd.cd_raw.iov_base = (char *)raw_local_mac; 1267 cd.cd_raw.iov_len = cd.cd_length; 1268 1269 ret = crypto_mac_final(ctx, &cd, NULL); 1270 if (ret != CRYPTO_SUCCESS) { 1271 ret = SET_ERROR(EIO); 1272 goto error; 1273 } 1274 1275 bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); 1276 1277 return (0); 1278 1279 error: 1280 bzero(portable_mac, ZIO_OBJSET_MAC_LEN); 1281 bzero(local_mac, ZIO_OBJSET_MAC_LEN); 1282 return (ret); 1283 } 1284 1285 static void 1286 zio_crypt_destroy_uio(uio_t *uio) 1287 { 1288 if (uio->uio_iov) 1289 kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t)); 1290 } 1291 1292 /* 1293 * This function parses an uncompressed indirect block and returns a checksum 1294 * of all the portable fields from all of the contained bps. The portable 1295 * fields are the MAC and all of the fields from blk_prop except for the dedup, 1296 * checksum, and psize bits. For an explanation of the purpose of this, see 1297 * the comment block on object set authentication. 1298 */ 1299 static int 1300 zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf, 1301 uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum) 1302 { 1303 blkptr_t *bp; 1304 int i, epb = datalen >> SPA_BLKPTRSHIFT; 1305 SHA2_CTX ctx; 1306 uint8_t digestbuf[SHA512_DIGEST_LENGTH]; 1307 1308 /* checksum all of the MACs from the layer below */ 1309 SHA2Init(SHA512, &ctx); 1310 for (i = 0, bp = buf; i < epb; i++, bp++) { 1311 zio_crypt_bp_do_indrect_checksum_updates(&ctx, version, 1312 byteswap, bp); 1313 } 1314 SHA2Final(digestbuf, &ctx); 1315 1316 if (generate) { 1317 bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN); 1318 return (0); 1319 } 1320 1321 if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0) 1322 return (SET_ERROR(ECKSUM)); 1323 1324 return (0); 1325 } 1326 1327 int 1328 zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf, 1329 uint_t datalen, boolean_t byteswap, uint8_t *cksum) 1330 { 1331 int ret; 1332 1333 /* 1334 * Unfortunately, callers of this function will not always have 1335 * easy access to the on-disk format version. This info is 1336 * normally found in the DSL Crypto Key, but the checksum-of-MACs 1337 * is expected to be verifiable even when the key isn't loaded. 1338 * Here, instead of doing a ZAP lookup for the version for each 1339 * zio, we simply try both existing formats. 1340 */ 1341 ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf, 1342 datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum); 1343 if (ret == ECKSUM) { 1344 ASSERT(!generate); 1345 ret = zio_crypt_do_indirect_mac_checksum_impl(generate, 1346 buf, datalen, 0, byteswap, cksum); 1347 } 1348 1349 return (ret); 1350 } 1351 1352 int 1353 zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd, 1354 uint_t datalen, boolean_t byteswap, uint8_t *cksum) 1355 { 1356 int ret; 1357 void *buf; 1358 1359 buf = abd_borrow_buf_copy(abd, datalen); 1360 ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen, 1361 byteswap, cksum); 1362 abd_return_buf(abd, buf, datalen); 1363 1364 return (ret); 1365 } 1366 1367 /* 1368 * Special case handling routine for encrypting / decrypting ZIL blocks. 1369 * We do not check for the older ZIL chain because the encryption feature 1370 * was not available before the newer ZIL chain was introduced. The goal 1371 * here is to encrypt everything except the blkptr_t of a lr_write_t and 1372 * the zil_chain_t header. Everything that is not encrypted is authenticated. 1373 */ 1374 static int 1375 zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf, 1376 uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio, 1377 uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len, 1378 boolean_t *no_crypt) 1379 { 1380 int ret; 1381 uint64_t txtype, lr_len; 1382 uint_t nr_src, nr_dst, crypt_len; 1383 uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; 1384 iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; 1385 uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp; 1386 zil_chain_t *zilc; 1387 lr_t *lr; 1388 uint8_t *aadbuf = zio_buf_alloc(datalen); 1389 1390 /* cipherbuf always needs an extra iovec for the MAC */ 1391 if (encrypt) { 1392 src = plainbuf; 1393 dst = cipherbuf; 1394 nr_src = 0; 1395 nr_dst = 1; 1396 } else { 1397 src = cipherbuf; 1398 dst = plainbuf; 1399 nr_src = 1; 1400 nr_dst = 0; 1401 } 1402 1403 /* find the start and end record of the log block */ 1404 zilc = (zil_chain_t *)src; 1405 slrp = src + sizeof (zil_chain_t); 1406 aadp = aadbuf; 1407 blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused); 1408 1409 /* calculate the number of encrypted iovecs we will need */ 1410 for (; slrp < blkend; slrp += lr_len) { 1411 lr = (lr_t *)slrp; 1412 1413 if (!byteswap) { 1414 txtype = lr->lrc_txtype; 1415 lr_len = lr->lrc_reclen; 1416 } else { 1417 txtype = BSWAP_64(lr->lrc_txtype); 1418 lr_len = BSWAP_64(lr->lrc_reclen); 1419 } 1420 1421 nr_iovecs++; 1422 if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t)) 1423 nr_iovecs++; 1424 } 1425 1426 nr_src += nr_iovecs; 1427 nr_dst += nr_iovecs; 1428 1429 /* allocate the iovec arrays */ 1430 if (nr_src != 0) { 1431 src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); 1432 if (src_iovecs == NULL) { 1433 ret = SET_ERROR(ENOMEM); 1434 goto error; 1435 } 1436 } 1437 1438 if (nr_dst != 0) { 1439 dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); 1440 if (dst_iovecs == NULL) { 1441 ret = SET_ERROR(ENOMEM); 1442 goto error; 1443 } 1444 } 1445 1446 /* 1447 * Copy the plain zil header over and authenticate everything except 1448 * the checksum that will store our MAC. If we are writing the data 1449 * the embedded checksum will not have been calculated yet, so we don't 1450 * authenticate that. 1451 */ 1452 bcopy(src, dst, sizeof (zil_chain_t)); 1453 bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t)); 1454 aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t); 1455 aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t); 1456 1457 /* loop over records again, filling in iovecs */ 1458 nr_iovecs = 0; 1459 slrp = src + sizeof (zil_chain_t); 1460 dlrp = dst + sizeof (zil_chain_t); 1461 1462 for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) { 1463 lr = (lr_t *)slrp; 1464 1465 if (!byteswap) { 1466 txtype = lr->lrc_txtype; 1467 lr_len = lr->lrc_reclen; 1468 } else { 1469 txtype = BSWAP_64(lr->lrc_txtype); 1470 lr_len = BSWAP_64(lr->lrc_reclen); 1471 } 1472 1473 /* copy the common lr_t */ 1474 bcopy(slrp, dlrp, sizeof (lr_t)); 1475 bcopy(slrp, aadp, sizeof (lr_t)); 1476 aadp += sizeof (lr_t); 1477 aad_len += sizeof (lr_t); 1478 1479 ASSERT3P(src_iovecs, !=, NULL); 1480 ASSERT3P(dst_iovecs, !=, NULL); 1481 1482 /* 1483 * If this is a TX_WRITE record we want to encrypt everything 1484 * except the bp if exists. If the bp does exist we want to 1485 * authenticate it. 1486 */ 1487 if (txtype == TX_WRITE) { 1488 crypt_len = sizeof (lr_write_t) - 1489 sizeof (lr_t) - sizeof (blkptr_t); 1490 src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); 1491 src_iovecs[nr_iovecs].iov_len = crypt_len; 1492 dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); 1493 dst_iovecs[nr_iovecs].iov_len = crypt_len; 1494 1495 /* copy the bp now since it will not be encrypted */ 1496 bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1497 dlrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1498 sizeof (blkptr_t)); 1499 bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t), 1500 aadp, sizeof (blkptr_t)); 1501 aadp += sizeof (blkptr_t); 1502 aad_len += sizeof (blkptr_t); 1503 nr_iovecs++; 1504 total_len += crypt_len; 1505 1506 if (lr_len != sizeof (lr_write_t)) { 1507 crypt_len = lr_len - sizeof (lr_write_t); 1508 src_iovecs[nr_iovecs].iov_base = 1509 slrp + sizeof (lr_write_t); 1510 src_iovecs[nr_iovecs].iov_len = crypt_len; 1511 dst_iovecs[nr_iovecs].iov_base = 1512 dlrp + sizeof (lr_write_t); 1513 dst_iovecs[nr_iovecs].iov_len = crypt_len; 1514 nr_iovecs++; 1515 total_len += crypt_len; 1516 } 1517 } else { 1518 crypt_len = lr_len - sizeof (lr_t); 1519 src_iovecs[nr_iovecs].iov_base = slrp + sizeof (lr_t); 1520 src_iovecs[nr_iovecs].iov_len = crypt_len; 1521 dst_iovecs[nr_iovecs].iov_base = dlrp + sizeof (lr_t); 1522 dst_iovecs[nr_iovecs].iov_len = crypt_len; 1523 nr_iovecs++; 1524 total_len += crypt_len; 1525 } 1526 } 1527 1528 *no_crypt = (nr_iovecs == 0); 1529 *enc_len = total_len; 1530 *authbuf = aadbuf; 1531 *auth_len = aad_len; 1532 1533 if (encrypt) { 1534 puio->uio_iov = src_iovecs; 1535 puio->uio_iovcnt = nr_src; 1536 cuio->uio_iov = dst_iovecs; 1537 cuio->uio_iovcnt = nr_dst; 1538 } else { 1539 puio->uio_iov = dst_iovecs; 1540 puio->uio_iovcnt = nr_dst; 1541 cuio->uio_iov = src_iovecs; 1542 cuio->uio_iovcnt = nr_src; 1543 } 1544 1545 return (0); 1546 1547 error: 1548 zio_buf_free(aadbuf, datalen); 1549 if (src_iovecs != NULL) 1550 kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); 1551 if (dst_iovecs != NULL) 1552 kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); 1553 1554 *enc_len = 0; 1555 *authbuf = NULL; 1556 *auth_len = 0; 1557 *no_crypt = B_FALSE; 1558 puio->uio_iov = NULL; 1559 puio->uio_iovcnt = 0; 1560 cuio->uio_iov = NULL; 1561 cuio->uio_iovcnt = 0; 1562 return (ret); 1563 } 1564 1565 /* 1566 * Special case handling routine for encrypting / decrypting dnode blocks. 1567 */ 1568 static int 1569 zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version, 1570 uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, 1571 uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, 1572 uint_t *auth_len, boolean_t *no_crypt) 1573 { 1574 int ret; 1575 uint_t nr_src, nr_dst, crypt_len; 1576 uint_t aad_len = 0, nr_iovecs = 0, total_len = 0; 1577 uint_t i, j, max_dnp = datalen >> DNODE_SHIFT; 1578 iovec_t *src_iovecs = NULL, *dst_iovecs = NULL; 1579 uint8_t *src, *dst, *aadp; 1580 dnode_phys_t *dnp, *adnp, *sdnp, *ddnp; 1581 uint8_t *aadbuf = zio_buf_alloc(datalen); 1582 1583 if (encrypt) { 1584 src = plainbuf; 1585 dst = cipherbuf; 1586 nr_src = 0; 1587 nr_dst = 1; 1588 } else { 1589 src = cipherbuf; 1590 dst = plainbuf; 1591 nr_src = 1; 1592 nr_dst = 0; 1593 } 1594 1595 sdnp = (dnode_phys_t *)src; 1596 ddnp = (dnode_phys_t *)dst; 1597 aadp = aadbuf; 1598 1599 /* 1600 * Count the number of iovecs we will need to do the encryption by 1601 * counting the number of bonus buffers that need to be encrypted. 1602 */ 1603 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 1604 /* 1605 * This block may still be byteswapped. However, all of the 1606 * values we use are either uint8_t's (for which byteswapping 1607 * is a noop) or a * != 0 check, which will work regardless 1608 * of whether or not we byteswap. 1609 */ 1610 if (sdnp[i].dn_type != DMU_OT_NONE && 1611 DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) && 1612 sdnp[i].dn_bonuslen != 0) { 1613 nr_iovecs++; 1614 } 1615 } 1616 1617 nr_src += nr_iovecs; 1618 nr_dst += nr_iovecs; 1619 1620 if (nr_src != 0) { 1621 src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP); 1622 if (src_iovecs == NULL) { 1623 ret = SET_ERROR(ENOMEM); 1624 goto error; 1625 } 1626 } 1627 1628 if (nr_dst != 0) { 1629 dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP); 1630 if (dst_iovecs == NULL) { 1631 ret = SET_ERROR(ENOMEM); 1632 goto error; 1633 } 1634 } 1635 1636 nr_iovecs = 0; 1637 1638 /* 1639 * Iterate through the dnodes again, this time filling in the uios 1640 * we allocated earlier. We also concatenate any data we want to 1641 * authenticate onto aadbuf. 1642 */ 1643 for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) { 1644 dnp = &sdnp[i]; 1645 1646 /* copy over the core fields and blkptrs (kept as plaintext) */ 1647 bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp); 1648 1649 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1650 bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]), 1651 sizeof (blkptr_t)); 1652 } 1653 1654 /* 1655 * Handle authenticated data. We authenticate everything in 1656 * the dnode that can be brought over when we do a raw send. 1657 * This includes all of the core fields as well as the MACs 1658 * stored in the bp checksums and all of the portable bits 1659 * from blk_prop. We include the dnode padding here in case it 1660 * ever gets used in the future. Some dn_flags and dn_used are 1661 * not portable so we mask those out values out of the 1662 * authenticated data. 1663 */ 1664 crypt_len = offsetof(dnode_phys_t, dn_blkptr); 1665 bcopy(dnp, aadp, crypt_len); 1666 adnp = (dnode_phys_t *)aadp; 1667 adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK; 1668 adnp->dn_used = 0; 1669 aadp += crypt_len; 1670 aad_len += crypt_len; 1671 1672 for (j = 0; j < dnp->dn_nblkptr; j++) { 1673 zio_crypt_bp_do_aad_updates(&aadp, &aad_len, 1674 version, byteswap, &dnp->dn_blkptr[j]); 1675 } 1676 1677 if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1678 zio_crypt_bp_do_aad_updates(&aadp, &aad_len, 1679 version, byteswap, DN_SPILL_BLKPTR(dnp)); 1680 } 1681 1682 /* 1683 * If this bonus buffer needs to be encrypted, we prepare an 1684 * iovec_t. The encryption / decryption functions will fill 1685 * this in for us with the encrypted or decrypted data. 1686 * Otherwise we add the bonus buffer to the authenticated 1687 * data buffer and copy it over to the destination. The 1688 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that 1689 * we can guarantee alignment with the AES block size 1690 * (128 bits). 1691 */ 1692 crypt_len = DN_MAX_BONUS_LEN(dnp); 1693 if (dnp->dn_type != DMU_OT_NONE && 1694 DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) && 1695 dnp->dn_bonuslen != 0) { 1696 ASSERT3U(nr_iovecs, <, nr_src); 1697 ASSERT3U(nr_iovecs, <, nr_dst); 1698 ASSERT3P(src_iovecs, !=, NULL); 1699 ASSERT3P(dst_iovecs, !=, NULL); 1700 src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp); 1701 src_iovecs[nr_iovecs].iov_len = crypt_len; 1702 dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]); 1703 dst_iovecs[nr_iovecs].iov_len = crypt_len; 1704 1705 nr_iovecs++; 1706 total_len += crypt_len; 1707 } else { 1708 bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len); 1709 bcopy(DN_BONUS(dnp), aadp, crypt_len); 1710 aadp += crypt_len; 1711 aad_len += crypt_len; 1712 } 1713 } 1714 1715 *no_crypt = (nr_iovecs == 0); 1716 *enc_len = total_len; 1717 *authbuf = aadbuf; 1718 *auth_len = aad_len; 1719 1720 if (encrypt) { 1721 puio->uio_iov = src_iovecs; 1722 puio->uio_iovcnt = nr_src; 1723 cuio->uio_iov = dst_iovecs; 1724 cuio->uio_iovcnt = nr_dst; 1725 } else { 1726 puio->uio_iov = dst_iovecs; 1727 puio->uio_iovcnt = nr_dst; 1728 cuio->uio_iov = src_iovecs; 1729 cuio->uio_iovcnt = nr_src; 1730 } 1731 1732 return (0); 1733 1734 error: 1735 zio_buf_free(aadbuf, datalen); 1736 if (src_iovecs != NULL) 1737 kmem_free(src_iovecs, nr_src * sizeof (iovec_t)); 1738 if (dst_iovecs != NULL) 1739 kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t)); 1740 1741 *enc_len = 0; 1742 *authbuf = NULL; 1743 *auth_len = 0; 1744 *no_crypt = B_FALSE; 1745 puio->uio_iov = NULL; 1746 puio->uio_iovcnt = 0; 1747 cuio->uio_iov = NULL; 1748 cuio->uio_iovcnt = 0; 1749 return (ret); 1750 } 1751 1752 static int 1753 zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf, 1754 uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio, 1755 uint_t *enc_len) 1756 { 1757 int ret; 1758 uint_t nr_plain = 1, nr_cipher = 2; 1759 iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL; 1760 1761 /* allocate the iovecs for the plain and cipher data */ 1762 plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t), 1763 KM_SLEEP); 1764 if (!plain_iovecs) { 1765 ret = SET_ERROR(ENOMEM); 1766 goto error; 1767 } 1768 1769 cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t), 1770 KM_SLEEP); 1771 if (!cipher_iovecs) { 1772 ret = SET_ERROR(ENOMEM); 1773 goto error; 1774 } 1775 1776 plain_iovecs[0].iov_base = plainbuf; 1777 plain_iovecs[0].iov_len = datalen; 1778 cipher_iovecs[0].iov_base = cipherbuf; 1779 cipher_iovecs[0].iov_len = datalen; 1780 1781 *enc_len = datalen; 1782 puio->uio_iov = plain_iovecs; 1783 puio->uio_iovcnt = nr_plain; 1784 cuio->uio_iov = cipher_iovecs; 1785 cuio->uio_iovcnt = nr_cipher; 1786 1787 return (0); 1788 1789 error: 1790 if (plain_iovecs != NULL) 1791 kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t)); 1792 if (cipher_iovecs != NULL) 1793 kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t)); 1794 1795 *enc_len = 0; 1796 puio->uio_iov = NULL; 1797 puio->uio_iovcnt = 0; 1798 cuio->uio_iov = NULL; 1799 cuio->uio_iovcnt = 0; 1800 return (ret); 1801 } 1802 1803 /* 1804 * This function builds up the plaintext (puio) and ciphertext (cuio) uios so 1805 * that they can be used for encryption and decryption by zio_do_crypt_uio(). 1806 * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks 1807 * requiring special handling to parse out pieces that are to be encrypted. The 1808 * authbuf is used by these special cases to store additional authenticated 1809 * data (AAD) for the encryption modes. 1810 */ 1811 static int 1812 zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot, 1813 uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, 1814 uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, 1815 uint_t *auth_len, boolean_t *no_crypt) 1816 { 1817 int ret; 1818 iovec_t *mac_iov; 1819 1820 ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE); 1821 1822 /* route to handler */ 1823 switch (ot) { 1824 case DMU_OT_INTENT_LOG: 1825 ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf, 1826 datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len, 1827 no_crypt); 1828 break; 1829 case DMU_OT_DNODE: 1830 ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf, 1831 cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf, 1832 auth_len, no_crypt); 1833 break; 1834 default: 1835 ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf, 1836 datalen, puio, cuio, enc_len); 1837 *authbuf = NULL; 1838 *auth_len = 0; 1839 *no_crypt = B_FALSE; 1840 break; 1841 } 1842 1843 if (ret != 0) 1844 goto error; 1845 1846 /* populate the uios */ 1847 puio->uio_segflg = UIO_SYSSPACE; 1848 cuio->uio_segflg = UIO_SYSSPACE; 1849 1850 mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]); 1851 mac_iov->iov_base = mac; 1852 mac_iov->iov_len = ZIO_DATA_MAC_LEN; 1853 1854 return (0); 1855 1856 error: 1857 return (ret); 1858 } 1859 1860 /* 1861 * Primary encryption / decryption entrypoint for zio data. 1862 */ 1863 int 1864 zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key, 1865 dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv, 1866 uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf, 1867 boolean_t *no_crypt) 1868 { 1869 int ret; 1870 boolean_t locked = B_FALSE; 1871 uint64_t crypt = key->zk_crypt; 1872 uint_t keydata_len = zio_crypt_table[crypt].ci_keylen; 1873 uint_t enc_len, auth_len; 1874 uio_t puio, cuio; 1875 uint8_t enc_keydata[MASTER_KEY_MAX_LEN]; 1876 crypto_key_t tmp_ckey, *ckey = NULL; 1877 crypto_ctx_template_t tmpl; 1878 uint8_t *authbuf = NULL; 1879 1880 /* 1881 * If the needed key is the current one, just use it. Otherwise we 1882 * need to generate a temporary one from the given salt + master key. 1883 * If we are encrypting, we must return a copy of the current salt 1884 * so that it can be stored in the blkptr_t. 1885 */ 1886 rw_enter(&key->zk_salt_lock, RW_READER); 1887 locked = B_TRUE; 1888 1889 if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) { 1890 ckey = &key->zk_current_key; 1891 tmpl = key->zk_current_tmpl; 1892 } else { 1893 rw_exit(&key->zk_salt_lock); 1894 locked = B_FALSE; 1895 1896 ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0, 1897 salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len); 1898 if (ret != 0) 1899 goto error; 1900 1901 tmp_ckey.ck_format = CRYPTO_KEY_RAW; 1902 tmp_ckey.ck_data = enc_keydata; 1903 tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len); 1904 1905 ckey = &tmp_ckey; 1906 tmpl = NULL; 1907 } 1908 1909 /* 1910 * Attempt to use QAT acceleration if we can. We currently don't 1911 * do this for metadnode and ZIL blocks, since they have a much 1912 * more involved buffer layout and the qat_crypt() function only 1913 * works in-place. 1914 */ 1915 if (qat_crypt_use_accel(datalen) && 1916 ot != DMU_OT_INTENT_LOG && ot != DMU_OT_DNODE) { 1917 uint8_t *srcbuf, *dstbuf; 1918 1919 if (encrypt) { 1920 srcbuf = plainbuf; 1921 dstbuf = cipherbuf; 1922 } else { 1923 srcbuf = cipherbuf; 1924 dstbuf = plainbuf; 1925 } 1926 1927 ret = qat_crypt((encrypt) ? QAT_ENCRYPT : QAT_DECRYPT, srcbuf, 1928 dstbuf, NULL, 0, iv, mac, ckey, key->zk_crypt, datalen); 1929 if (ret == CPA_STATUS_SUCCESS) { 1930 if (locked) { 1931 rw_exit(&key->zk_salt_lock); 1932 locked = B_FALSE; 1933 } 1934 1935 return (0); 1936 } 1937 /* If the hardware implementation fails fall back to software */ 1938 } 1939 1940 bzero(&puio, sizeof (uio_t)); 1941 bzero(&cuio, sizeof (uio_t)); 1942 1943 /* create uios for encryption */ 1944 ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf, 1945 cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len, 1946 &authbuf, &auth_len, no_crypt); 1947 if (ret != 0) 1948 goto error; 1949 1950 /* perform the encryption / decryption in software */ 1951 ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len, 1952 &puio, &cuio, authbuf, auth_len); 1953 if (ret != 0) 1954 goto error; 1955 1956 if (locked) { 1957 rw_exit(&key->zk_salt_lock); 1958 locked = B_FALSE; 1959 } 1960 1961 if (authbuf != NULL) 1962 zio_buf_free(authbuf, datalen); 1963 if (ckey == &tmp_ckey) 1964 bzero(enc_keydata, keydata_len); 1965 zio_crypt_destroy_uio(&puio); 1966 zio_crypt_destroy_uio(&cuio); 1967 1968 return (0); 1969 1970 error: 1971 if (locked) 1972 rw_exit(&key->zk_salt_lock); 1973 if (authbuf != NULL) 1974 zio_buf_free(authbuf, datalen); 1975 if (ckey == &tmp_ckey) 1976 bzero(enc_keydata, keydata_len); 1977 zio_crypt_destroy_uio(&puio); 1978 zio_crypt_destroy_uio(&cuio); 1979 1980 return (ret); 1981 } 1982 1983 /* 1984 * Simple wrapper around zio_do_crypt_data() to work with abd's instead of 1985 * linear buffers. 1986 */ 1987 int 1988 zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot, 1989 boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac, 1990 uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt) 1991 { 1992 int ret; 1993 void *ptmp, *ctmp; 1994 1995 if (encrypt) { 1996 ptmp = abd_borrow_buf_copy(pabd, datalen); 1997 ctmp = abd_borrow_buf(cabd, datalen); 1998 } else { 1999 ptmp = abd_borrow_buf(pabd, datalen); 2000 ctmp = abd_borrow_buf_copy(cabd, datalen); 2001 } 2002 2003 ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac, 2004 datalen, ptmp, ctmp, no_crypt); 2005 if (ret != 0) 2006 goto error; 2007 2008 if (encrypt) { 2009 abd_return_buf(pabd, ptmp, datalen); 2010 abd_return_buf_copy(cabd, ctmp, datalen); 2011 } else { 2012 abd_return_buf_copy(pabd, ptmp, datalen); 2013 abd_return_buf(cabd, ctmp, datalen); 2014 } 2015 2016 return (0); 2017 2018 error: 2019 if (encrypt) { 2020 abd_return_buf(pabd, ptmp, datalen); 2021 abd_return_buf_copy(cabd, ctmp, datalen); 2022 } else { 2023 abd_return_buf_copy(pabd, ptmp, datalen); 2024 abd_return_buf(cabd, ctmp, datalen); 2025 } 2026 2027 return (ret); 2028 } 2029 2030 #if defined(_KERNEL) 2031 /* BEGIN CSTYLED */ 2032 module_param(zfs_key_max_salt_uses, ulong, 0644); 2033 MODULE_PARM_DESC(zfs_key_max_salt_uses, "Max number of times a salt value " 2034 "can be used for generating encryption keys before it is rotated"); 2035 /* END CSTYLED */ 2036 #endif 2037