1 /* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16 /* 17 * Copyright (c) 2017, Datto, Inc. All rights reserved. 18 * Copyright (c) 2018 by Delphix. All rights reserved. 19 */ 20 21 #include <sys/dsl_crypt.h> 22 #include <sys/dsl_pool.h> 23 #include <sys/zap.h> 24 #include <sys/zil.h> 25 #include <sys/dsl_dir.h> 26 #include <sys/dsl_prop.h> 27 #include <sys/spa_impl.h> 28 #include <sys/dmu_objset.h> 29 #include <sys/zvol.h> 30 31 /* 32 * This file's primary purpose is for managing master encryption keys in 33 * memory and on disk. For more info on how these keys are used, see the 34 * block comment in zio_crypt.c. 35 * 36 * All master keys are stored encrypted on disk in the form of the DSL 37 * Crypto Key ZAP object. The binary key data in this object is always 38 * randomly generated and is encrypted with the user's wrapping key. This 39 * layer of indirection allows the user to change their key without 40 * needing to re-encrypt the entire dataset. The ZAP also holds on to the 41 * (non-encrypted) encryption algorithm identifier, IV, and MAC needed to 42 * safely decrypt the master key. For more info on the user's key see the 43 * block comment in libzfs_crypto.c 44 * 45 * In-memory encryption keys are managed through the spa_keystore. The 46 * keystore consists of 3 AVL trees, which are as follows: 47 * 48 * The Wrapping Key Tree: 49 * The wrapping key (wkey) tree stores the user's keys that are fed into the 50 * kernel through 'zfs load-key' and related commands. Datasets inherit their 51 * parent's wkey by default, so these structures are refcounted. The wrapping 52 * keys remain in memory until they are explicitly unloaded (with 53 * "zfs unload-key"). Unloading is only possible when no datasets are using 54 * them (refcount=0). 55 * 56 * The DSL Crypto Key Tree: 57 * The DSL Crypto Keys (DCK) are the in-memory representation of decrypted 58 * master keys. They are used by the functions in zio_crypt.c to perform 59 * encryption, decryption, and authentication. Snapshots and clones of a given 60 * dataset will share a DSL Crypto Key, so they are also refcounted. Once the 61 * refcount on a key hits zero, it is immediately zeroed out and freed. 62 * 63 * The Crypto Key Mapping Tree: 64 * The zio layer needs to lookup master keys by their dataset object id. Since 65 * the DSL Crypto Keys can belong to multiple datasets, we maintain a tree of 66 * dsl_key_mapping_t's which essentially just map the dataset object id to its 67 * appropriate DSL Crypto Key. The management for creating and destroying these 68 * mappings hooks into the code for owning and disowning datasets. Usually, 69 * there will only be one active dataset owner, but there are times 70 * (particularly during dataset creation and destruction) when this may not be 71 * true or the dataset may not be initialized enough to own. As a result, this 72 * object is also refcounted. 73 */ 74 75 /* 76 * This tunable allows datasets to be raw received even if the stream does 77 * not include IVset guids or if the guids don't match. This is used as part 78 * of the resolution for ZPOOL_ERRATA_ZOL_8308_ENCRYPTION. 79 */ 80 int zfs_disable_ivset_guid_check = 0; 81 82 static void 83 dsl_wrapping_key_hold(dsl_wrapping_key_t *wkey, void *tag) 84 { 85 (void) zfs_refcount_add(&wkey->wk_refcnt, tag); 86 } 87 88 static void 89 dsl_wrapping_key_rele(dsl_wrapping_key_t *wkey, void *tag) 90 { 91 (void) zfs_refcount_remove(&wkey->wk_refcnt, tag); 92 } 93 94 static void 95 dsl_wrapping_key_free(dsl_wrapping_key_t *wkey) 96 { 97 ASSERT0(zfs_refcount_count(&wkey->wk_refcnt)); 98 99 if (wkey->wk_key.ck_data) { 100 memset(wkey->wk_key.ck_data, 0, 101 CRYPTO_BITS2BYTES(wkey->wk_key.ck_length)); 102 kmem_free(wkey->wk_key.ck_data, 103 CRYPTO_BITS2BYTES(wkey->wk_key.ck_length)); 104 } 105 106 zfs_refcount_destroy(&wkey->wk_refcnt); 107 kmem_free(wkey, sizeof (dsl_wrapping_key_t)); 108 } 109 110 static void 111 dsl_wrapping_key_create(uint8_t *wkeydata, zfs_keyformat_t keyformat, 112 uint64_t salt, uint64_t iters, dsl_wrapping_key_t **wkey_out) 113 { 114 dsl_wrapping_key_t *wkey; 115 116 /* allocate the wrapping key */ 117 wkey = kmem_alloc(sizeof (dsl_wrapping_key_t), KM_SLEEP); 118 119 /* allocate and initialize the underlying crypto key */ 120 wkey->wk_key.ck_data = kmem_alloc(WRAPPING_KEY_LEN, KM_SLEEP); 121 122 wkey->wk_key.ck_length = CRYPTO_BYTES2BITS(WRAPPING_KEY_LEN); 123 memcpy(wkey->wk_key.ck_data, wkeydata, WRAPPING_KEY_LEN); 124 125 /* initialize the rest of the struct */ 126 zfs_refcount_create(&wkey->wk_refcnt); 127 wkey->wk_keyformat = keyformat; 128 wkey->wk_salt = salt; 129 wkey->wk_iters = iters; 130 131 *wkey_out = wkey; 132 } 133 134 int 135 dsl_crypto_params_create_nvlist(dcp_cmd_t cmd, nvlist_t *props, 136 nvlist_t *crypto_args, dsl_crypto_params_t **dcp_out) 137 { 138 int ret; 139 uint64_t crypt = ZIO_CRYPT_INHERIT; 140 uint64_t keyformat = ZFS_KEYFORMAT_NONE; 141 uint64_t salt = 0, iters = 0; 142 dsl_crypto_params_t *dcp = NULL; 143 dsl_wrapping_key_t *wkey = NULL; 144 uint8_t *wkeydata = NULL; 145 uint_t wkeydata_len = 0; 146 char *keylocation = NULL; 147 148 dcp = kmem_zalloc(sizeof (dsl_crypto_params_t), KM_SLEEP); 149 dcp->cp_cmd = cmd; 150 151 /* get relevant arguments from the nvlists */ 152 if (props != NULL) { 153 (void) nvlist_lookup_uint64(props, 154 zfs_prop_to_name(ZFS_PROP_ENCRYPTION), &crypt); 155 (void) nvlist_lookup_uint64(props, 156 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), &keyformat); 157 (void) nvlist_lookup_string(props, 158 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), &keylocation); 159 (void) nvlist_lookup_uint64(props, 160 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), &salt); 161 (void) nvlist_lookup_uint64(props, 162 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), &iters); 163 164 dcp->cp_crypt = crypt; 165 } 166 167 if (crypto_args != NULL) { 168 (void) nvlist_lookup_uint8_array(crypto_args, "wkeydata", 169 &wkeydata, &wkeydata_len); 170 } 171 172 /* check for valid command */ 173 if (dcp->cp_cmd >= DCP_CMD_MAX) { 174 ret = SET_ERROR(EINVAL); 175 goto error; 176 } else { 177 dcp->cp_cmd = cmd; 178 } 179 180 /* check for valid crypt */ 181 if (dcp->cp_crypt >= ZIO_CRYPT_FUNCTIONS) { 182 ret = SET_ERROR(EINVAL); 183 goto error; 184 } else { 185 dcp->cp_crypt = crypt; 186 } 187 188 /* check for valid keyformat */ 189 if (keyformat >= ZFS_KEYFORMAT_FORMATS) { 190 ret = SET_ERROR(EINVAL); 191 goto error; 192 } 193 194 /* check for a valid keylocation (of any kind) and copy it in */ 195 if (keylocation != NULL) { 196 if (!zfs_prop_valid_keylocation(keylocation, B_FALSE)) { 197 ret = SET_ERROR(EINVAL); 198 goto error; 199 } 200 201 dcp->cp_keylocation = spa_strdup(keylocation); 202 } 203 204 /* check wrapping key length, if given */ 205 if (wkeydata != NULL && wkeydata_len != WRAPPING_KEY_LEN) { 206 ret = SET_ERROR(EINVAL); 207 goto error; 208 } 209 210 /* if the user asked for the default crypt, determine that now */ 211 if (dcp->cp_crypt == ZIO_CRYPT_ON) 212 dcp->cp_crypt = ZIO_CRYPT_ON_VALUE; 213 214 /* create the wrapping key from the raw data */ 215 if (wkeydata != NULL) { 216 /* create the wrapping key with the verified parameters */ 217 dsl_wrapping_key_create(wkeydata, keyformat, salt, 218 iters, &wkey); 219 dcp->cp_wkey = wkey; 220 } 221 222 /* 223 * Remove the encryption properties from the nvlist since they are not 224 * maintained through the DSL. 225 */ 226 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_ENCRYPTION)); 227 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_KEYFORMAT)); 228 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT)); 229 (void) nvlist_remove_all(props, 230 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS)); 231 232 *dcp_out = dcp; 233 234 return (0); 235 236 error: 237 kmem_free(dcp, sizeof (dsl_crypto_params_t)); 238 *dcp_out = NULL; 239 return (ret); 240 } 241 242 void 243 dsl_crypto_params_free(dsl_crypto_params_t *dcp, boolean_t unload) 244 { 245 if (dcp == NULL) 246 return; 247 248 if (dcp->cp_keylocation != NULL) 249 spa_strfree(dcp->cp_keylocation); 250 if (unload && dcp->cp_wkey != NULL) 251 dsl_wrapping_key_free(dcp->cp_wkey); 252 253 kmem_free(dcp, sizeof (dsl_crypto_params_t)); 254 } 255 256 static int 257 spa_crypto_key_compare(const void *a, const void *b) 258 { 259 const dsl_crypto_key_t *dcka = a; 260 const dsl_crypto_key_t *dckb = b; 261 262 if (dcka->dck_obj < dckb->dck_obj) 263 return (-1); 264 if (dcka->dck_obj > dckb->dck_obj) 265 return (1); 266 return (0); 267 } 268 269 static int 270 spa_key_mapping_compare(const void *a, const void *b) 271 { 272 const dsl_key_mapping_t *kma = a; 273 const dsl_key_mapping_t *kmb = b; 274 275 if (kma->km_dsobj < kmb->km_dsobj) 276 return (-1); 277 if (kma->km_dsobj > kmb->km_dsobj) 278 return (1); 279 return (0); 280 } 281 282 static int 283 spa_wkey_compare(const void *a, const void *b) 284 { 285 const dsl_wrapping_key_t *wka = a; 286 const dsl_wrapping_key_t *wkb = b; 287 288 if (wka->wk_ddobj < wkb->wk_ddobj) 289 return (-1); 290 if (wka->wk_ddobj > wkb->wk_ddobj) 291 return (1); 292 return (0); 293 } 294 295 void 296 spa_keystore_init(spa_keystore_t *sk) 297 { 298 rw_init(&sk->sk_dk_lock, NULL, RW_DEFAULT, NULL); 299 rw_init(&sk->sk_km_lock, NULL, RW_DEFAULT, NULL); 300 rw_init(&sk->sk_wkeys_lock, NULL, RW_DEFAULT, NULL); 301 avl_create(&sk->sk_dsl_keys, spa_crypto_key_compare, 302 sizeof (dsl_crypto_key_t), 303 offsetof(dsl_crypto_key_t, dck_avl_link)); 304 avl_create(&sk->sk_key_mappings, spa_key_mapping_compare, 305 sizeof (dsl_key_mapping_t), 306 offsetof(dsl_key_mapping_t, km_avl_link)); 307 avl_create(&sk->sk_wkeys, spa_wkey_compare, sizeof (dsl_wrapping_key_t), 308 offsetof(dsl_wrapping_key_t, wk_avl_link)); 309 } 310 311 void 312 spa_keystore_fini(spa_keystore_t *sk) 313 { 314 dsl_wrapping_key_t *wkey; 315 void *cookie = NULL; 316 317 ASSERT(avl_is_empty(&sk->sk_dsl_keys)); 318 ASSERT(avl_is_empty(&sk->sk_key_mappings)); 319 320 while ((wkey = avl_destroy_nodes(&sk->sk_wkeys, &cookie)) != NULL) 321 dsl_wrapping_key_free(wkey); 322 323 avl_destroy(&sk->sk_wkeys); 324 avl_destroy(&sk->sk_key_mappings); 325 avl_destroy(&sk->sk_dsl_keys); 326 rw_destroy(&sk->sk_wkeys_lock); 327 rw_destroy(&sk->sk_km_lock); 328 rw_destroy(&sk->sk_dk_lock); 329 } 330 331 static int 332 dsl_dir_get_encryption_root_ddobj(dsl_dir_t *dd, uint64_t *rddobj) 333 { 334 if (dd->dd_crypto_obj == 0) 335 return (SET_ERROR(ENOENT)); 336 337 return (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 338 DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, rddobj)); 339 } 340 341 static int 342 dsl_dir_get_encryption_version(dsl_dir_t *dd, uint64_t *version) 343 { 344 *version = 0; 345 346 if (dd->dd_crypto_obj == 0) 347 return (SET_ERROR(ENOENT)); 348 349 /* version 0 is implied by ENOENT */ 350 (void) zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 351 DSL_CRYPTO_KEY_VERSION, 8, 1, version); 352 353 return (0); 354 } 355 356 boolean_t 357 dsl_dir_incompatible_encryption_version(dsl_dir_t *dd) 358 { 359 int ret; 360 uint64_t version = 0; 361 362 ret = dsl_dir_get_encryption_version(dd, &version); 363 if (ret != 0) 364 return (B_FALSE); 365 366 return (version != ZIO_CRYPT_KEY_CURRENT_VERSION); 367 } 368 369 static int 370 spa_keystore_wkey_hold_ddobj_impl(spa_t *spa, uint64_t ddobj, 371 void *tag, dsl_wrapping_key_t **wkey_out) 372 { 373 int ret; 374 dsl_wrapping_key_t search_wkey; 375 dsl_wrapping_key_t *found_wkey; 376 377 ASSERT(RW_LOCK_HELD(&spa->spa_keystore.sk_wkeys_lock)); 378 379 /* init the search wrapping key */ 380 search_wkey.wk_ddobj = ddobj; 381 382 /* lookup the wrapping key */ 383 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, &search_wkey, NULL); 384 if (!found_wkey) { 385 ret = SET_ERROR(ENOENT); 386 goto error; 387 } 388 389 /* increment the refcount */ 390 dsl_wrapping_key_hold(found_wkey, tag); 391 392 *wkey_out = found_wkey; 393 return (0); 394 395 error: 396 *wkey_out = NULL; 397 return (ret); 398 } 399 400 static int 401 spa_keystore_wkey_hold_dd(spa_t *spa, dsl_dir_t *dd, void *tag, 402 dsl_wrapping_key_t **wkey_out) 403 { 404 int ret; 405 dsl_wrapping_key_t *wkey; 406 uint64_t rddobj; 407 boolean_t locked = B_FALSE; 408 409 if (!RW_WRITE_HELD(&spa->spa_keystore.sk_wkeys_lock)) { 410 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_READER); 411 locked = B_TRUE; 412 } 413 414 /* get the ddobj that the keylocation property was inherited from */ 415 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 416 if (ret != 0) 417 goto error; 418 419 /* lookup the wkey in the avl tree */ 420 ret = spa_keystore_wkey_hold_ddobj_impl(spa, rddobj, tag, &wkey); 421 if (ret != 0) 422 goto error; 423 424 /* unlock the wkey tree if we locked it */ 425 if (locked) 426 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 427 428 *wkey_out = wkey; 429 return (0); 430 431 error: 432 if (locked) 433 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 434 435 *wkey_out = NULL; 436 return (ret); 437 } 438 439 int 440 dsl_crypto_can_set_keylocation(const char *dsname, const char *keylocation) 441 { 442 int ret = 0; 443 dsl_dir_t *dd = NULL; 444 dsl_pool_t *dp = NULL; 445 uint64_t rddobj; 446 447 /* hold the dsl dir */ 448 ret = dsl_pool_hold(dsname, FTAG, &dp); 449 if (ret != 0) 450 goto out; 451 452 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 453 if (ret != 0) { 454 dd = NULL; 455 goto out; 456 } 457 458 /* if dd is not encrypted, the value may only be "none" */ 459 if (dd->dd_crypto_obj == 0) { 460 if (strcmp(keylocation, "none") != 0) { 461 ret = SET_ERROR(EACCES); 462 goto out; 463 } 464 465 ret = 0; 466 goto out; 467 } 468 469 /* check for a valid keylocation for encrypted datasets */ 470 if (!zfs_prop_valid_keylocation(keylocation, B_TRUE)) { 471 ret = SET_ERROR(EINVAL); 472 goto out; 473 } 474 475 /* check that this is an encryption root */ 476 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 477 if (ret != 0) 478 goto out; 479 480 if (rddobj != dd->dd_object) { 481 ret = SET_ERROR(EACCES); 482 goto out; 483 } 484 485 dsl_dir_rele(dd, FTAG); 486 dsl_pool_rele(dp, FTAG); 487 488 return (0); 489 490 out: 491 if (dd != NULL) 492 dsl_dir_rele(dd, FTAG); 493 if (dp != NULL) 494 dsl_pool_rele(dp, FTAG); 495 496 return (ret); 497 } 498 499 static void 500 dsl_crypto_key_free(dsl_crypto_key_t *dck) 501 { 502 ASSERT(zfs_refcount_count(&dck->dck_holds) == 0); 503 504 /* destroy the zio_crypt_key_t */ 505 zio_crypt_key_destroy(&dck->dck_key); 506 507 /* free the refcount, wrapping key, and lock */ 508 zfs_refcount_destroy(&dck->dck_holds); 509 if (dck->dck_wkey) 510 dsl_wrapping_key_rele(dck->dck_wkey, dck); 511 512 /* free the key */ 513 kmem_free(dck, sizeof (dsl_crypto_key_t)); 514 } 515 516 static void 517 dsl_crypto_key_rele(dsl_crypto_key_t *dck, void *tag) 518 { 519 if (zfs_refcount_remove(&dck->dck_holds, tag) == 0) 520 dsl_crypto_key_free(dck); 521 } 522 523 static int 524 dsl_crypto_key_open(objset_t *mos, dsl_wrapping_key_t *wkey, 525 uint64_t dckobj, void *tag, dsl_crypto_key_t **dck_out) 526 { 527 int ret; 528 uint64_t crypt = 0, guid = 0, version = 0; 529 uint8_t raw_keydata[MASTER_KEY_MAX_LEN]; 530 uint8_t raw_hmac_keydata[SHA512_HMAC_KEYLEN]; 531 uint8_t iv[WRAPPING_IV_LEN]; 532 uint8_t mac[WRAPPING_MAC_LEN]; 533 dsl_crypto_key_t *dck; 534 535 /* allocate and initialize the key */ 536 dck = kmem_zalloc(sizeof (dsl_crypto_key_t), KM_SLEEP); 537 538 /* fetch all of the values we need from the ZAP */ 539 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 540 &crypt); 541 if (ret != 0) 542 goto error; 543 544 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &guid); 545 if (ret != 0) 546 goto error; 547 548 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 549 MASTER_KEY_MAX_LEN, raw_keydata); 550 if (ret != 0) 551 goto error; 552 553 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 554 SHA512_HMAC_KEYLEN, raw_hmac_keydata); 555 if (ret != 0) 556 goto error; 557 558 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 559 iv); 560 if (ret != 0) 561 goto error; 562 563 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 564 mac); 565 if (ret != 0) 566 goto error; 567 568 /* the initial on-disk format for encryption did not have a version */ 569 (void) zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_VERSION, 8, 1, &version); 570 571 /* 572 * Unwrap the keys. If there is an error return EACCES to indicate 573 * an authentication failure. 574 */ 575 ret = zio_crypt_key_unwrap(&wkey->wk_key, crypt, version, guid, 576 raw_keydata, raw_hmac_keydata, iv, mac, &dck->dck_key); 577 if (ret != 0) { 578 ret = SET_ERROR(EACCES); 579 goto error; 580 } 581 582 /* finish initializing the dsl_crypto_key_t */ 583 zfs_refcount_create(&dck->dck_holds); 584 dsl_wrapping_key_hold(wkey, dck); 585 dck->dck_wkey = wkey; 586 dck->dck_obj = dckobj; 587 zfs_refcount_add(&dck->dck_holds, tag); 588 589 *dck_out = dck; 590 return (0); 591 592 error: 593 if (dck != NULL) { 594 memset(dck, 0, sizeof (dsl_crypto_key_t)); 595 kmem_free(dck, sizeof (dsl_crypto_key_t)); 596 } 597 598 *dck_out = NULL; 599 return (ret); 600 } 601 602 static int 603 spa_keystore_dsl_key_hold_impl(spa_t *spa, uint64_t dckobj, void *tag, 604 dsl_crypto_key_t **dck_out) 605 { 606 int ret; 607 dsl_crypto_key_t search_dck; 608 dsl_crypto_key_t *found_dck; 609 610 ASSERT(RW_LOCK_HELD(&spa->spa_keystore.sk_dk_lock)); 611 612 /* init the search key */ 613 search_dck.dck_obj = dckobj; 614 615 /* find the matching key in the keystore */ 616 found_dck = avl_find(&spa->spa_keystore.sk_dsl_keys, &search_dck, NULL); 617 if (!found_dck) { 618 ret = SET_ERROR(ENOENT); 619 goto error; 620 } 621 622 /* increment the refcount */ 623 zfs_refcount_add(&found_dck->dck_holds, tag); 624 625 *dck_out = found_dck; 626 return (0); 627 628 error: 629 *dck_out = NULL; 630 return (ret); 631 } 632 633 static int 634 spa_keystore_dsl_key_hold_dd(spa_t *spa, dsl_dir_t *dd, void *tag, 635 dsl_crypto_key_t **dck_out) 636 { 637 int ret; 638 avl_index_t where; 639 dsl_crypto_key_t *dck_io = NULL, *dck_ks = NULL; 640 dsl_wrapping_key_t *wkey = NULL; 641 uint64_t dckobj = dd->dd_crypto_obj; 642 643 /* Lookup the key in the tree of currently loaded keys */ 644 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_READER); 645 ret = spa_keystore_dsl_key_hold_impl(spa, dckobj, tag, &dck_ks); 646 rw_exit(&spa->spa_keystore.sk_dk_lock); 647 if (ret == 0) { 648 *dck_out = dck_ks; 649 return (0); 650 } 651 652 /* Lookup the wrapping key from the keystore */ 653 ret = spa_keystore_wkey_hold_dd(spa, dd, FTAG, &wkey); 654 if (ret != 0) { 655 *dck_out = NULL; 656 return (SET_ERROR(EACCES)); 657 } 658 659 /* Read the key from disk */ 660 ret = dsl_crypto_key_open(spa->spa_meta_objset, wkey, dckobj, 661 tag, &dck_io); 662 if (ret != 0) { 663 dsl_wrapping_key_rele(wkey, FTAG); 664 *dck_out = NULL; 665 return (ret); 666 } 667 668 /* 669 * Add the key to the keystore. It may already exist if it was 670 * added while performing the read from disk. In this case discard 671 * it and return the key from the keystore. 672 */ 673 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_WRITER); 674 ret = spa_keystore_dsl_key_hold_impl(spa, dckobj, tag, &dck_ks); 675 if (ret != 0) { 676 avl_find(&spa->spa_keystore.sk_dsl_keys, dck_io, &where); 677 avl_insert(&spa->spa_keystore.sk_dsl_keys, dck_io, where); 678 *dck_out = dck_io; 679 } else { 680 dsl_crypto_key_free(dck_io); 681 *dck_out = dck_ks; 682 } 683 684 /* Release the wrapping key (the dsl key now has a reference to it) */ 685 dsl_wrapping_key_rele(wkey, FTAG); 686 rw_exit(&spa->spa_keystore.sk_dk_lock); 687 688 return (0); 689 } 690 691 void 692 spa_keystore_dsl_key_rele(spa_t *spa, dsl_crypto_key_t *dck, void *tag) 693 { 694 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_WRITER); 695 696 if (zfs_refcount_remove(&dck->dck_holds, tag) == 0) { 697 avl_remove(&spa->spa_keystore.sk_dsl_keys, dck); 698 dsl_crypto_key_free(dck); 699 } 700 701 rw_exit(&spa->spa_keystore.sk_dk_lock); 702 } 703 704 int 705 spa_keystore_load_wkey_impl(spa_t *spa, dsl_wrapping_key_t *wkey) 706 { 707 int ret; 708 avl_index_t where; 709 dsl_wrapping_key_t *found_wkey; 710 711 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 712 713 /* insert the wrapping key into the keystore */ 714 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, wkey, &where); 715 if (found_wkey != NULL) { 716 ret = SET_ERROR(EEXIST); 717 goto error_unlock; 718 } 719 avl_insert(&spa->spa_keystore.sk_wkeys, wkey, where); 720 721 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 722 723 return (0); 724 725 error_unlock: 726 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 727 return (ret); 728 } 729 730 int 731 spa_keystore_load_wkey(const char *dsname, dsl_crypto_params_t *dcp, 732 boolean_t noop) 733 { 734 int ret; 735 dsl_dir_t *dd = NULL; 736 dsl_crypto_key_t *dck = NULL; 737 dsl_wrapping_key_t *wkey = dcp->cp_wkey; 738 dsl_pool_t *dp = NULL; 739 uint64_t rddobj, keyformat, salt, iters; 740 741 /* 742 * We don't validate the wrapping key's keyformat, salt, or iters 743 * since they will never be needed after the DCK has been wrapped. 744 */ 745 if (dcp->cp_wkey == NULL || 746 dcp->cp_cmd != DCP_CMD_NONE || 747 dcp->cp_crypt != ZIO_CRYPT_INHERIT || 748 dcp->cp_keylocation != NULL) 749 return (SET_ERROR(EINVAL)); 750 751 ret = dsl_pool_hold(dsname, FTAG, &dp); 752 if (ret != 0) 753 goto error; 754 755 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 756 ret = SET_ERROR(ENOTSUP); 757 goto error; 758 } 759 760 /* hold the dsl dir */ 761 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 762 if (ret != 0) { 763 dd = NULL; 764 goto error; 765 } 766 767 /* confirm that dd is the encryption root */ 768 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 769 if (ret != 0 || rddobj != dd->dd_object) { 770 ret = SET_ERROR(EINVAL); 771 goto error; 772 } 773 774 /* initialize the wkey's ddobj */ 775 wkey->wk_ddobj = dd->dd_object; 776 777 /* verify that the wkey is correct by opening its dsl key */ 778 ret = dsl_crypto_key_open(dp->dp_meta_objset, wkey, 779 dd->dd_crypto_obj, FTAG, &dck); 780 if (ret != 0) 781 goto error; 782 783 /* initialize the wkey encryption parameters from the DSL Crypto Key */ 784 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 785 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &keyformat); 786 if (ret != 0) 787 goto error; 788 789 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 790 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt); 791 if (ret != 0) 792 goto error; 793 794 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 795 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters); 796 if (ret != 0) 797 goto error; 798 799 ASSERT3U(keyformat, <, ZFS_KEYFORMAT_FORMATS); 800 ASSERT3U(keyformat, !=, ZFS_KEYFORMAT_NONE); 801 IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, iters != 0); 802 IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, salt != 0); 803 IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, iters == 0); 804 IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, salt == 0); 805 806 wkey->wk_keyformat = keyformat; 807 wkey->wk_salt = salt; 808 wkey->wk_iters = iters; 809 810 /* 811 * At this point we have verified the wkey and confirmed that it can 812 * be used to decrypt a DSL Crypto Key. We can simply cleanup and 813 * return if this is all the user wanted to do. 814 */ 815 if (noop) 816 goto error; 817 818 /* insert the wrapping key into the keystore */ 819 ret = spa_keystore_load_wkey_impl(dp->dp_spa, wkey); 820 if (ret != 0) 821 goto error; 822 823 dsl_crypto_key_rele(dck, FTAG); 824 dsl_dir_rele(dd, FTAG); 825 dsl_pool_rele(dp, FTAG); 826 827 /* create any zvols under this ds */ 828 zvol_create_minors_recursive(dsname); 829 830 return (0); 831 832 error: 833 if (dck != NULL) 834 dsl_crypto_key_rele(dck, FTAG); 835 if (dd != NULL) 836 dsl_dir_rele(dd, FTAG); 837 if (dp != NULL) 838 dsl_pool_rele(dp, FTAG); 839 840 return (ret); 841 } 842 843 int 844 spa_keystore_unload_wkey_impl(spa_t *spa, uint64_t ddobj) 845 { 846 int ret; 847 dsl_wrapping_key_t search_wkey; 848 dsl_wrapping_key_t *found_wkey; 849 850 /* init the search wrapping key */ 851 search_wkey.wk_ddobj = ddobj; 852 853 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 854 855 /* remove the wrapping key from the keystore */ 856 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, 857 &search_wkey, NULL); 858 if (!found_wkey) { 859 ret = SET_ERROR(EACCES); 860 goto error_unlock; 861 } else if (zfs_refcount_count(&found_wkey->wk_refcnt) != 0) { 862 ret = SET_ERROR(EBUSY); 863 goto error_unlock; 864 } 865 avl_remove(&spa->spa_keystore.sk_wkeys, found_wkey); 866 867 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 868 869 /* free the wrapping key */ 870 dsl_wrapping_key_free(found_wkey); 871 872 return (0); 873 874 error_unlock: 875 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 876 return (ret); 877 } 878 879 int 880 spa_keystore_unload_wkey(const char *dsname) 881 { 882 int ret = 0; 883 dsl_dir_t *dd = NULL; 884 dsl_pool_t *dp = NULL; 885 spa_t *spa = NULL; 886 887 ret = spa_open(dsname, &spa, FTAG); 888 if (ret != 0) 889 return (ret); 890 891 /* 892 * Wait for any outstanding txg IO to complete, releasing any 893 * remaining references on the wkey. 894 */ 895 if (spa_mode(spa) != SPA_MODE_READ) 896 txg_wait_synced(spa->spa_dsl_pool, 0); 897 898 spa_close(spa, FTAG); 899 900 /* hold the dsl dir */ 901 ret = dsl_pool_hold(dsname, FTAG, &dp); 902 if (ret != 0) 903 goto error; 904 905 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 906 ret = (SET_ERROR(ENOTSUP)); 907 goto error; 908 } 909 910 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 911 if (ret != 0) { 912 dd = NULL; 913 goto error; 914 } 915 916 /* unload the wkey */ 917 ret = spa_keystore_unload_wkey_impl(dp->dp_spa, dd->dd_object); 918 if (ret != 0) 919 goto error; 920 921 dsl_dir_rele(dd, FTAG); 922 dsl_pool_rele(dp, FTAG); 923 924 /* remove any zvols under this ds */ 925 zvol_remove_minors(dp->dp_spa, dsname, B_TRUE); 926 927 return (0); 928 929 error: 930 if (dd != NULL) 931 dsl_dir_rele(dd, FTAG); 932 if (dp != NULL) 933 dsl_pool_rele(dp, FTAG); 934 935 return (ret); 936 } 937 938 void 939 key_mapping_add_ref(dsl_key_mapping_t *km, void *tag) 940 { 941 ASSERT3U(zfs_refcount_count(&km->km_refcnt), >=, 1); 942 zfs_refcount_add(&km->km_refcnt, tag); 943 } 944 945 /* 946 * The locking here is a little tricky to ensure we don't cause unnecessary 947 * performance problems. We want to release a key mapping whenever someone 948 * decrements the refcount to 0, but freeing the mapping requires removing 949 * it from the spa_keystore, which requires holding sk_km_lock as a writer. 950 * Most of the time we don't want to hold this lock as a writer, since the 951 * same lock is held as a reader for each IO that needs to encrypt / decrypt 952 * data for any dataset and in practice we will only actually free the 953 * mapping after unmounting a dataset. 954 */ 955 void 956 key_mapping_rele(spa_t *spa, dsl_key_mapping_t *km, void *tag) 957 { 958 ASSERT3U(zfs_refcount_count(&km->km_refcnt), >=, 1); 959 960 if (zfs_refcount_remove(&km->km_refcnt, tag) != 0) 961 return; 962 963 /* 964 * We think we are going to need to free the mapping. Add a 965 * reference to prevent most other releasers from thinking 966 * this might be their responsibility. This is inherently 967 * racy, so we will confirm that we are legitimately the 968 * last holder once we have the sk_km_lock as a writer. 969 */ 970 zfs_refcount_add(&km->km_refcnt, FTAG); 971 972 rw_enter(&spa->spa_keystore.sk_km_lock, RW_WRITER); 973 if (zfs_refcount_remove(&km->km_refcnt, FTAG) != 0) { 974 rw_exit(&spa->spa_keystore.sk_km_lock); 975 return; 976 } 977 978 avl_remove(&spa->spa_keystore.sk_key_mappings, km); 979 rw_exit(&spa->spa_keystore.sk_km_lock); 980 981 spa_keystore_dsl_key_rele(spa, km->km_key, km); 982 zfs_refcount_destroy(&km->km_refcnt); 983 kmem_free(km, sizeof (dsl_key_mapping_t)); 984 } 985 986 int 987 spa_keystore_create_mapping(spa_t *spa, dsl_dataset_t *ds, void *tag, 988 dsl_key_mapping_t **km_out) 989 { 990 int ret; 991 avl_index_t where; 992 dsl_key_mapping_t *km, *found_km; 993 boolean_t should_free = B_FALSE; 994 995 /* Allocate and initialize the mapping */ 996 km = kmem_zalloc(sizeof (dsl_key_mapping_t), KM_SLEEP); 997 zfs_refcount_create(&km->km_refcnt); 998 999 ret = spa_keystore_dsl_key_hold_dd(spa, ds->ds_dir, km, &km->km_key); 1000 if (ret != 0) { 1001 zfs_refcount_destroy(&km->km_refcnt); 1002 kmem_free(km, sizeof (dsl_key_mapping_t)); 1003 1004 if (km_out != NULL) 1005 *km_out = NULL; 1006 return (ret); 1007 } 1008 1009 km->km_dsobj = ds->ds_object; 1010 1011 rw_enter(&spa->spa_keystore.sk_km_lock, RW_WRITER); 1012 1013 /* 1014 * If a mapping already exists, simply increment its refcount and 1015 * cleanup the one we made. We want to allocate / free outside of 1016 * the lock because this lock is also used by the zio layer to lookup 1017 * key mappings. Otherwise, use the one we created. Normally, there will 1018 * only be one active reference at a time (the objset owner), but there 1019 * are times when there could be multiple async users. 1020 */ 1021 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, km, &where); 1022 if (found_km != NULL) { 1023 should_free = B_TRUE; 1024 zfs_refcount_add(&found_km->km_refcnt, tag); 1025 if (km_out != NULL) 1026 *km_out = found_km; 1027 } else { 1028 zfs_refcount_add(&km->km_refcnt, tag); 1029 avl_insert(&spa->spa_keystore.sk_key_mappings, km, where); 1030 if (km_out != NULL) 1031 *km_out = km; 1032 } 1033 1034 rw_exit(&spa->spa_keystore.sk_km_lock); 1035 1036 if (should_free) { 1037 spa_keystore_dsl_key_rele(spa, km->km_key, km); 1038 zfs_refcount_destroy(&km->km_refcnt); 1039 kmem_free(km, sizeof (dsl_key_mapping_t)); 1040 } 1041 1042 return (0); 1043 } 1044 1045 int 1046 spa_keystore_remove_mapping(spa_t *spa, uint64_t dsobj, void *tag) 1047 { 1048 int ret; 1049 dsl_key_mapping_t search_km; 1050 dsl_key_mapping_t *found_km; 1051 1052 /* init the search key mapping */ 1053 search_km.km_dsobj = dsobj; 1054 1055 rw_enter(&spa->spa_keystore.sk_km_lock, RW_READER); 1056 1057 /* find the matching mapping */ 1058 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, 1059 &search_km, NULL); 1060 if (found_km == NULL) { 1061 ret = SET_ERROR(ENOENT); 1062 goto error_unlock; 1063 } 1064 1065 rw_exit(&spa->spa_keystore.sk_km_lock); 1066 1067 key_mapping_rele(spa, found_km, tag); 1068 1069 return (0); 1070 1071 error_unlock: 1072 rw_exit(&spa->spa_keystore.sk_km_lock); 1073 return (ret); 1074 } 1075 1076 /* 1077 * This function is primarily used by the zio and arc layer to lookup 1078 * DSL Crypto Keys for encryption. Callers must release the key with 1079 * spa_keystore_dsl_key_rele(). The function may also be called with 1080 * dck_out == NULL and tag == NULL to simply check that a key exists 1081 * without getting a reference to it. 1082 */ 1083 int 1084 spa_keystore_lookup_key(spa_t *spa, uint64_t dsobj, void *tag, 1085 dsl_crypto_key_t **dck_out) 1086 { 1087 int ret; 1088 dsl_key_mapping_t search_km; 1089 dsl_key_mapping_t *found_km; 1090 1091 ASSERT((tag != NULL && dck_out != NULL) || 1092 (tag == NULL && dck_out == NULL)); 1093 1094 /* init the search key mapping */ 1095 search_km.km_dsobj = dsobj; 1096 1097 rw_enter(&spa->spa_keystore.sk_km_lock, RW_READER); 1098 1099 /* remove the mapping from the tree */ 1100 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, &search_km, 1101 NULL); 1102 if (found_km == NULL) { 1103 ret = SET_ERROR(ENOENT); 1104 goto error_unlock; 1105 } 1106 1107 if (found_km && tag) 1108 zfs_refcount_add(&found_km->km_key->dck_holds, tag); 1109 1110 rw_exit(&spa->spa_keystore.sk_km_lock); 1111 1112 if (dck_out != NULL) 1113 *dck_out = found_km->km_key; 1114 return (0); 1115 1116 error_unlock: 1117 rw_exit(&spa->spa_keystore.sk_km_lock); 1118 1119 if (dck_out != NULL) 1120 *dck_out = NULL; 1121 return (ret); 1122 } 1123 1124 static int 1125 dmu_objset_check_wkey_loaded(dsl_dir_t *dd) 1126 { 1127 int ret; 1128 dsl_wrapping_key_t *wkey = NULL; 1129 1130 ret = spa_keystore_wkey_hold_dd(dd->dd_pool->dp_spa, dd, FTAG, 1131 &wkey); 1132 if (ret != 0) 1133 return (SET_ERROR(EACCES)); 1134 1135 dsl_wrapping_key_rele(wkey, FTAG); 1136 1137 return (0); 1138 } 1139 1140 static zfs_keystatus_t 1141 dsl_dataset_get_keystatus(dsl_dir_t *dd) 1142 { 1143 /* check if this dd has a has a dsl key */ 1144 if (dd->dd_crypto_obj == 0) 1145 return (ZFS_KEYSTATUS_NONE); 1146 1147 return (dmu_objset_check_wkey_loaded(dd) == 0 ? 1148 ZFS_KEYSTATUS_AVAILABLE : ZFS_KEYSTATUS_UNAVAILABLE); 1149 } 1150 1151 static int 1152 dsl_dir_get_crypt(dsl_dir_t *dd, uint64_t *crypt) 1153 { 1154 if (dd->dd_crypto_obj == 0) { 1155 *crypt = ZIO_CRYPT_OFF; 1156 return (0); 1157 } 1158 1159 return (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 1160 DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, crypt)); 1161 } 1162 1163 static void 1164 dsl_crypto_key_sync_impl(objset_t *mos, uint64_t dckobj, uint64_t crypt, 1165 uint64_t root_ddobj, uint64_t guid, uint8_t *iv, uint8_t *mac, 1166 uint8_t *keydata, uint8_t *hmac_keydata, uint64_t keyformat, 1167 uint64_t salt, uint64_t iters, dmu_tx_t *tx) 1168 { 1169 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 1170 &crypt, tx)); 1171 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, 1172 &root_ddobj, tx)); 1173 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, 1174 &guid, tx)); 1175 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 1176 iv, tx)); 1177 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 1178 mac, tx)); 1179 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 1180 MASTER_KEY_MAX_LEN, keydata, tx)); 1181 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 1182 SHA512_HMAC_KEYLEN, hmac_keydata, tx)); 1183 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 1184 8, 1, &keyformat, tx)); 1185 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 1186 8, 1, &salt, tx)); 1187 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 1188 8, 1, &iters, tx)); 1189 } 1190 1191 static void 1192 dsl_crypto_key_sync(dsl_crypto_key_t *dck, dmu_tx_t *tx) 1193 { 1194 zio_crypt_key_t *key = &dck->dck_key; 1195 dsl_wrapping_key_t *wkey = dck->dck_wkey; 1196 uint8_t keydata[MASTER_KEY_MAX_LEN]; 1197 uint8_t hmac_keydata[SHA512_HMAC_KEYLEN]; 1198 uint8_t iv[WRAPPING_IV_LEN]; 1199 uint8_t mac[WRAPPING_MAC_LEN]; 1200 1201 ASSERT(dmu_tx_is_syncing(tx)); 1202 ASSERT3U(key->zk_crypt, <, ZIO_CRYPT_FUNCTIONS); 1203 1204 /* encrypt and store the keys along with the IV and MAC */ 1205 VERIFY0(zio_crypt_key_wrap(&dck->dck_wkey->wk_key, key, iv, mac, 1206 keydata, hmac_keydata)); 1207 1208 /* update the ZAP with the obtained values */ 1209 dsl_crypto_key_sync_impl(tx->tx_pool->dp_meta_objset, dck->dck_obj, 1210 key->zk_crypt, wkey->wk_ddobj, key->zk_guid, iv, mac, keydata, 1211 hmac_keydata, wkey->wk_keyformat, wkey->wk_salt, wkey->wk_iters, 1212 tx); 1213 } 1214 1215 typedef struct spa_keystore_change_key_args { 1216 const char *skcka_dsname; 1217 dsl_crypto_params_t *skcka_cp; 1218 } spa_keystore_change_key_args_t; 1219 1220 static int 1221 spa_keystore_change_key_check(void *arg, dmu_tx_t *tx) 1222 { 1223 int ret; 1224 dsl_dir_t *dd = NULL; 1225 dsl_pool_t *dp = dmu_tx_pool(tx); 1226 spa_keystore_change_key_args_t *skcka = arg; 1227 dsl_crypto_params_t *dcp = skcka->skcka_cp; 1228 uint64_t rddobj; 1229 1230 /* check for the encryption feature */ 1231 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 1232 ret = SET_ERROR(ENOTSUP); 1233 goto error; 1234 } 1235 1236 /* check for valid key change command */ 1237 if (dcp->cp_cmd != DCP_CMD_NEW_KEY && 1238 dcp->cp_cmd != DCP_CMD_INHERIT && 1239 dcp->cp_cmd != DCP_CMD_FORCE_NEW_KEY && 1240 dcp->cp_cmd != DCP_CMD_FORCE_INHERIT) { 1241 ret = SET_ERROR(EINVAL); 1242 goto error; 1243 } 1244 1245 /* hold the dd */ 1246 ret = dsl_dir_hold(dp, skcka->skcka_dsname, FTAG, &dd, NULL); 1247 if (ret != 0) { 1248 dd = NULL; 1249 goto error; 1250 } 1251 1252 /* verify that the dataset is encrypted */ 1253 if (dd->dd_crypto_obj == 0) { 1254 ret = SET_ERROR(EINVAL); 1255 goto error; 1256 } 1257 1258 /* clones must always use their origin's key */ 1259 if (dsl_dir_is_clone(dd)) { 1260 ret = SET_ERROR(EINVAL); 1261 goto error; 1262 } 1263 1264 /* lookup the ddobj we are inheriting the keylocation from */ 1265 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 1266 if (ret != 0) 1267 goto error; 1268 1269 /* Handle inheritance */ 1270 if (dcp->cp_cmd == DCP_CMD_INHERIT || 1271 dcp->cp_cmd == DCP_CMD_FORCE_INHERIT) { 1272 /* no other encryption params should be given */ 1273 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT || 1274 dcp->cp_keylocation != NULL || 1275 dcp->cp_wkey != NULL) { 1276 ret = SET_ERROR(EINVAL); 1277 goto error; 1278 } 1279 1280 /* check that this is an encryption root */ 1281 if (dd->dd_object != rddobj) { 1282 ret = SET_ERROR(EINVAL); 1283 goto error; 1284 } 1285 1286 /* check that the parent is encrypted */ 1287 if (dd->dd_parent->dd_crypto_obj == 0) { 1288 ret = SET_ERROR(EINVAL); 1289 goto error; 1290 } 1291 1292 /* if we are rewrapping check that both keys are loaded */ 1293 if (dcp->cp_cmd == DCP_CMD_INHERIT) { 1294 ret = dmu_objset_check_wkey_loaded(dd); 1295 if (ret != 0) 1296 goto error; 1297 1298 ret = dmu_objset_check_wkey_loaded(dd->dd_parent); 1299 if (ret != 0) 1300 goto error; 1301 } 1302 1303 dsl_dir_rele(dd, FTAG); 1304 return (0); 1305 } 1306 1307 /* handle forcing an encryption root without rewrapping */ 1308 if (dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY) { 1309 /* no other encryption params should be given */ 1310 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT || 1311 dcp->cp_keylocation != NULL || 1312 dcp->cp_wkey != NULL) { 1313 ret = SET_ERROR(EINVAL); 1314 goto error; 1315 } 1316 1317 /* check that this is not an encryption root */ 1318 if (dd->dd_object == rddobj) { 1319 ret = SET_ERROR(EINVAL); 1320 goto error; 1321 } 1322 1323 dsl_dir_rele(dd, FTAG); 1324 return (0); 1325 } 1326 1327 /* crypt cannot be changed after creation */ 1328 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT) { 1329 ret = SET_ERROR(EINVAL); 1330 goto error; 1331 } 1332 1333 /* we are not inheritting our parent's wkey so we need one ourselves */ 1334 if (dcp->cp_wkey == NULL) { 1335 ret = SET_ERROR(EINVAL); 1336 goto error; 1337 } 1338 1339 /* check for a valid keyformat for the new wrapping key */ 1340 if (dcp->cp_wkey->wk_keyformat >= ZFS_KEYFORMAT_FORMATS || 1341 dcp->cp_wkey->wk_keyformat == ZFS_KEYFORMAT_NONE) { 1342 ret = SET_ERROR(EINVAL); 1343 goto error; 1344 } 1345 1346 /* 1347 * If this dataset is not currently an encryption root we need a new 1348 * keylocation for this dataset's new wrapping key. Otherwise we can 1349 * just keep the one we already had. 1350 */ 1351 if (dd->dd_object != rddobj && dcp->cp_keylocation == NULL) { 1352 ret = SET_ERROR(EINVAL); 1353 goto error; 1354 } 1355 1356 /* check that the keylocation is valid if it is not NULL */ 1357 if (dcp->cp_keylocation != NULL && 1358 !zfs_prop_valid_keylocation(dcp->cp_keylocation, B_TRUE)) { 1359 ret = SET_ERROR(EINVAL); 1360 goto error; 1361 } 1362 1363 /* passphrases require pbkdf2 salt and iters */ 1364 if (dcp->cp_wkey->wk_keyformat == ZFS_KEYFORMAT_PASSPHRASE) { 1365 if (dcp->cp_wkey->wk_salt == 0 || 1366 dcp->cp_wkey->wk_iters < MIN_PBKDF2_ITERATIONS) { 1367 ret = SET_ERROR(EINVAL); 1368 goto error; 1369 } 1370 } else { 1371 if (dcp->cp_wkey->wk_salt != 0 || dcp->cp_wkey->wk_iters != 0) { 1372 ret = SET_ERROR(EINVAL); 1373 goto error; 1374 } 1375 } 1376 1377 /* make sure the dd's wkey is loaded */ 1378 ret = dmu_objset_check_wkey_loaded(dd); 1379 if (ret != 0) 1380 goto error; 1381 1382 dsl_dir_rele(dd, FTAG); 1383 1384 return (0); 1385 1386 error: 1387 if (dd != NULL) 1388 dsl_dir_rele(dd, FTAG); 1389 1390 return (ret); 1391 } 1392 1393 /* 1394 * This function deals with the intricacies of updating wrapping 1395 * key references and encryption roots recursively in the event 1396 * of a call to 'zfs change-key' or 'zfs promote'. The 'skip' 1397 * parameter should always be set to B_FALSE when called 1398 * externally. 1399 */ 1400 static void 1401 spa_keystore_change_key_sync_impl(uint64_t rddobj, uint64_t ddobj, 1402 uint64_t new_rddobj, dsl_wrapping_key_t *wkey, boolean_t skip, 1403 dmu_tx_t *tx) 1404 { 1405 int ret; 1406 zap_cursor_t *zc; 1407 zap_attribute_t *za; 1408 dsl_pool_t *dp = dmu_tx_pool(tx); 1409 dsl_dir_t *dd = NULL; 1410 dsl_crypto_key_t *dck = NULL; 1411 uint64_t curr_rddobj; 1412 1413 ASSERT(RW_WRITE_HELD(&dp->dp_spa->spa_keystore.sk_wkeys_lock)); 1414 1415 /* hold the dd */ 1416 VERIFY0(dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd)); 1417 1418 /* ignore special dsl dirs */ 1419 if (dd->dd_myname[0] == '$' || dd->dd_myname[0] == '%') { 1420 dsl_dir_rele(dd, FTAG); 1421 return; 1422 } 1423 1424 ret = dsl_dir_get_encryption_root_ddobj(dd, &curr_rddobj); 1425 VERIFY(ret == 0 || ret == ENOENT); 1426 1427 /* 1428 * Stop recursing if this dsl dir didn't inherit from the root 1429 * or if this dd is a clone. 1430 */ 1431 if (ret == ENOENT || 1432 (!skip && (curr_rddobj != rddobj || dsl_dir_is_clone(dd)))) { 1433 dsl_dir_rele(dd, FTAG); 1434 return; 1435 } 1436 1437 /* 1438 * If we don't have a wrapping key just update the dck to reflect the 1439 * new encryption root. Otherwise rewrap the entire dck and re-sync it 1440 * to disk. If skip is set, we don't do any of this work. 1441 */ 1442 if (!skip) { 1443 if (wkey == NULL) { 1444 VERIFY0(zap_update(dp->dp_meta_objset, 1445 dd->dd_crypto_obj, 1446 DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, 1447 &new_rddobj, tx)); 1448 } else { 1449 VERIFY0(spa_keystore_dsl_key_hold_dd(dp->dp_spa, dd, 1450 FTAG, &dck)); 1451 dsl_wrapping_key_hold(wkey, dck); 1452 dsl_wrapping_key_rele(dck->dck_wkey, dck); 1453 dck->dck_wkey = wkey; 1454 dsl_crypto_key_sync(dck, tx); 1455 spa_keystore_dsl_key_rele(dp->dp_spa, dck, FTAG); 1456 } 1457 } 1458 1459 zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1460 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); 1461 1462 /* Recurse into all child dsl dirs. */ 1463 for (zap_cursor_init(zc, dp->dp_meta_objset, 1464 dsl_dir_phys(dd)->dd_child_dir_zapobj); 1465 zap_cursor_retrieve(zc, za) == 0; 1466 zap_cursor_advance(zc)) { 1467 spa_keystore_change_key_sync_impl(rddobj, 1468 za->za_first_integer, new_rddobj, wkey, B_FALSE, tx); 1469 } 1470 zap_cursor_fini(zc); 1471 1472 /* 1473 * Recurse into all dsl dirs of clones. We utilize the skip parameter 1474 * here so that we don't attempt to process the clones directly. This 1475 * is because the clone and its origin share the same dck, which has 1476 * already been updated. 1477 */ 1478 for (zap_cursor_init(zc, dp->dp_meta_objset, 1479 dsl_dir_phys(dd)->dd_clones); 1480 zap_cursor_retrieve(zc, za) == 0; 1481 zap_cursor_advance(zc)) { 1482 dsl_dataset_t *clone; 1483 1484 VERIFY0(dsl_dataset_hold_obj(dp, za->za_first_integer, 1485 FTAG, &clone)); 1486 spa_keystore_change_key_sync_impl(rddobj, 1487 clone->ds_dir->dd_object, new_rddobj, wkey, B_TRUE, tx); 1488 dsl_dataset_rele(clone, FTAG); 1489 } 1490 zap_cursor_fini(zc); 1491 1492 kmem_free(za, sizeof (zap_attribute_t)); 1493 kmem_free(zc, sizeof (zap_cursor_t)); 1494 1495 dsl_dir_rele(dd, FTAG); 1496 } 1497 1498 static void 1499 spa_keystore_change_key_sync(void *arg, dmu_tx_t *tx) 1500 { 1501 dsl_dataset_t *ds; 1502 avl_index_t where; 1503 dsl_pool_t *dp = dmu_tx_pool(tx); 1504 spa_t *spa = dp->dp_spa; 1505 spa_keystore_change_key_args_t *skcka = arg; 1506 dsl_crypto_params_t *dcp = skcka->skcka_cp; 1507 dsl_wrapping_key_t *wkey = NULL, *found_wkey; 1508 dsl_wrapping_key_t wkey_search; 1509 char *keylocation = dcp->cp_keylocation; 1510 uint64_t rddobj, new_rddobj; 1511 1512 /* create and initialize the wrapping key */ 1513 VERIFY0(dsl_dataset_hold(dp, skcka->skcka_dsname, FTAG, &ds)); 1514 ASSERT(!ds->ds_is_snapshot); 1515 1516 if (dcp->cp_cmd == DCP_CMD_NEW_KEY || 1517 dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY) { 1518 /* 1519 * We are changing to a new wkey. Set additional properties 1520 * which can be sent along with this ioctl. Note that this 1521 * command can set keylocation even if it can't normally be 1522 * set via 'zfs set' due to a non-local keylocation. 1523 */ 1524 if (dcp->cp_cmd == DCP_CMD_NEW_KEY) { 1525 wkey = dcp->cp_wkey; 1526 wkey->wk_ddobj = ds->ds_dir->dd_object; 1527 } else { 1528 keylocation = "prompt"; 1529 } 1530 1531 if (keylocation != NULL) { 1532 dsl_prop_set_sync_impl(ds, 1533 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1534 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, 1535 keylocation, tx); 1536 } 1537 1538 VERIFY0(dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj)); 1539 new_rddobj = ds->ds_dir->dd_object; 1540 } else { 1541 /* 1542 * We are inheritting the parent's wkey. Unset any local 1543 * keylocation and grab a reference to the wkey. 1544 */ 1545 if (dcp->cp_cmd == DCP_CMD_INHERIT) { 1546 VERIFY0(spa_keystore_wkey_hold_dd(spa, 1547 ds->ds_dir->dd_parent, FTAG, &wkey)); 1548 } 1549 1550 dsl_prop_set_sync_impl(ds, 1551 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), ZPROP_SRC_NONE, 1552 0, 0, NULL, tx); 1553 1554 rddobj = ds->ds_dir->dd_object; 1555 VERIFY0(dsl_dir_get_encryption_root_ddobj(ds->ds_dir->dd_parent, 1556 &new_rddobj)); 1557 } 1558 1559 if (wkey == NULL) { 1560 ASSERT(dcp->cp_cmd == DCP_CMD_FORCE_INHERIT || 1561 dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY); 1562 } 1563 1564 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 1565 1566 /* recurse through all children and rewrap their keys */ 1567 spa_keystore_change_key_sync_impl(rddobj, ds->ds_dir->dd_object, 1568 new_rddobj, wkey, B_FALSE, tx); 1569 1570 /* 1571 * All references to the old wkey should be released now (if it 1572 * existed). Replace the wrapping key. 1573 */ 1574 wkey_search.wk_ddobj = ds->ds_dir->dd_object; 1575 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, &wkey_search, NULL); 1576 if (found_wkey != NULL) { 1577 ASSERT0(zfs_refcount_count(&found_wkey->wk_refcnt)); 1578 avl_remove(&spa->spa_keystore.sk_wkeys, found_wkey); 1579 dsl_wrapping_key_free(found_wkey); 1580 } 1581 1582 if (dcp->cp_cmd == DCP_CMD_NEW_KEY) { 1583 avl_find(&spa->spa_keystore.sk_wkeys, wkey, &where); 1584 avl_insert(&spa->spa_keystore.sk_wkeys, wkey, where); 1585 } else if (wkey != NULL) { 1586 dsl_wrapping_key_rele(wkey, FTAG); 1587 } 1588 1589 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 1590 1591 dsl_dataset_rele(ds, FTAG); 1592 } 1593 1594 int 1595 spa_keystore_change_key(const char *dsname, dsl_crypto_params_t *dcp) 1596 { 1597 spa_keystore_change_key_args_t skcka; 1598 1599 /* initialize the args struct */ 1600 skcka.skcka_dsname = dsname; 1601 skcka.skcka_cp = dcp; 1602 1603 /* 1604 * Perform the actual work in syncing context. The blocks modified 1605 * here could be calculated but it would require holding the pool 1606 * lock and traversing all of the datasets that will have their keys 1607 * changed. 1608 */ 1609 return (dsl_sync_task(dsname, spa_keystore_change_key_check, 1610 spa_keystore_change_key_sync, &skcka, 15, 1611 ZFS_SPACE_CHECK_RESERVED)); 1612 } 1613 1614 int 1615 dsl_dir_rename_crypt_check(dsl_dir_t *dd, dsl_dir_t *newparent) 1616 { 1617 int ret; 1618 uint64_t curr_rddobj, parent_rddobj; 1619 1620 if (dd->dd_crypto_obj == 0) 1621 return (0); 1622 1623 ret = dsl_dir_get_encryption_root_ddobj(dd, &curr_rddobj); 1624 if (ret != 0) 1625 goto error; 1626 1627 /* 1628 * if this is not an encryption root, we must make sure we are not 1629 * moving dd to a new encryption root 1630 */ 1631 if (dd->dd_object != curr_rddobj) { 1632 ret = dsl_dir_get_encryption_root_ddobj(newparent, 1633 &parent_rddobj); 1634 if (ret != 0) 1635 goto error; 1636 1637 if (parent_rddobj != curr_rddobj) { 1638 ret = SET_ERROR(EACCES); 1639 goto error; 1640 } 1641 } 1642 1643 return (0); 1644 1645 error: 1646 return (ret); 1647 } 1648 1649 /* 1650 * Check to make sure that a promote from targetdd to origindd will not require 1651 * any key rewraps. 1652 */ 1653 int 1654 dsl_dataset_promote_crypt_check(dsl_dir_t *target, dsl_dir_t *origin) 1655 { 1656 int ret; 1657 uint64_t rddobj, op_rddobj, tp_rddobj; 1658 1659 /* If the dataset is not encrypted we don't need to check anything */ 1660 if (origin->dd_crypto_obj == 0) 1661 return (0); 1662 1663 /* 1664 * If we are not changing the first origin snapshot in a chain 1665 * the encryption root won't change either. 1666 */ 1667 if (dsl_dir_is_clone(origin)) 1668 return (0); 1669 1670 /* 1671 * If the origin is the encryption root we will update 1672 * the DSL Crypto Key to point to the target instead. 1673 */ 1674 ret = dsl_dir_get_encryption_root_ddobj(origin, &rddobj); 1675 if (ret != 0) 1676 return (ret); 1677 1678 if (rddobj == origin->dd_object) 1679 return (0); 1680 1681 /* 1682 * The origin is inheriting its encryption root from its parent. 1683 * Check that the parent of the target has the same encryption root. 1684 */ 1685 ret = dsl_dir_get_encryption_root_ddobj(origin->dd_parent, &op_rddobj); 1686 if (ret == ENOENT) 1687 return (SET_ERROR(EACCES)); 1688 else if (ret != 0) 1689 return (ret); 1690 1691 ret = dsl_dir_get_encryption_root_ddobj(target->dd_parent, &tp_rddobj); 1692 if (ret == ENOENT) 1693 return (SET_ERROR(EACCES)); 1694 else if (ret != 0) 1695 return (ret); 1696 1697 if (op_rddobj != tp_rddobj) 1698 return (SET_ERROR(EACCES)); 1699 1700 return (0); 1701 } 1702 1703 void 1704 dsl_dataset_promote_crypt_sync(dsl_dir_t *target, dsl_dir_t *origin, 1705 dmu_tx_t *tx) 1706 { 1707 uint64_t rddobj; 1708 dsl_pool_t *dp = target->dd_pool; 1709 dsl_dataset_t *targetds; 1710 dsl_dataset_t *originds; 1711 char *keylocation; 1712 1713 if (origin->dd_crypto_obj == 0) 1714 return; 1715 if (dsl_dir_is_clone(origin)) 1716 return; 1717 1718 VERIFY0(dsl_dir_get_encryption_root_ddobj(origin, &rddobj)); 1719 1720 if (rddobj != origin->dd_object) 1721 return; 1722 1723 /* 1724 * If the target is being promoted to the encryption root update the 1725 * DSL Crypto Key and keylocation to reflect that. We also need to 1726 * update the DSL Crypto Keys of all children inheritting their 1727 * encryption root to point to the new target. Otherwise, the check 1728 * function ensured that the encryption root will not change. 1729 */ 1730 keylocation = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP); 1731 1732 VERIFY0(dsl_dataset_hold_obj(dp, 1733 dsl_dir_phys(target)->dd_head_dataset_obj, FTAG, &targetds)); 1734 VERIFY0(dsl_dataset_hold_obj(dp, 1735 dsl_dir_phys(origin)->dd_head_dataset_obj, FTAG, &originds)); 1736 1737 VERIFY0(dsl_prop_get_dd(origin, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1738 1, ZAP_MAXVALUELEN, keylocation, NULL, B_FALSE)); 1739 dsl_prop_set_sync_impl(targetds, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1740 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, keylocation, tx); 1741 dsl_prop_set_sync_impl(originds, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1742 ZPROP_SRC_NONE, 0, 0, NULL, tx); 1743 1744 rw_enter(&dp->dp_spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 1745 spa_keystore_change_key_sync_impl(rddobj, origin->dd_object, 1746 target->dd_object, NULL, B_FALSE, tx); 1747 rw_exit(&dp->dp_spa->spa_keystore.sk_wkeys_lock); 1748 1749 dsl_dataset_rele(targetds, FTAG); 1750 dsl_dataset_rele(originds, FTAG); 1751 kmem_free(keylocation, ZAP_MAXVALUELEN); 1752 } 1753 1754 int 1755 dmu_objset_create_crypt_check(dsl_dir_t *parentdd, dsl_crypto_params_t *dcp, 1756 boolean_t *will_encrypt) 1757 { 1758 int ret; 1759 uint64_t pcrypt, crypt; 1760 dsl_crypto_params_t dummy_dcp = { 0 }; 1761 1762 if (will_encrypt != NULL) 1763 *will_encrypt = B_FALSE; 1764 1765 if (dcp == NULL) 1766 dcp = &dummy_dcp; 1767 1768 if (dcp->cp_cmd != DCP_CMD_NONE) 1769 return (SET_ERROR(EINVAL)); 1770 1771 if (parentdd != NULL) { 1772 ret = dsl_dir_get_crypt(parentdd, &pcrypt); 1773 if (ret != 0) 1774 return (ret); 1775 } else { 1776 pcrypt = ZIO_CRYPT_OFF; 1777 } 1778 1779 crypt = (dcp->cp_crypt == ZIO_CRYPT_INHERIT) ? pcrypt : dcp->cp_crypt; 1780 1781 ASSERT3U(pcrypt, !=, ZIO_CRYPT_INHERIT); 1782 ASSERT3U(crypt, !=, ZIO_CRYPT_INHERIT); 1783 1784 /* check for valid dcp with no encryption (inherited or local) */ 1785 if (crypt == ZIO_CRYPT_OFF) { 1786 /* Must not specify encryption params */ 1787 if (dcp->cp_wkey != NULL || 1788 (dcp->cp_keylocation != NULL && 1789 strcmp(dcp->cp_keylocation, "none") != 0)) 1790 return (SET_ERROR(EINVAL)); 1791 1792 return (0); 1793 } 1794 1795 if (will_encrypt != NULL) 1796 *will_encrypt = B_TRUE; 1797 1798 /* 1799 * We will now definitely be encrypting. Check the feature flag. When 1800 * creating the pool the caller will check this for us since we won't 1801 * technically have the feature activated yet. 1802 */ 1803 if (parentdd != NULL && 1804 !spa_feature_is_enabled(parentdd->dd_pool->dp_spa, 1805 SPA_FEATURE_ENCRYPTION)) { 1806 return (SET_ERROR(EOPNOTSUPP)); 1807 } 1808 1809 /* Check for errata #4 (encryption enabled, bookmark_v2 disabled) */ 1810 if (parentdd != NULL && 1811 !spa_feature_is_enabled(parentdd->dd_pool->dp_spa, 1812 SPA_FEATURE_BOOKMARK_V2)) { 1813 return (SET_ERROR(EOPNOTSUPP)); 1814 } 1815 1816 /* handle inheritance */ 1817 if (dcp->cp_wkey == NULL) { 1818 ASSERT3P(parentdd, !=, NULL); 1819 1820 /* key must be fully unspecified */ 1821 if (dcp->cp_keylocation != NULL) 1822 return (SET_ERROR(EINVAL)); 1823 1824 /* parent must have a key to inherit */ 1825 if (pcrypt == ZIO_CRYPT_OFF) 1826 return (SET_ERROR(EINVAL)); 1827 1828 /* check for parent key */ 1829 ret = dmu_objset_check_wkey_loaded(parentdd); 1830 if (ret != 0) 1831 return (ret); 1832 1833 return (0); 1834 } 1835 1836 /* At this point we should have a fully specified key. Check location */ 1837 if (dcp->cp_keylocation == NULL || 1838 !zfs_prop_valid_keylocation(dcp->cp_keylocation, B_TRUE)) 1839 return (SET_ERROR(EINVAL)); 1840 1841 /* Must have fully specified keyformat */ 1842 switch (dcp->cp_wkey->wk_keyformat) { 1843 case ZFS_KEYFORMAT_HEX: 1844 case ZFS_KEYFORMAT_RAW: 1845 /* requires no pbkdf2 iters and salt */ 1846 if (dcp->cp_wkey->wk_salt != 0 || dcp->cp_wkey->wk_iters != 0) 1847 return (SET_ERROR(EINVAL)); 1848 break; 1849 case ZFS_KEYFORMAT_PASSPHRASE: 1850 /* requires pbkdf2 iters and salt */ 1851 if (dcp->cp_wkey->wk_salt == 0 || 1852 dcp->cp_wkey->wk_iters < MIN_PBKDF2_ITERATIONS) 1853 return (SET_ERROR(EINVAL)); 1854 break; 1855 case ZFS_KEYFORMAT_NONE: 1856 default: 1857 /* keyformat must be specified and valid */ 1858 return (SET_ERROR(EINVAL)); 1859 } 1860 1861 return (0); 1862 } 1863 1864 void 1865 dsl_dataset_create_crypt_sync(uint64_t dsobj, dsl_dir_t *dd, 1866 dsl_dataset_t *origin, dsl_crypto_params_t *dcp, dmu_tx_t *tx) 1867 { 1868 dsl_pool_t *dp = dd->dd_pool; 1869 uint64_t crypt; 1870 dsl_wrapping_key_t *wkey; 1871 1872 /* clones always use their origin's wrapping key */ 1873 if (dsl_dir_is_clone(dd)) { 1874 ASSERT3P(dcp, ==, NULL); 1875 1876 /* 1877 * If this is an encrypted clone we just need to clone the 1878 * dck into dd. Zapify the dd so we can do that. 1879 */ 1880 if (origin->ds_dir->dd_crypto_obj != 0) { 1881 dmu_buf_will_dirty(dd->dd_dbuf, tx); 1882 dsl_dir_zapify(dd, tx); 1883 1884 dd->dd_crypto_obj = 1885 dsl_crypto_key_clone_sync(origin->ds_dir, tx); 1886 VERIFY0(zap_add(dp->dp_meta_objset, dd->dd_object, 1887 DD_FIELD_CRYPTO_KEY_OBJ, sizeof (uint64_t), 1, 1888 &dd->dd_crypto_obj, tx)); 1889 } 1890 1891 return; 1892 } 1893 1894 /* 1895 * A NULL dcp at this point indicates this is the origin dataset 1896 * which does not have an objset to encrypt. Raw receives will handle 1897 * encryption separately later. In both cases we can simply return. 1898 */ 1899 if (dcp == NULL || dcp->cp_cmd == DCP_CMD_RAW_RECV) 1900 return; 1901 1902 crypt = dcp->cp_crypt; 1903 wkey = dcp->cp_wkey; 1904 1905 /* figure out the effective crypt */ 1906 if (crypt == ZIO_CRYPT_INHERIT && dd->dd_parent != NULL) 1907 VERIFY0(dsl_dir_get_crypt(dd->dd_parent, &crypt)); 1908 1909 /* if we aren't doing encryption just return */ 1910 if (crypt == ZIO_CRYPT_OFF || crypt == ZIO_CRYPT_INHERIT) 1911 return; 1912 1913 /* zapify the dd so that we can add the crypto key obj to it */ 1914 dmu_buf_will_dirty(dd->dd_dbuf, tx); 1915 dsl_dir_zapify(dd, tx); 1916 1917 /* use the new key if given or inherit from the parent */ 1918 if (wkey == NULL) { 1919 VERIFY0(spa_keystore_wkey_hold_dd(dp->dp_spa, 1920 dd->dd_parent, FTAG, &wkey)); 1921 } else { 1922 wkey->wk_ddobj = dd->dd_object; 1923 } 1924 1925 ASSERT3P(wkey, !=, NULL); 1926 1927 /* Create or clone the DSL crypto key and activate the feature */ 1928 dd->dd_crypto_obj = dsl_crypto_key_create_sync(crypt, wkey, tx); 1929 VERIFY0(zap_add(dp->dp_meta_objset, dd->dd_object, 1930 DD_FIELD_CRYPTO_KEY_OBJ, sizeof (uint64_t), 1, &dd->dd_crypto_obj, 1931 tx)); 1932 dsl_dataset_activate_feature(dsobj, SPA_FEATURE_ENCRYPTION, 1933 (void *)B_TRUE, tx); 1934 1935 /* 1936 * If we inherited the wrapping key we release our reference now. 1937 * Otherwise, this is a new key and we need to load it into the 1938 * keystore. 1939 */ 1940 if (dcp->cp_wkey == NULL) { 1941 dsl_wrapping_key_rele(wkey, FTAG); 1942 } else { 1943 VERIFY0(spa_keystore_load_wkey_impl(dp->dp_spa, wkey)); 1944 } 1945 } 1946 1947 typedef struct dsl_crypto_recv_key_arg { 1948 uint64_t dcrka_dsobj; 1949 uint64_t dcrka_fromobj; 1950 dmu_objset_type_t dcrka_ostype; 1951 nvlist_t *dcrka_nvl; 1952 boolean_t dcrka_do_key; 1953 } dsl_crypto_recv_key_arg_t; 1954 1955 static int 1956 dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dsl_dataset_t *fromds, 1957 dmu_objset_type_t ostype, nvlist_t *nvl, dmu_tx_t *tx) 1958 { 1959 int ret; 1960 objset_t *os; 1961 dnode_t *mdn; 1962 uint8_t *buf = NULL; 1963 uint_t len; 1964 uint64_t intval, nlevels, blksz, ibs; 1965 uint64_t nblkptr, maxblkid; 1966 1967 if (ostype != DMU_OST_ZFS && ostype != DMU_OST_ZVOL) 1968 return (SET_ERROR(EINVAL)); 1969 1970 /* raw receives also need info about the structure of the metadnode */ 1971 ret = nvlist_lookup_uint64(nvl, "mdn_compress", &intval); 1972 if (ret != 0 || intval >= ZIO_COMPRESS_LEGACY_FUNCTIONS) 1973 return (SET_ERROR(EINVAL)); 1974 1975 ret = nvlist_lookup_uint64(nvl, "mdn_checksum", &intval); 1976 if (ret != 0 || intval >= ZIO_CHECKSUM_LEGACY_FUNCTIONS) 1977 return (SET_ERROR(EINVAL)); 1978 1979 ret = nvlist_lookup_uint64(nvl, "mdn_nlevels", &nlevels); 1980 if (ret != 0 || nlevels > DN_MAX_LEVELS) 1981 return (SET_ERROR(EINVAL)); 1982 1983 ret = nvlist_lookup_uint64(nvl, "mdn_blksz", &blksz); 1984 if (ret != 0 || blksz < SPA_MINBLOCKSIZE) 1985 return (SET_ERROR(EINVAL)); 1986 else if (blksz > spa_maxblocksize(tx->tx_pool->dp_spa)) 1987 return (SET_ERROR(ENOTSUP)); 1988 1989 ret = nvlist_lookup_uint64(nvl, "mdn_indblkshift", &ibs); 1990 if (ret != 0 || ibs < DN_MIN_INDBLKSHIFT || ibs > DN_MAX_INDBLKSHIFT) 1991 return (SET_ERROR(ENOTSUP)); 1992 1993 ret = nvlist_lookup_uint64(nvl, "mdn_nblkptr", &nblkptr); 1994 if (ret != 0 || nblkptr != DN_MAX_NBLKPTR) 1995 return (SET_ERROR(ENOTSUP)); 1996 1997 ret = nvlist_lookup_uint64(nvl, "mdn_maxblkid", &maxblkid); 1998 if (ret != 0) 1999 return (SET_ERROR(EINVAL)); 2000 2001 ret = nvlist_lookup_uint8_array(nvl, "portable_mac", &buf, &len); 2002 if (ret != 0 || len != ZIO_OBJSET_MAC_LEN) 2003 return (SET_ERROR(EINVAL)); 2004 2005 ret = dmu_objset_from_ds(ds, &os); 2006 if (ret != 0) 2007 return (ret); 2008 2009 mdn = DMU_META_DNODE(os); 2010 2011 /* 2012 * If we already created the objset, make sure its unchangeable 2013 * properties match the ones received in the nvlist. 2014 */ 2015 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2016 if (!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) && 2017 (mdn->dn_nlevels != nlevels || mdn->dn_datablksz != blksz || 2018 mdn->dn_indblkshift != ibs || mdn->dn_nblkptr != nblkptr)) { 2019 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2020 return (SET_ERROR(EINVAL)); 2021 } 2022 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2023 2024 /* 2025 * Check that the ivset guid of the fromds matches the one from the 2026 * send stream. Older versions of the encryption code did not have 2027 * an ivset guid on the from dataset and did not send one in the 2028 * stream. For these streams we provide the 2029 * zfs_disable_ivset_guid_check tunable to allow these datasets to 2030 * be received with a generated ivset guid. 2031 */ 2032 if (fromds != NULL && !zfs_disable_ivset_guid_check) { 2033 uint64_t from_ivset_guid = 0; 2034 intval = 0; 2035 2036 (void) nvlist_lookup_uint64(nvl, "from_ivset_guid", &intval); 2037 (void) zap_lookup(tx->tx_pool->dp_meta_objset, 2038 fromds->ds_object, DS_FIELD_IVSET_GUID, 2039 sizeof (from_ivset_guid), 1, &from_ivset_guid); 2040 2041 if (intval == 0 || from_ivset_guid == 0) 2042 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISSING)); 2043 2044 if (intval != from_ivset_guid) 2045 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISMATCH)); 2046 } 2047 2048 return (0); 2049 } 2050 2051 static void 2052 dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype, 2053 nvlist_t *nvl, dmu_tx_t *tx) 2054 { 2055 dsl_pool_t *dp = tx->tx_pool; 2056 objset_t *os; 2057 dnode_t *mdn; 2058 zio_t *zio; 2059 uint8_t *portable_mac; 2060 uint_t len; 2061 uint64_t compress, checksum, nlevels, blksz, ibs, maxblkid; 2062 boolean_t newds = B_FALSE; 2063 2064 VERIFY0(dmu_objset_from_ds(ds, &os)); 2065 mdn = DMU_META_DNODE(os); 2066 2067 /* 2068 * Fetch the values we need from the nvlist. "to_ivset_guid" must 2069 * be set on the snapshot, which doesn't exist yet. The receive 2070 * code will take care of this for us later. 2071 */ 2072 compress = fnvlist_lookup_uint64(nvl, "mdn_compress"); 2073 checksum = fnvlist_lookup_uint64(nvl, "mdn_checksum"); 2074 nlevels = fnvlist_lookup_uint64(nvl, "mdn_nlevels"); 2075 blksz = fnvlist_lookup_uint64(nvl, "mdn_blksz"); 2076 ibs = fnvlist_lookup_uint64(nvl, "mdn_indblkshift"); 2077 maxblkid = fnvlist_lookup_uint64(nvl, "mdn_maxblkid"); 2078 VERIFY0(nvlist_lookup_uint8_array(nvl, "portable_mac", &portable_mac, 2079 &len)); 2080 2081 /* if we haven't created an objset for the ds yet, do that now */ 2082 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2083 if (BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) { 2084 (void) dmu_objset_create_impl_dnstats(dp->dp_spa, ds, 2085 dsl_dataset_get_blkptr(ds), ostype, nlevels, blksz, 2086 ibs, tx); 2087 newds = B_TRUE; 2088 } 2089 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2090 2091 /* 2092 * Set the portable MAC. The local MAC will always be zero since the 2093 * incoming data will all be portable and user accounting will be 2094 * deferred until the next mount. Afterwards, flag the os to be 2095 * written out raw next time. 2096 */ 2097 arc_release(os->os_phys_buf, &os->os_phys_buf); 2098 memcpy(os->os_phys->os_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); 2099 memset(os->os_phys->os_local_mac, 0, ZIO_OBJSET_MAC_LEN); 2100 os->os_flags &= ~OBJSET_FLAG_USERACCOUNTING_COMPLETE; 2101 os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE; 2102 2103 /* set metadnode compression and checksum */ 2104 mdn->dn_compress = compress; 2105 mdn->dn_checksum = checksum; 2106 2107 rw_enter(&mdn->dn_struct_rwlock, RW_WRITER); 2108 dnode_new_blkid(mdn, maxblkid, tx, B_FALSE, B_TRUE); 2109 rw_exit(&mdn->dn_struct_rwlock); 2110 2111 /* 2112 * We can't normally dirty the dataset in syncing context unless 2113 * we are creating a new dataset. In this case, we perform a 2114 * pseudo txg sync here instead. 2115 */ 2116 if (newds) { 2117 dsl_dataset_dirty(ds, tx); 2118 } else { 2119 zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED); 2120 dsl_dataset_sync(ds, zio, tx); 2121 VERIFY0(zio_wait(zio)); 2122 2123 /* dsl_dataset_sync_done will drop this reference. */ 2124 dmu_buf_add_ref(ds->ds_dbuf, ds); 2125 dsl_dataset_sync_done(ds, tx); 2126 } 2127 } 2128 2129 int 2130 dsl_crypto_recv_raw_key_check(dsl_dataset_t *ds, nvlist_t *nvl, dmu_tx_t *tx) 2131 { 2132 int ret; 2133 objset_t *mos = tx->tx_pool->dp_meta_objset; 2134 uint8_t *buf = NULL; 2135 uint_t len; 2136 uint64_t intval, key_guid, version; 2137 boolean_t is_passphrase = B_FALSE; 2138 2139 ASSERT(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT); 2140 2141 /* 2142 * Read and check all the encryption values from the nvlist. We need 2143 * all of the fields of a DSL Crypto Key, as well as a fully specified 2144 * wrapping key. 2145 */ 2146 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, &intval); 2147 if (ret != 0 || intval >= ZIO_CRYPT_FUNCTIONS || 2148 intval <= ZIO_CRYPT_OFF) 2149 return (SET_ERROR(EINVAL)); 2150 2151 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID, &intval); 2152 if (ret != 0) 2153 return (SET_ERROR(EINVAL)); 2154 2155 /* 2156 * If this is an incremental receive make sure the given key guid 2157 * matches the one we already have. 2158 */ 2159 if (ds->ds_dir->dd_crypto_obj != 0) { 2160 ret = zap_lookup(mos, ds->ds_dir->dd_crypto_obj, 2161 DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid); 2162 if (ret != 0) 2163 return (ret); 2164 if (intval != key_guid) 2165 return (SET_ERROR(EACCES)); 2166 } 2167 2168 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2169 &buf, &len); 2170 if (ret != 0 || len != MASTER_KEY_MAX_LEN) 2171 return (SET_ERROR(EINVAL)); 2172 2173 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2174 &buf, &len); 2175 if (ret != 0 || len != SHA512_HMAC_KEYLEN) 2176 return (SET_ERROR(EINVAL)); 2177 2178 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_IV, &buf, &len); 2179 if (ret != 0 || len != WRAPPING_IV_LEN) 2180 return (SET_ERROR(EINVAL)); 2181 2182 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, &buf, &len); 2183 if (ret != 0 || len != WRAPPING_MAC_LEN) 2184 return (SET_ERROR(EINVAL)); 2185 2186 /* 2187 * We don't support receiving old on-disk formats. The version 0 2188 * implementation protected several fields in an objset that were 2189 * not always portable during a raw receive. As a result, we call 2190 * the old version an on-disk errata #3. 2191 */ 2192 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_VERSION, &version); 2193 if (ret != 0 || version != ZIO_CRYPT_KEY_CURRENT_VERSION) 2194 return (SET_ERROR(ENOTSUP)); 2195 2196 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 2197 &intval); 2198 if (ret != 0 || intval >= ZFS_KEYFORMAT_FORMATS || 2199 intval == ZFS_KEYFORMAT_NONE) 2200 return (SET_ERROR(EINVAL)); 2201 2202 is_passphrase = (intval == ZFS_KEYFORMAT_PASSPHRASE); 2203 2204 /* 2205 * for raw receives we allow any number of pbkdf2iters since there 2206 * won't be a chance for the user to change it. 2207 */ 2208 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 2209 &intval); 2210 if (ret != 0 || (is_passphrase == (intval == 0))) 2211 return (SET_ERROR(EINVAL)); 2212 2213 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 2214 &intval); 2215 if (ret != 0 || (is_passphrase == (intval == 0))) 2216 return (SET_ERROR(EINVAL)); 2217 2218 return (0); 2219 } 2220 2221 void 2222 dsl_crypto_recv_raw_key_sync(dsl_dataset_t *ds, nvlist_t *nvl, dmu_tx_t *tx) 2223 { 2224 dsl_pool_t *dp = tx->tx_pool; 2225 objset_t *mos = dp->dp_meta_objset; 2226 dsl_dir_t *dd = ds->ds_dir; 2227 uint_t len; 2228 uint64_t rddobj, one = 1; 2229 uint8_t *keydata, *hmac_keydata, *iv, *mac; 2230 uint64_t crypt, key_guid, keyformat, iters, salt; 2231 uint64_t version = ZIO_CRYPT_KEY_CURRENT_VERSION; 2232 char *keylocation = "prompt"; 2233 2234 /* lookup the values we need to create the DSL Crypto Key */ 2235 crypt = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE); 2236 key_guid = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID); 2237 keyformat = fnvlist_lookup_uint64(nvl, 2238 zfs_prop_to_name(ZFS_PROP_KEYFORMAT)); 2239 iters = fnvlist_lookup_uint64(nvl, 2240 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS)); 2241 salt = fnvlist_lookup_uint64(nvl, 2242 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT)); 2243 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2244 &keydata, &len)); 2245 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2246 &hmac_keydata, &len)); 2247 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_IV, &iv, &len)); 2248 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, &mac, &len)); 2249 2250 /* if this is a new dataset setup the DSL Crypto Key. */ 2251 if (dd->dd_crypto_obj == 0) { 2252 /* zapify the dsl dir so we can add the key object to it */ 2253 dmu_buf_will_dirty(dd->dd_dbuf, tx); 2254 dsl_dir_zapify(dd, tx); 2255 2256 /* create the DSL Crypto Key on disk and activate the feature */ 2257 dd->dd_crypto_obj = zap_create(mos, 2258 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 2259 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, 2260 dd->dd_crypto_obj, DSL_CRYPTO_KEY_REFCOUNT, 2261 sizeof (uint64_t), 1, &one, tx)); 2262 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, 2263 dd->dd_crypto_obj, DSL_CRYPTO_KEY_VERSION, 2264 sizeof (uint64_t), 1, &version, tx)); 2265 2266 dsl_dataset_activate_feature(ds->ds_object, 2267 SPA_FEATURE_ENCRYPTION, (void *)B_TRUE, tx); 2268 ds->ds_feature[SPA_FEATURE_ENCRYPTION] = (void *)B_TRUE; 2269 2270 /* save the dd_crypto_obj on disk */ 2271 VERIFY0(zap_add(mos, dd->dd_object, DD_FIELD_CRYPTO_KEY_OBJ, 2272 sizeof (uint64_t), 1, &dd->dd_crypto_obj, tx)); 2273 2274 /* 2275 * Set the keylocation to prompt by default. If keylocation 2276 * has been provided via the properties, this will be overridden 2277 * later. 2278 */ 2279 dsl_prop_set_sync_impl(ds, 2280 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 2281 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, 2282 keylocation, tx); 2283 2284 rddobj = dd->dd_object; 2285 } else { 2286 VERIFY0(dsl_dir_get_encryption_root_ddobj(dd, &rddobj)); 2287 } 2288 2289 /* sync the key data to the ZAP object on disk */ 2290 dsl_crypto_key_sync_impl(mos, dd->dd_crypto_obj, crypt, 2291 rddobj, key_guid, iv, mac, keydata, hmac_keydata, keyformat, salt, 2292 iters, tx); 2293 } 2294 2295 static int 2296 dsl_crypto_recv_key_check(void *arg, dmu_tx_t *tx) 2297 { 2298 int ret; 2299 dsl_crypto_recv_key_arg_t *dcrka = arg; 2300 dsl_dataset_t *ds = NULL, *fromds = NULL; 2301 2302 ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_dsobj, 2303 FTAG, &ds); 2304 if (ret != 0) 2305 goto out; 2306 2307 if (dcrka->dcrka_fromobj != 0) { 2308 ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_fromobj, 2309 FTAG, &fromds); 2310 if (ret != 0) 2311 goto out; 2312 } 2313 2314 ret = dsl_crypto_recv_raw_objset_check(ds, fromds, 2315 dcrka->dcrka_ostype, dcrka->dcrka_nvl, tx); 2316 if (ret != 0) 2317 goto out; 2318 2319 /* 2320 * We run this check even if we won't be doing this part of 2321 * the receive now so that we don't make the user wait until 2322 * the receive finishes to fail. 2323 */ 2324 ret = dsl_crypto_recv_raw_key_check(ds, dcrka->dcrka_nvl, tx); 2325 if (ret != 0) 2326 goto out; 2327 2328 out: 2329 if (ds != NULL) 2330 dsl_dataset_rele(ds, FTAG); 2331 if (fromds != NULL) 2332 dsl_dataset_rele(fromds, FTAG); 2333 return (ret); 2334 } 2335 2336 static void 2337 dsl_crypto_recv_key_sync(void *arg, dmu_tx_t *tx) 2338 { 2339 dsl_crypto_recv_key_arg_t *dcrka = arg; 2340 dsl_dataset_t *ds; 2341 2342 VERIFY0(dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_dsobj, 2343 FTAG, &ds)); 2344 dsl_crypto_recv_raw_objset_sync(ds, dcrka->dcrka_ostype, 2345 dcrka->dcrka_nvl, tx); 2346 if (dcrka->dcrka_do_key) 2347 dsl_crypto_recv_raw_key_sync(ds, dcrka->dcrka_nvl, tx); 2348 dsl_dataset_rele(ds, FTAG); 2349 } 2350 2351 /* 2352 * This function is used to sync an nvlist representing a DSL Crypto Key and 2353 * the associated encryption parameters. The key will be written exactly as is 2354 * without wrapping it. 2355 */ 2356 int 2357 dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj, uint64_t fromobj, 2358 dmu_objset_type_t ostype, nvlist_t *nvl, boolean_t do_key) 2359 { 2360 dsl_crypto_recv_key_arg_t dcrka; 2361 2362 dcrka.dcrka_dsobj = dsobj; 2363 dcrka.dcrka_fromobj = fromobj; 2364 dcrka.dcrka_ostype = ostype; 2365 dcrka.dcrka_nvl = nvl; 2366 dcrka.dcrka_do_key = do_key; 2367 2368 return (dsl_sync_task(poolname, dsl_crypto_recv_key_check, 2369 dsl_crypto_recv_key_sync, &dcrka, 1, ZFS_SPACE_CHECK_NORMAL)); 2370 } 2371 2372 int 2373 dsl_crypto_populate_key_nvlist(objset_t *os, uint64_t from_ivset_guid, 2374 nvlist_t **nvl_out) 2375 { 2376 int ret; 2377 dsl_dataset_t *ds = os->os_dsl_dataset; 2378 dnode_t *mdn; 2379 uint64_t rddobj; 2380 nvlist_t *nvl = NULL; 2381 uint64_t dckobj = ds->ds_dir->dd_crypto_obj; 2382 dsl_dir_t *rdd = NULL; 2383 dsl_pool_t *dp = ds->ds_dir->dd_pool; 2384 objset_t *mos = dp->dp_meta_objset; 2385 uint64_t crypt = 0, key_guid = 0, format = 0; 2386 uint64_t iters = 0, salt = 0, version = 0; 2387 uint64_t to_ivset_guid = 0; 2388 uint8_t raw_keydata[MASTER_KEY_MAX_LEN]; 2389 uint8_t raw_hmac_keydata[SHA512_HMAC_KEYLEN]; 2390 uint8_t iv[WRAPPING_IV_LEN]; 2391 uint8_t mac[WRAPPING_MAC_LEN]; 2392 2393 ASSERT(dckobj != 0); 2394 2395 mdn = DMU_META_DNODE(os); 2396 2397 nvl = fnvlist_alloc(); 2398 2399 /* lookup values from the DSL Crypto Key */ 2400 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 2401 &crypt); 2402 if (ret != 0) 2403 goto error; 2404 2405 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid); 2406 if (ret != 0) 2407 goto error; 2408 2409 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 2410 MASTER_KEY_MAX_LEN, raw_keydata); 2411 if (ret != 0) 2412 goto error; 2413 2414 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 2415 SHA512_HMAC_KEYLEN, raw_hmac_keydata); 2416 if (ret != 0) 2417 goto error; 2418 2419 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 2420 iv); 2421 if (ret != 0) 2422 goto error; 2423 2424 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 2425 mac); 2426 if (ret != 0) 2427 goto error; 2428 2429 /* see zfs_disable_ivset_guid_check tunable for errata info */ 2430 ret = zap_lookup(mos, ds->ds_object, DS_FIELD_IVSET_GUID, 8, 1, 2431 &to_ivset_guid); 2432 if (ret != 0) 2433 ASSERT3U(dp->dp_spa->spa_errata, !=, 0); 2434 2435 /* 2436 * We don't support raw sends of legacy on-disk formats. See the 2437 * comment in dsl_crypto_recv_key_check() for details. 2438 */ 2439 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_VERSION, 8, 1, &version); 2440 if (ret != 0 || version != ZIO_CRYPT_KEY_CURRENT_VERSION) { 2441 dp->dp_spa->spa_errata = ZPOOL_ERRATA_ZOL_6845_ENCRYPTION; 2442 ret = SET_ERROR(ENOTSUP); 2443 goto error; 2444 } 2445 2446 /* 2447 * Lookup wrapping key properties. An early version of the code did 2448 * not correctly add these values to the wrapping key or the DSL 2449 * Crypto Key on disk for non encryption roots, so to be safe we 2450 * always take the slightly circuitous route of looking it up from 2451 * the encryption root's key. 2452 */ 2453 ret = dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj); 2454 if (ret != 0) 2455 goto error; 2456 2457 dsl_pool_config_enter(dp, FTAG); 2458 2459 ret = dsl_dir_hold_obj(dp, rddobj, NULL, FTAG, &rdd); 2460 if (ret != 0) 2461 goto error_unlock; 2462 2463 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2464 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &format); 2465 if (ret != 0) 2466 goto error_unlock; 2467 2468 if (format == ZFS_KEYFORMAT_PASSPHRASE) { 2469 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2470 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters); 2471 if (ret != 0) 2472 goto error_unlock; 2473 2474 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2475 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt); 2476 if (ret != 0) 2477 goto error_unlock; 2478 } 2479 2480 dsl_dir_rele(rdd, FTAG); 2481 dsl_pool_config_exit(dp, FTAG); 2482 2483 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, crypt); 2484 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_GUID, key_guid); 2485 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_VERSION, version); 2486 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2487 raw_keydata, MASTER_KEY_MAX_LEN)); 2488 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2489 raw_hmac_keydata, SHA512_HMAC_KEYLEN)); 2490 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_IV, iv, 2491 WRAPPING_IV_LEN)); 2492 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, mac, 2493 WRAPPING_MAC_LEN)); 2494 VERIFY0(nvlist_add_uint8_array(nvl, "portable_mac", 2495 os->os_phys->os_portable_mac, ZIO_OBJSET_MAC_LEN)); 2496 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), format); 2497 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), iters); 2498 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), salt); 2499 fnvlist_add_uint64(nvl, "mdn_checksum", mdn->dn_checksum); 2500 fnvlist_add_uint64(nvl, "mdn_compress", mdn->dn_compress); 2501 fnvlist_add_uint64(nvl, "mdn_nlevels", mdn->dn_nlevels); 2502 fnvlist_add_uint64(nvl, "mdn_blksz", mdn->dn_datablksz); 2503 fnvlist_add_uint64(nvl, "mdn_indblkshift", mdn->dn_indblkshift); 2504 fnvlist_add_uint64(nvl, "mdn_nblkptr", mdn->dn_nblkptr); 2505 fnvlist_add_uint64(nvl, "mdn_maxblkid", mdn->dn_maxblkid); 2506 fnvlist_add_uint64(nvl, "to_ivset_guid", to_ivset_guid); 2507 fnvlist_add_uint64(nvl, "from_ivset_guid", from_ivset_guid); 2508 2509 *nvl_out = nvl; 2510 return (0); 2511 2512 error_unlock: 2513 dsl_pool_config_exit(dp, FTAG); 2514 error: 2515 if (rdd != NULL) 2516 dsl_dir_rele(rdd, FTAG); 2517 nvlist_free(nvl); 2518 2519 *nvl_out = NULL; 2520 return (ret); 2521 } 2522 2523 uint64_t 2524 dsl_crypto_key_create_sync(uint64_t crypt, dsl_wrapping_key_t *wkey, 2525 dmu_tx_t *tx) 2526 { 2527 dsl_crypto_key_t dck; 2528 uint64_t version = ZIO_CRYPT_KEY_CURRENT_VERSION; 2529 uint64_t one = 1ULL; 2530 2531 ASSERT(dmu_tx_is_syncing(tx)); 2532 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 2533 ASSERT3U(crypt, >, ZIO_CRYPT_OFF); 2534 2535 /* create the DSL Crypto Key ZAP object */ 2536 dck.dck_obj = zap_create(tx->tx_pool->dp_meta_objset, 2537 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 2538 2539 /* fill in the key (on the stack) and sync it to disk */ 2540 dck.dck_wkey = wkey; 2541 VERIFY0(zio_crypt_key_init(crypt, &dck.dck_key)); 2542 2543 dsl_crypto_key_sync(&dck, tx); 2544 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, dck.dck_obj, 2545 DSL_CRYPTO_KEY_REFCOUNT, sizeof (uint64_t), 1, &one, tx)); 2546 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, dck.dck_obj, 2547 DSL_CRYPTO_KEY_VERSION, sizeof (uint64_t), 1, &version, tx)); 2548 2549 zio_crypt_key_destroy(&dck.dck_key); 2550 memset(&dck.dck_key, 0, sizeof (zio_crypt_key_t)); 2551 2552 return (dck.dck_obj); 2553 } 2554 2555 uint64_t 2556 dsl_crypto_key_clone_sync(dsl_dir_t *origindd, dmu_tx_t *tx) 2557 { 2558 objset_t *mos = tx->tx_pool->dp_meta_objset; 2559 2560 ASSERT(dmu_tx_is_syncing(tx)); 2561 2562 VERIFY0(zap_increment(mos, origindd->dd_crypto_obj, 2563 DSL_CRYPTO_KEY_REFCOUNT, 1, tx)); 2564 2565 return (origindd->dd_crypto_obj); 2566 } 2567 2568 void 2569 dsl_crypto_key_destroy_sync(uint64_t dckobj, dmu_tx_t *tx) 2570 { 2571 objset_t *mos = tx->tx_pool->dp_meta_objset; 2572 uint64_t refcnt; 2573 2574 /* Decrement the refcount, destroy if this is the last reference */ 2575 VERIFY0(zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_REFCOUNT, 2576 sizeof (uint64_t), 1, &refcnt)); 2577 2578 if (refcnt != 1) { 2579 VERIFY0(zap_increment(mos, dckobj, DSL_CRYPTO_KEY_REFCOUNT, 2580 -1, tx)); 2581 } else { 2582 VERIFY0(zap_destroy(mos, dckobj, tx)); 2583 } 2584 } 2585 2586 void 2587 dsl_dataset_crypt_stats(dsl_dataset_t *ds, nvlist_t *nv) 2588 { 2589 uint64_t intval; 2590 dsl_dir_t *dd = ds->ds_dir; 2591 dsl_dir_t *enc_root; 2592 char buf[ZFS_MAX_DATASET_NAME_LEN]; 2593 2594 if (dd->dd_crypto_obj == 0) 2595 return; 2596 2597 intval = dsl_dataset_get_keystatus(dd); 2598 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEYSTATUS, intval); 2599 2600 if (dsl_dir_get_crypt(dd, &intval) == 0) 2601 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_ENCRYPTION, intval); 2602 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2603 DSL_CRYPTO_KEY_GUID, 8, 1, &intval) == 0) { 2604 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEY_GUID, intval); 2605 } 2606 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2607 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &intval) == 0) { 2608 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEYFORMAT, intval); 2609 } 2610 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2611 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &intval) == 0) { 2612 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_PBKDF2_SALT, intval); 2613 } 2614 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2615 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &intval) == 0) { 2616 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_PBKDF2_ITERS, intval); 2617 } 2618 if (zap_lookup(dd->dd_pool->dp_meta_objset, ds->ds_object, 2619 DS_FIELD_IVSET_GUID, 8, 1, &intval) == 0) { 2620 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_IVSET_GUID, intval); 2621 } 2622 2623 if (dsl_dir_get_encryption_root_ddobj(dd, &intval) == 0) { 2624 if (dsl_dir_hold_obj(dd->dd_pool, intval, NULL, FTAG, 2625 &enc_root) == 0) { 2626 dsl_dir_name(enc_root, buf); 2627 dsl_dir_rele(enc_root, FTAG); 2628 dsl_prop_nvlist_add_string(nv, 2629 ZFS_PROP_ENCRYPTION_ROOT, buf); 2630 } 2631 } 2632 } 2633 2634 int 2635 spa_crypt_get_salt(spa_t *spa, uint64_t dsobj, uint8_t *salt) 2636 { 2637 int ret; 2638 dsl_crypto_key_t *dck = NULL; 2639 2640 /* look up the key from the spa's keystore */ 2641 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2642 if (ret != 0) 2643 goto error; 2644 2645 ret = zio_crypt_key_get_salt(&dck->dck_key, salt); 2646 if (ret != 0) 2647 goto error; 2648 2649 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2650 return (0); 2651 2652 error: 2653 if (dck != NULL) 2654 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2655 return (ret); 2656 } 2657 2658 /* 2659 * Objset blocks are a special case for MAC generation. These blocks have 2 2660 * 256-bit MACs which are embedded within the block itself, rather than a 2661 * single 128 bit MAC. As a result, this function handles encoding and decoding 2662 * the MACs on its own, unlike other functions in this file. 2663 */ 2664 int 2665 spa_do_crypt_objset_mac_abd(boolean_t generate, spa_t *spa, uint64_t dsobj, 2666 abd_t *abd, uint_t datalen, boolean_t byteswap) 2667 { 2668 int ret; 2669 dsl_crypto_key_t *dck = NULL; 2670 void *buf = abd_borrow_buf_copy(abd, datalen); 2671 objset_phys_t *osp = buf; 2672 uint8_t portable_mac[ZIO_OBJSET_MAC_LEN]; 2673 uint8_t local_mac[ZIO_OBJSET_MAC_LEN]; 2674 2675 /* look up the key from the spa's keystore */ 2676 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2677 if (ret != 0) 2678 goto error; 2679 2680 /* calculate both HMACs */ 2681 ret = zio_crypt_do_objset_hmacs(&dck->dck_key, buf, datalen, 2682 byteswap, portable_mac, local_mac); 2683 if (ret != 0) 2684 goto error; 2685 2686 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2687 2688 /* if we are generating encode the HMACs in the objset_phys_t */ 2689 if (generate) { 2690 memcpy(osp->os_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN); 2691 memcpy(osp->os_local_mac, local_mac, ZIO_OBJSET_MAC_LEN); 2692 abd_return_buf_copy(abd, buf, datalen); 2693 return (0); 2694 } 2695 2696 if (memcmp(portable_mac, osp->os_portable_mac, 2697 ZIO_OBJSET_MAC_LEN) != 0 || 2698 memcmp(local_mac, osp->os_local_mac, ZIO_OBJSET_MAC_LEN) != 0) { 2699 abd_return_buf(abd, buf, datalen); 2700 return (SET_ERROR(ECKSUM)); 2701 } 2702 2703 abd_return_buf(abd, buf, datalen); 2704 2705 return (0); 2706 2707 error: 2708 if (dck != NULL) 2709 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2710 abd_return_buf(abd, buf, datalen); 2711 return (ret); 2712 } 2713 2714 int 2715 spa_do_crypt_mac_abd(boolean_t generate, spa_t *spa, uint64_t dsobj, abd_t *abd, 2716 uint_t datalen, uint8_t *mac) 2717 { 2718 int ret; 2719 dsl_crypto_key_t *dck = NULL; 2720 uint8_t *buf = abd_borrow_buf_copy(abd, datalen); 2721 uint8_t digestbuf[ZIO_DATA_MAC_LEN]; 2722 2723 /* look up the key from the spa's keystore */ 2724 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2725 if (ret != 0) 2726 goto error; 2727 2728 /* perform the hmac */ 2729 ret = zio_crypt_do_hmac(&dck->dck_key, buf, datalen, 2730 digestbuf, ZIO_DATA_MAC_LEN); 2731 if (ret != 0) 2732 goto error; 2733 2734 abd_return_buf(abd, buf, datalen); 2735 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2736 2737 /* 2738 * Truncate and fill in mac buffer if we were asked to generate a MAC. 2739 * Otherwise verify that the MAC matched what we expected. 2740 */ 2741 if (generate) { 2742 memcpy(mac, digestbuf, ZIO_DATA_MAC_LEN); 2743 return (0); 2744 } 2745 2746 if (memcmp(digestbuf, mac, ZIO_DATA_MAC_LEN) != 0) 2747 return (SET_ERROR(ECKSUM)); 2748 2749 return (0); 2750 2751 error: 2752 if (dck != NULL) 2753 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2754 abd_return_buf(abd, buf, datalen); 2755 return (ret); 2756 } 2757 2758 /* 2759 * This function serves as a multiplexer for encryption and decryption of 2760 * all blocks (except the L2ARC). For encryption, it will populate the IV, 2761 * salt, MAC, and cabd (the ciphertext). On decryption it will simply use 2762 * these fields to populate pabd (the plaintext). 2763 */ 2764 int 2765 spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, const zbookmark_phys_t *zb, 2766 dmu_object_type_t ot, boolean_t dedup, boolean_t bswap, uint8_t *salt, 2767 uint8_t *iv, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd, 2768 boolean_t *no_crypt) 2769 { 2770 int ret; 2771 dsl_crypto_key_t *dck = NULL; 2772 uint8_t *plainbuf = NULL, *cipherbuf = NULL; 2773 2774 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION)); 2775 2776 /* look up the key from the spa's keystore */ 2777 ret = spa_keystore_lookup_key(spa, zb->zb_objset, FTAG, &dck); 2778 if (ret != 0) { 2779 ret = SET_ERROR(EACCES); 2780 return (ret); 2781 } 2782 2783 if (encrypt) { 2784 plainbuf = abd_borrow_buf_copy(pabd, datalen); 2785 cipherbuf = abd_borrow_buf(cabd, datalen); 2786 } else { 2787 plainbuf = abd_borrow_buf(pabd, datalen); 2788 cipherbuf = abd_borrow_buf_copy(cabd, datalen); 2789 } 2790 2791 /* 2792 * Both encryption and decryption functions need a salt for key 2793 * generation and an IV. When encrypting a non-dedup block, we 2794 * generate the salt and IV randomly to be stored by the caller. Dedup 2795 * blocks perform a (more expensive) HMAC of the plaintext to obtain 2796 * the salt and the IV. ZIL blocks have their salt and IV generated 2797 * at allocation time in zio_alloc_zil(). On decryption, we simply use 2798 * the provided values. 2799 */ 2800 if (encrypt && ot != DMU_OT_INTENT_LOG && !dedup) { 2801 ret = zio_crypt_key_get_salt(&dck->dck_key, salt); 2802 if (ret != 0) 2803 goto error; 2804 2805 ret = zio_crypt_generate_iv(iv); 2806 if (ret != 0) 2807 goto error; 2808 } else if (encrypt && dedup) { 2809 ret = zio_crypt_generate_iv_salt_dedup(&dck->dck_key, 2810 plainbuf, datalen, iv, salt); 2811 if (ret != 0) 2812 goto error; 2813 } 2814 2815 /* call lower level function to perform encryption / decryption */ 2816 ret = zio_do_crypt_data(encrypt, &dck->dck_key, ot, bswap, salt, iv, 2817 mac, datalen, plainbuf, cipherbuf, no_crypt); 2818 2819 /* 2820 * Handle injected decryption faults. Unfortunately, we cannot inject 2821 * faults for dnode blocks because we might trigger the panic in 2822 * dbuf_prepare_encrypted_dnode_leaf(), which exists because syncing 2823 * context is not prepared to handle malicious decryption failures. 2824 */ 2825 if (zio_injection_enabled && !encrypt && ot != DMU_OT_DNODE && ret == 0) 2826 ret = zio_handle_decrypt_injection(spa, zb, ot, ECKSUM); 2827 if (ret != 0) 2828 goto error; 2829 2830 if (encrypt) { 2831 abd_return_buf(pabd, plainbuf, datalen); 2832 abd_return_buf_copy(cabd, cipherbuf, datalen); 2833 } else { 2834 abd_return_buf_copy(pabd, plainbuf, datalen); 2835 abd_return_buf(cabd, cipherbuf, datalen); 2836 } 2837 2838 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2839 2840 return (0); 2841 2842 error: 2843 if (encrypt) { 2844 /* zero out any state we might have changed while encrypting */ 2845 memset(salt, 0, ZIO_DATA_SALT_LEN); 2846 memset(iv, 0, ZIO_DATA_IV_LEN); 2847 memset(mac, 0, ZIO_DATA_MAC_LEN); 2848 abd_return_buf(pabd, plainbuf, datalen); 2849 abd_return_buf_copy(cabd, cipherbuf, datalen); 2850 } else { 2851 abd_return_buf_copy(pabd, plainbuf, datalen); 2852 abd_return_buf(cabd, cipherbuf, datalen); 2853 } 2854 2855 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2856 2857 return (ret); 2858 } 2859 2860 ZFS_MODULE_PARAM(zfs, zfs_, disable_ivset_guid_check, INT, ZMOD_RW, 2861 "Set to allow raw receives without IVset guids"); 2862