1#!/bin/ksh -p 2# 3# CDDL HEADER START 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14# CDDL HEADER END 15# 16 17# 18# Copyright (c) 2019 Datto, Inc. All rights reserved. 19# Copyright (c) 2022 Axcient. 20# 21 22. $STF_SUITE/include/libtest.shlib 23 24# 25# DESCRIPTION: 26# OpenZFS should be able to heal data using corrective recv 27# 28# STRATEGY: 29# 0. Create a file, checksum the file to be corrupted then compare it's checksum 30# with the one obtained after healing under different testing scenarios: 31# 1. Test healing (aka corrective) recv from a full send file 32# 2. Test healing recv (aka heal recv) from an incremental send file 33# 3. Test healing recv when compression on-disk is off but source was compressed 34# 4. Test heal recv when compression on-disk is on but source was uncompressed 35# 5. Test heal recv when compression doesn't match between send file and on-disk 36# 6. Test healing recv of an encrypted dataset using an unencrypted send file 37# 7. Test healing recv (on an encrypted dataset) using a raw send file 38# 8. Test healing when specifying destination filesystem only (no snapshot) 39# 9. Test incremental recv aftear healing recv 40# 41 42verify_runnable "both" 43 44DISK=${DISKS%% *} 45 46backup=$TEST_BASE_DIR/backup 47raw_backup=$TEST_BASE_DIR/raw_backup 48ibackup=$TEST_BASE_DIR/ibackup 49unc_backup=$TEST_BASE_DIR/unc_backup 50 51function cleanup 52{ 53 log_must rm -f $backup $raw_backup $ibackup $unc_backup 54 55 poolexists $TESTPOOL && destroy_pool $TESTPOOL 56 log_must zpool create -f $TESTPOOL $DISK 57} 58 59function test_corrective_recv 60{ 61 log_must zpool scrub -w $TESTPOOL 62 log_must zpool status -v $TESTPOOL 63 log_must eval "zpool status -v $TESTPOOL | \ 64 grep \"Permanent errors have been detected\"" 65 66 # make sure we will read the corruption from disk by flushing the ARC 67 log_must zinject -a 68 69 log_must eval "zfs recv -c $1 < $2" 70 71 log_must zpool scrub -w $TESTPOOL 72 log_must zpool status -v $TESTPOOL 73 log_mustnot eval "zpool status -v $TESTPOOL | \ 74 grep \"Permanent errors have been detected\"" 75 typeset cksum=$(md5digest $file) 76 [[ "$cksum" == "$checksum" ]] || \ 77 log_fail "Checksums differ ($cksum != $checksum)" 78} 79 80log_onexit cleanup 81 82log_assert "ZFS corrective receive should be able to heal data corruption" 83 84typeset passphrase="password" 85typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0" 86 87log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL" 88log_must zpool create -f -o feature@head_errlog=disabled $TESTPOOL $DISK 89 90log_must eval "echo $passphrase > /$TESTPOOL/pwd" 91 92log_must zfs create -o primarycache=none \ 93 -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1 94 95log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync 96log_must eval "echo 'aaaaaaaa' >> "$file 97typeset checksum=$(md5digest $file) 98 99log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1 100 101# create full send file 102log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 > $backup" 103 104log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync 105log_must eval "echo 'bbbbbbbb' >> "$file"1" 106log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2 107# create incremental send file 108log_must eval "zfs send -i $TESTPOOL/$TESTFS1@snap1 \ 109 $TESTPOOL/$TESTFS1@snap2 > $ibackup" 110 111corrupt_blocks_at_level $file 0 112# test healing recv from a full send file 113test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup 114 115corrupt_blocks_at_level $file"1" 0 116# test healing recv from an incremental send file 117test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup 118 119# create new uncompressed dataset using our send file 120log_must eval "zfs recv -o compression=off -o primarycache=none \ 121 $TESTPOOL/$TESTFS2 < $backup" 122typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2) 123[[ "$compr" == "off" ]] || \ 124 log_fail "Unexpected compression $compr in recved dataset" 125corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0 126# test healing recv when compression on-disk is off but source was compressed 127test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup 128 129# create a full sendfile from an uncompressed source 130log_must eval "zfs send $TESTPOOL/$TESTFS2@snap1 > $unc_backup" 131log_must eval "zfs recv -o compression=gzip -o primarycache=none \ 132 $TESTPOOL/testfs3 < $unc_backup" 133typeset compr=$(get_prop compression $TESTPOOL/testfs3) 134[[ "$compr" == "gzip" ]] || \ 135 log_fail "Unexpected compression $compr in recved dataset" 136corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0 137# test healing recv when compression on-disk is on but source was uncompressed 138test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup 139 140# create new compressed dataset using our send file 141log_must eval "zfs recv -o compression=gzip -o primarycache=none \ 142 $TESTPOOL/testfs4 < $backup" 143typeset compr=$(get_prop compression $TESTPOOL/testfs4) 144[[ "$compr" == "gzip" ]] || \ 145 log_fail "Unexpected compression $compr in recved dataset" 146corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0 147# test healing recv when compression doesn't match between send file and on-disk 148test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup 149 150# create new encrypted (and compressed) dataset using our send file 151log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \ 152 -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \ 153 $TESTPOOL/testfs5 < $backup" 154typeset encr=$(get_prop encryption $TESTPOOL/testfs5) 155[[ "$encr" == "aes-256-ccm" ]] || \ 156 log_fail "Unexpected encryption $encr in recved dataset" 157log_must eval "zfs send --raw $TESTPOOL/testfs5@snap1 > $raw_backup" 158log_must eval "zfs send $TESTPOOL/testfs5@snap1 > $backup" 159corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 160# test healing recv of an encrypted dataset using an unencrypted send file 161test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup 162corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0 163log_must zfs unmount $TESTPOOL/testfs5 164log_must zfs unload-key $TESTPOOL/testfs5 165# test healing recv (on an encrypted dataset) using a raw send file 166test_corrective_recv "$TESTPOOL/testfs5@snap1" $raw_backup 167# non raw send file healing an encrypted dataset with an unloaded key will fail 168log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup" 169 170log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1 171corrupt_blocks_at_level $file 0 172# test healing when specifying destination filesystem only (no snapshot) 173test_corrective_recv $TESTPOOL/$TESTFS1 $backup 174# test incremental recv aftear healing recv 175log_must eval "zfs recv $TESTPOOL/$TESTFS1 < $ibackup" 176 177# test that healing recv can not be combined with incompatible recv options 178log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup" 179log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup" 180log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup" 181log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup" 182log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup" 183log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup" 184 185# ensure healing recv doesn't work when snap GUIDS don't match 186log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup" 187log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup" 188 189# test that healing recv doesn't work on non-existing snapshots 190log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup" 191 192log_pass "OpenZFS corrective recv works for data healing" 193