1#!/bin/ksh -p
2#
3# CDDL HEADER START
4#
5# This file and its contents are supplied under the terms of the
6# Common Development and Distribution License ("CDDL"), version 1.0.
7# You may only use this file in accordance with the terms of version
8# 1.0 of the CDDL.
9#
10# A full copy of the text of the CDDL should have accompanied this
11# source.  A copy of the CDDL is also available via the Internet at
12# http://www.illumos.org/license/CDDL.
13#
14# CDDL HEADER END
15#
16
17#
18# Copyright (c) 2019 Datto, Inc. All rights reserved.
19# Copyright (c) 2022 Axcient.
20#
21
22. $STF_SUITE/include/libtest.shlib
23
24#
25# DESCRIPTION:
26# OpenZFS should be able to heal data using corrective recv
27#
28# STRATEGY:
29# 0. Create a file, checksum the file to be corrupted then compare it's checksum
30#    with the one obtained after healing under different testing scenarios:
31# 1. Test healing (aka corrective) recv from a full send file
32# 2. Test healing recv (aka heal recv) from an incremental send file
33# 3. Test healing recv when compression on-disk is off but source was compressed
34# 4. Test heal recv when compression on-disk is on but source was uncompressed
35# 5. Test heal recv when compression doesn't match between send file and on-disk
36# 6. Test healing recv of an encrypted dataset using an unencrypted send file
37# 7. Test healing recv (on an encrypted dataset) using a raw send file
38# 8. Test healing when specifying destination filesystem only (no snapshot)
39# 9. Test incremental recv aftear healing recv
40#
41
42verify_runnable "both"
43
44DISK=${DISKS%% *}
45
46backup=$TEST_BASE_DIR/backup
47raw_backup=$TEST_BASE_DIR/raw_backup
48ibackup=$TEST_BASE_DIR/ibackup
49unc_backup=$TEST_BASE_DIR/unc_backup
50
51function cleanup
52{
53	log_must rm -f $backup $raw_backup $ibackup $unc_backup
54
55	poolexists $TESTPOOL && destroy_pool $TESTPOOL
56	log_must zpool create -f $TESTPOOL $DISK
57}
58
59function test_corrective_recv
60{
61	log_must zpool scrub -w $TESTPOOL
62	log_must zpool status -v $TESTPOOL
63	log_must eval "zpool status -v $TESTPOOL | \
64	    grep \"Permanent errors have been detected\""
65
66	# make sure we will read the corruption from disk by flushing the ARC
67	log_must zinject -a
68
69	log_must eval "zfs recv -c $1 < $2"
70
71	log_must zpool scrub -w $TESTPOOL
72	log_must zpool status -v $TESTPOOL
73	log_mustnot eval "zpool status -v $TESTPOOL | \
74	    grep \"Permanent errors have been detected\""
75	typeset cksum=$(md5digest $file)
76	[[ "$cksum" == "$checksum" ]] || \
77		log_fail "Checksums differ ($cksum != $checksum)"
78}
79
80log_onexit cleanup
81
82log_assert "ZFS corrective receive should be able to heal data corruption"
83
84typeset passphrase="password"
85typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0"
86
87log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL"
88log_must zpool create -f $TESTPOOL $DISK
89
90log_must eval "echo $passphrase > /$TESTPOOL/pwd"
91
92log_must zfs create -o primarycache=none \
93    -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1
94
95log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
96log_must eval "echo 'aaaaaaaa' >> "$file
97typeset checksum=$(md5digest $file)
98
99log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
100
101# create full send file
102log_must eval "zfs send $TESTPOOL/$TESTFS1@snap1 > $backup"
103
104log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync
105log_must eval "echo 'bbbbbbbb' >> "$file"1"
106log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2
107# create incremental send file
108log_must eval "zfs send -i $TESTPOOL/$TESTFS1@snap1 \
109    $TESTPOOL/$TESTFS1@snap2 > $ibackup"
110
111corrupt_blocks_at_level $file 0
112# test healing recv from a full send file
113test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup
114
115corrupt_blocks_at_level $file"1" 0
116# test healing recv from an incremental send file
117test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup
118
119# create new uncompressed dataset using our send file
120log_must eval "zfs recv -o compression=off -o primarycache=none \
121    $TESTPOOL/$TESTFS2 < $backup"
122typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2)
123[[ "$compr" == "off" ]] || \
124	log_fail "Unexpected compression $compr in recved dataset"
125corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0
126# test healing recv when compression on-disk is off but source was compressed
127test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup
128
129# create a full sendfile from an uncompressed source
130log_must eval "zfs send $TESTPOOL/$TESTFS2@snap1 > $unc_backup"
131log_must eval "zfs recv -o compression=gzip -o primarycache=none \
132    $TESTPOOL/testfs3 < $unc_backup"
133typeset compr=$(get_prop compression $TESTPOOL/testfs3)
134[[ "$compr" == "gzip" ]] || \
135	log_fail "Unexpected compression $compr in recved dataset"
136corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0
137# test healing recv when compression on-disk is on but source was uncompressed
138test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup
139
140# create new compressed dataset using our send file
141log_must eval "zfs recv -o compression=gzip -o primarycache=none \
142    $TESTPOOL/testfs4 < $backup"
143typeset compr=$(get_prop compression $TESTPOOL/testfs4)
144[[ "$compr" == "gzip" ]] || \
145	log_fail "Unexpected compression $compr in recved dataset"
146corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0
147# test healing recv when compression doesn't match between send file and on-disk
148test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup
149
150# create new encrypted (and compressed) dataset using our send file
151log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \
152    -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \
153    $TESTPOOL/testfs5 < $backup"
154typeset encr=$(get_prop encryption $TESTPOOL/testfs5)
155[[ "$encr" == "aes-256-ccm" ]] || \
156	log_fail "Unexpected encryption $encr in recved dataset"
157log_must eval "zfs send --raw $TESTPOOL/testfs5@snap1 > $raw_backup"
158log_must eval "zfs send $TESTPOOL/testfs5@snap1 > $backup"
159corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0
160# test healing recv of an encrypted dataset using an unencrypted send file
161test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup
162corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0
163log_must zfs unmount $TESTPOOL/testfs5
164log_must zfs unload-key $TESTPOOL/testfs5
165# test healing recv (on an encrypted dataset) using a raw send file
166# This is a special case since with unloaded keys we cannot report errors
167# in the filesystem.
168log_must zpool scrub -w $TESTPOOL
169log_must zpool status -v $TESTPOOL
170log_mustnot eval "zpool status -v $TESTPOOL | \
171    grep \"permission denied\""
172# make sure we will read the corruption from disk by flushing the ARC
173log_must zinject -a
174log_must eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $raw_backup"
175
176log_must zpool scrub -w $TESTPOOL
177log_must zpool status -v $TESTPOOL
178log_mustnot eval "zpool status -v $TESTPOOL | \
179    grep \"Permanent errors have been detected\""
180typeset cksum=$(md5digest $file)
181[[ "$cksum" == "$checksum" ]] || \
182	log_fail "Checksums differ ($cksum != $checksum)"
183
184# non raw send file healing an encrypted dataset with an unloaded key will fail
185log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup"
186
187log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1
188corrupt_blocks_at_level $file 0
189# test healing when specifying destination filesystem only (no snapshot)
190test_corrective_recv $TESTPOOL/$TESTFS1 $backup
191# test incremental recv aftear healing recv
192log_must eval "zfs recv $TESTPOOL/$TESTFS1 < $ibackup"
193
194# test that healing recv can not be combined with incompatible recv options
195log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup"
196log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup"
197log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup"
198log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup"
199log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup"
200log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup"
201
202# ensure healing recv doesn't work when snap GUIDS don't match
203log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup"
204log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup"
205
206# test that healing recv doesn't work on non-existing snapshots
207log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup"
208
209log_pass "OpenZFS corrective recv works for data healing"
210