1#!/bin/ksh -p
2#
3# CDDL HEADER START
4#
5# This file and its contents are supplied under the terms of the
6# Common Development and Distribution License ("CDDL"), version 1.0.
7# You may only use this file in accordance with the terms of version
8# 1.0 of the CDDL.
9#
10# A full copy of the text of the CDDL should have accompanied this
11# source.  A copy of the CDDL is also available via the Internet at
12# http://www.illumos.org/license/CDDL.
13#
14# CDDL HEADER END
15#
16
17#
18# Copyright (c) 2019 Datto, Inc. All rights reserved.
19# Copyright (c) 2022 Axcient.
20#
21
22. $STF_SUITE/include/libtest.shlib
23
24#
25# DESCRIPTION:
26# OpenZFS should be able to heal data using corrective recv when the send file
27#   was generated with the --large-block flag
28#
29# STRATEGY:
30# 0. Create a file, checksum the file to be corrupted then compare it's checksum
31#    with the one obtained after healing under different testing scenarios:
32# 1. Test healing (aka corrective) recv from a full send file
33# 2. Test healing recv (aka heal recv) from an incremental send file
34# 3. Test healing recv when compression on-disk is off but source was compressed
35# 4. Test heal recv when compression on-disk is on but source was uncompressed
36# 5. Test heal recv when compression doesn't match between send file and on-disk
37# 6. Test healing recv of an encrypted dataset using an unencrypted send file
38# 7. Test healing recv (on an encrypted dataset) using a raw send file
39# 8. Test healing when specifying destination filesystem only (no snapshot)
40# 9. Test incremental recv aftear healing recv
41#
42
43verify_runnable "both"
44
45DISK=${DISKS%% *}
46
47backup=$TEST_BASE_DIR/backup
48raw_backup=$TEST_BASE_DIR/raw_backup
49ibackup=$TEST_BASE_DIR/ibackup
50unc_backup=$TEST_BASE_DIR/unc_backup
51
52function cleanup
53{
54	log_must rm -f $backup $raw_backup $ibackup $unc_backup
55
56	poolexists $TESTPOOL && destroy_pool $TESTPOOL
57	log_must zpool create -f $TESTPOOL $DISK
58}
59
60function test_corrective_recv
61{
62	log_must zpool scrub -w $TESTPOOL
63	log_must zpool status -v $TESTPOOL
64	log_must eval "zpool status -v $TESTPOOL | \
65	    grep \"Permanent errors have been detected\""
66
67	# make sure we will read the corruption from disk by flushing the ARC
68	log_must zinject -a
69
70	log_must eval "zfs recv -c $1 < $2"
71
72	log_must zpool scrub -w $TESTPOOL
73	log_must zpool status -v $TESTPOOL
74	log_mustnot eval "zpool status -v $TESTPOOL | \
75	    grep \"Permanent errors have been detected\""
76	typeset cksum=$(md5digest $file)
77	[[ "$cksum" == "$checksum" ]] || \
78		log_fail "Checksums differ ($cksum != $checksum)"
79}
80
81log_onexit cleanup
82
83log_assert "ZFS corrective receive should be able to heal data corruption"
84
85typeset passphrase="password"
86typeset file="/$TESTPOOL/$TESTFS1/$TESTFILE0"
87
88log_must eval "poolexists $TESTPOOL && destroy_pool $TESTPOOL"
89log_must zpool create -f -o feature@large_blocks=enabled \
90    -o feature@head_errlog=disabled $TESTPOOL $DISK
91
92log_must eval "echo $passphrase > /$TESTPOOL/pwd"
93
94log_must zfs create -o recordsize=1m -o primarycache=none \
95    -o atime=off -o compression=lz4 $TESTPOOL/$TESTFS1
96
97log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
98log_must eval "echo 'aaaaaaaa' >> "$file
99typeset checksum=$(md5digest $file)
100
101log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
102
103# create full send file
104log_must eval "zfs send -L $TESTPOOL/$TESTFS1@snap1 > $backup"
105
106log_must dd if=/dev/urandom of=$file"1" bs=1024 count=1024 oflag=sync
107log_must eval "echo 'bbbbbbbb' >> "$file"1"
108log_must zfs snapshot $TESTPOOL/$TESTFS1@snap2
109# create incremental send file
110log_must eval "zfs send -Li $TESTPOOL/$TESTFS1@snap1 \
111    $TESTPOOL/$TESTFS1@snap2 > $ibackup"
112
113corrupt_blocks_at_level $file 0
114# test healing recv from a full send file
115test_corrective_recv $TESTPOOL/$TESTFS1@snap1 $backup
116
117corrupt_blocks_at_level $file"1" 0
118# test healing recv from an incremental send file
119test_corrective_recv $TESTPOOL/$TESTFS1@snap2 $ibackup
120
121# create new uncompressed dataset using our send file
122log_must eval "zfs recv -o compression=off -o primarycache=none \
123    $TESTPOOL/$TESTFS2 < $backup"
124typeset compr=$(get_prop compression $TESTPOOL/$TESTFS2)
125[[ "$compr" == "off" ]] || \
126	log_fail "Unexpected compression $compr in recved dataset"
127corrupt_blocks_at_level "/$TESTPOOL/$TESTFS2/$TESTFILE0" 0
128# test healing recv when compression on-disk is off but source was compressed
129test_corrective_recv "$TESTPOOL/$TESTFS2@snap1" $backup
130
131# create a full sendfile from an uncompressed source
132log_must eval "zfs send -L $TESTPOOL/$TESTFS2@snap1 > $unc_backup"
133log_must eval "zfs recv -o compression=gzip -o primarycache=none \
134    -o recordsize=1m $TESTPOOL/testfs3 < $unc_backup"
135typeset compr=$(get_prop compression $TESTPOOL/testfs3)
136[[ "$compr" == "gzip" ]] || \
137	log_fail "Unexpected compression $compr in recved dataset"
138corrupt_blocks_at_level "/$TESTPOOL/testfs3/$TESTFILE0" 0
139# test healing recv when compression on-disk is on but source was uncompressed
140test_corrective_recv "$TESTPOOL/testfs3@snap1" $unc_backup
141
142# create new compressed dataset using our send file
143log_must eval "zfs recv -o compression=gzip -o primarycache=none \
144    -o recordsize=1m $TESTPOOL/testfs4 < $backup"
145typeset compr=$(get_prop compression $TESTPOOL/testfs4)
146[[ "$compr" == "gzip" ]] || \
147	log_fail "Unexpected compression $compr in recved dataset"
148corrupt_blocks_at_level "/$TESTPOOL/testfs4/$TESTFILE0" 0
149# test healing recv when compression doesn't match between send file and on-disk
150test_corrective_recv "$TESTPOOL/testfs4@snap1" $backup
151
152# create new encrypted (and compressed) dataset using our send file
153log_must eval "zfs recv -o encryption=aes-256-ccm -o keyformat=passphrase \
154    -o recordsize=1m -o keylocation=file:///$TESTPOOL/pwd -o primarycache=none \
155    $TESTPOOL/testfs5 < $backup"
156typeset encr=$(get_prop encryption $TESTPOOL/testfs5)
157[[ "$encr" == "aes-256-ccm" ]] || \
158	log_fail "Unexpected encryption $encr in recved dataset"
159log_must eval "zfs send -L --raw $TESTPOOL/testfs5@snap1 > $raw_backup"
160log_must eval "zfs send -L $TESTPOOL/testfs5@snap1 > $backup"
161corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0
162# test healing recv of an encrypted dataset using an unencrypted send file
163test_corrective_recv "$TESTPOOL/testfs5@snap1" $backup
164corrupt_blocks_at_level "/$TESTPOOL/testfs5/$TESTFILE0" 0
165log_must zfs unmount $TESTPOOL/testfs5
166log_must zfs unload-key $TESTPOOL/testfs5
167# test healing recv (on an encrypted dataset) using a raw send file
168test_corrective_recv "$TESTPOOL/testfs5@snap1" $raw_backup
169# non raw send file healing an encrypted dataset with an unloaded key will fail
170log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap1 < $backup"
171
172log_must zfs rollback -r $TESTPOOL/$TESTFS1@snap1
173corrupt_blocks_at_level $file 0
174# test healing when specifying destination filesystem only (no snapshot)
175test_corrective_recv $TESTPOOL/$TESTFS1 $backup
176# test incremental recv aftear healing recv
177log_must eval "zfs recv -o recordsize=1m $TESTPOOL/$TESTFS1 < $ibackup"
178
179# test that healing recv can not be combined with incompatible recv options
180log_mustnot eval "zfs recv -h -c $TESTPOOL/$TESTFS1@snap1 < $backup"
181log_mustnot eval "zfs recv -F -c $TESTPOOL/$TESTFS1@snap1 < $backup"
182log_mustnot eval "zfs recv -s -c $TESTPOOL/$TESTFS1@snap1 < $backup"
183log_mustnot eval "zfs recv -u -c $TESTPOOL/$TESTFS1@snap1 < $backup"
184log_mustnot eval "zfs recv -d -c $TESTPOOL/$TESTFS1@snap1 < $backup"
185log_mustnot eval "zfs recv -e -c $TESTPOOL/$TESTFS1@snap1 < $backup"
186
187# ensure healing recv doesn't work when snap GUIDS don't match
188log_mustnot eval "zfs recv -c $TESTPOOL/testfs5@snap2 < $backup"
189log_mustnot eval "zfs recv -c $TESTPOOL/testfs5 < $backup"
190
191# test that healing recv doesn't work on non-existing snapshots
192log_mustnot eval "zfs recv -c $TESTPOOL/$TESTFS1@missing < $backup"
193
194log_pass "OpenZFS corrective recv works for data healing"
195