1#!/bin/ksh -p
2
3#
4# CDDL HEADER START
5#
6# This file and its contents are supplied under the terms of the
7# Common Development and Distribution License ("CDDL"), version 1.0.
8# You may only use this file in accordance with the terms of version
9# 1.0 of the CDDL.
10#
11# A full copy of the text of the CDDL should have accompanied this
12# source.  A copy of the CDDL is also available via the Internet at
13# http://www.illumos.org/license/CDDL.
14#
15# CDDL HEADER END
16#
17
18#
19# Copyright (c) 2017 by Intel Corporation. All rights reserved.
20#
21
22. $STF_SUITE/include/libtest.shlib
23. $STF_SUITE/tests/functional/fault/fault.cfg
24
25#
26# DESCRIPTION:
27# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when
28# drive is faulted due to IO ERRORS.
29#
30# STRATEGY:
31# 1. Create a pool with hot spares
32# 2. Create a filesystem with the primary cache disable to force reads
33# 3. Write a file to the pool to be read back
34# 4. Inject IO ERRORS on read with a zinject error handler
35# 5. Verify the ZED kicks in a hot spare and expected pool/device status
36# 6. Clear the fault
37# 7. Verify the hot spare is available and expected pool/device status
38#
39
40verify_runnable "both"
41
42function cleanup
43{
44	log_must zinject -c all
45	destroy_pool $TESTPOOL
46	rm -f $VDEV_FILES $SPARE_FILE
47}
48
49log_assert "Testing automated auto-spare FMA test"
50
51log_onexit cleanup
52
53# Clear events from previous runs
54zed_events_drain
55
56TESTFILE="/$TESTPOOL/$TESTFS/testfile"
57
58for type in "mirror" "raidz" "raidz2" "draid:1s"; do
59	if [ "$type" = "draid:1s" ]; then
60		# 1. Create a dRAID pool with a distributed hot spare
61		#
62		# Corruption is injected in the file-2 instead of file-1
63		# vdev since the dRAID permutation at these offsets maps
64		# to distributed spare space and not data devices.
65		#
66		log_must truncate -s $MINVDEVSIZE $VDEV_FILES
67		log_must zpool create -f $TESTPOOL $type $VDEV_FILES
68		SPARE="draid1-0-0"
69		FAULT="$TEST_BASE_DIR/file-2"
70	else
71		# 1. Create a pool with hot spares
72		log_must truncate -s $MINVDEVSIZE $VDEV_FILES $SPARE_FILE
73		log_must zpool create -f $TESTPOOL $type $VDEV_FILES \
74		    spare $SPARE_FILE
75		SPARE=$SPARE_FILE
76		FAULT=$FAULT_FILE
77	fi
78
79	# 2. Create a filesystem with the primary cache disable to force reads
80	log_must zfs create -o primarycache=none $TESTPOOL/$TESTFS
81	log_must zfs set recordsize=16k $TESTPOOL/$TESTFS
82
83	# 3. Write a file to the pool to be read back
84	log_must dd if=/dev/urandom of=$TESTFILE bs=1M count=64
85
86	# 4. Inject IO ERRORS on read with a zinject error handler
87	log_must zinject -d $FAULT -e io -T read $TESTPOOL
88	log_must cp $TESTFILE /dev/null
89
90	# 5. Verify the ZED kicks in a hot spare and expected pool/device status
91	log_note "Wait for ZED to auto-spare"
92	log_must wait_vdev_state $TESTPOOL $FAULT "FAULTED" 60
93	log_must wait_vdev_state $TESTPOOL $SPARE "ONLINE" 60
94	log_must wait_hotspare_state $TESTPOOL $SPARE "INUSE"
95	log_must check_state $TESTPOOL "" "DEGRADED"
96
97	# The ZED will use a sequential resilver for dRAID. Wait for the
98	# resilver and subsequent scrub to complete before moving on.
99	if [ "$type" = "draid:1s" ]; then
100		log_must wait_scrubbed $TESTPOOL
101	fi
102
103	# 6. Clear the fault
104	log_must zinject -c all
105	log_must zpool clear $TESTPOOL $FAULT
106
107	# 7. Verify the hot spare is available and expected pool/device status
108	log_must wait_vdev_state $TESTPOOL $FAULT "ONLINE" 60
109	log_must wait_hotspare_state $TESTPOOL $SPARE "AVAIL"
110
111	log_must is_pool_resilvered $TESTPOOL
112	log_must check_state $TESTPOOL "" "ONLINE"
113
114	cleanup
115done
116
117log_pass "Auto-spare test successful"
118