1#!/usr/local/bin/ksh93 -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright 2014 Spectra Logic.  All rights reserved.
25# Use is subject to license terms.
26#
27#
28. $STF_SUITE/tests/hotspare/hotspare.kshlib
29
30################################################################################
31#
32# __stc_assertion_start
33#
34# ID: zfs_hotspare_004_pos
35#
36# DESCRIPTION:
37#	If two vdevs get removed from a pool with two spares at the same time,
38#	both spares will be activated.
39#
40#
41# STRATEGY:
42#	1. Create 1 storage pool with two hot spares.
43#	2. Stop the zfsd process.
44#	3. Fault two vdevs
45#	4. Resume the zfsd process
46#	5. Verify that the spares are in use.
47#	6. Pause zfsd
48#	7. Clear the errors on the faulted vdevs
49#	8. Resume zfsd
50#	9. Verify that the vdevs ges resilvered and the spares get removed
51#
52# TESTABILITY: explicit
53#
54# TEST_AUTOMATION_LEVEL: automated
55#
56# CODING STATUS: COMPLETED (2014-05-13)
57#
58# __stc_assertion_end
59#
60###############################################################################
61
62verify_runnable "global"
63
64function cleanup
65{
66	poolexists $TESTPOOL && \
67		destroy_pool $TESTPOOL
68
69	partition_cleanup
70	$KILL -s SIGCONT `$PGREP zfsd`
71}
72
73function verify_assertion # spare_dev
74{
75	typeset err_dev1=${devarray[3]}
76	typeset err_dev2=${devarray[4]}
77	typeset sdev=$1
78
79	$KILL -s SIGSTOP `$PGREP zfsd`
80
81	log_must $ZINJECT -d $err_dev1 -A fault $TESTPOOL
82	log_must $ZINJECT -d $err_dev2 -A fault $TESTPOOL
83	log_must check_state $TESTPOOL $err_dev1 "FAULTED"
84	log_must check_state $TESTPOOL $err_dev2 "FAULTED"
85
86	$KILL -s SIGCONT `$PGREP zfsd`
87
88	# ZFSD can take up to 60 seconds to degrade an array in response to
89	# errors (though it's usually faster).
90	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
91		check_state $TESTPOOL "$sdev0" "INUSE"
92		cond1=$?
93		check_state $TESTPOOL "$sdev1" "INUSE"
94		cond2=$?
95		if [[ $cond1 -eq 0 && $cond2 -eq 0 ]]; then
96			break
97		fi
98		$SLEEP 6
99	done
100	log_must $ZPOOL status $TESTPOOL
101	log_must check_state $TESTPOOL "$sdev1" "INUSE"
102	log_must check_state $TESTPOOL "$sdev2" "INUSE"
103
104	$KILL -s SIGSTOP `$PGREP zfsd`
105	$ZPOOL clear $TESTPOOL
106	$KILL -s SIGCONT `$PGREP zfsd`
107
108	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
109		check_state $TESTPOOL "$sdev0" "AVAIL"
110		cond1=$?
111		check_state $TESTPOOL "$sdev1" "AVAIL"
112		cond2=$?
113		if [[ $cond1 -eq 0 && $cond2 -eq 0 ]]; then
114			break
115		fi
116		$SLEEP 6
117	done
118	log_must $ZPOOL status $TESTPOOL
119	log_must check_state $TESTPOOL "$sdev1" "AVAIL"
120	log_must check_state $TESTPOOL "$sdev2" "AVAIL"
121
122	# do cleanup
123	destroy_pool $TESTPOOL
124}
125
126
127log_assert "Two simultaneously faulted vdevs will be replaced by available spares"
128
129log_onexit cleanup
130
131ensure_zfsd_running
132set_devs
133typeset  sdev0="${devarray[0]}"
134typeset  sdev1="${devarray[1]}"
135
136set -A my_keywords "mirror" "raidz2"
137for keyword in "${my_keywords[@]}" ; do
138	setup_hotspares "$keyword"
139	verify_assertion $sdev
140done
141
142log_pass
143