1#!/usr/local/bin/ksh93 -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23# $FreeBSD$
24
25#
26# Copyright 2014 Spectra Logic.  All rights reserved.
27# Use is subject to license terms.
28#
29#
30. $STF_SUITE/tests/hotspare/hotspare.kshlib
31
32################################################################################
33#
34# __stc_assertion_start
35#
36# ID: zfs_hotspare_004_pos
37#
38# DESCRIPTION:
39#	If two vdevs get removed from a pool with two spares at the same time,
40#	both spares will be activated.
41#
42#
43# STRATEGY:
44#	1. Create 1 storage pool with two hot spares.
45#	2. Stop the zfsd process.
46#	3. Fault two vdevs
47#	4. Resume the zfsd process
48#	5. Verify that the spares are in use.
49#	6. Pause zfsd
50#	7. Clear the errors on the faulted vdevs
51#	8. Resume zfsd
52#	9. Verify that the vdevs ges resilvered and the spares get removed
53#
54# TESTABILITY: explicit
55#
56# TEST_AUTOMATION_LEVEL: automated
57#
58# CODING STATUS: COMPLETED (2014-05-13)
59#
60# __stc_assertion_end
61#
62###############################################################################
63
64verify_runnable "global"
65
66function cleanup
67{
68	poolexists $TESTPOOL && \
69		destroy_pool $TESTPOOL
70
71	partition_cleanup
72	$KILL -s SIGCONT `$PGREP zfsd`
73}
74
75function verify_assertion # spare_dev
76{
77	typeset err_dev1=${devarray[3]}
78	typeset err_dev2=${devarray[4]}
79	typeset sdev=$1
80
81	$KILL -s SIGSTOP `$PGREP zfsd`
82
83	log_must $ZINJECT -d $err_dev1 -A fault $TESTPOOL
84	log_must $ZINJECT -d $err_dev2 -A fault $TESTPOOL
85	log_must check_state $TESTPOOL $err_dev1 "FAULTED"
86	log_must check_state $TESTPOOL $err_dev2 "FAULTED"
87
88	$KILL -s SIGCONT `$PGREP zfsd`
89
90	# ZFSD can take up to 60 seconds to degrade an array in response to
91	# errors (though it's usually faster).
92	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
93		check_state $TESTPOOL "$sdev0" "INUSE"
94		cond1=$?
95		check_state $TESTPOOL "$sdev1" "INUSE"
96		cond2=$?
97		if [[ $cond1 -eq 0 && $cond2 -eq 0 ]]; then
98			break
99		fi
100		$SLEEP 6
101	done
102	log_must $ZPOOL status $TESTPOOL
103	log_must check_state $TESTPOOL "$sdev1" "INUSE"
104	log_must check_state $TESTPOOL "$sdev2" "INUSE"
105
106	$KILL -s SIGSTOP `$PGREP zfsd`
107	$ZPOOL clear $TESTPOOL
108	$KILL -s SIGCONT `$PGREP zfsd`
109
110	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
111		check_state $TESTPOOL "$sdev0" "AVAIL"
112		cond1=$?
113		check_state $TESTPOOL "$sdev1" "AVAIL"
114		cond2=$?
115		if [[ $cond1 -eq 0 && $cond2 -eq 0 ]]; then
116			break
117		fi
118		$SLEEP 6
119	done
120	log_must $ZPOOL status $TESTPOOL
121	log_must check_state $TESTPOOL "$sdev1" "AVAIL"
122	log_must check_state $TESTPOOL "$sdev2" "AVAIL"
123
124	# do cleanup
125	destroy_pool $TESTPOOL
126}
127
128
129log_assert "Two simultaneously faulted vdevs will be replaced by available spares"
130
131log_onexit cleanup
132
133ensure_zfsd_running
134set_devs
135typeset  sdev0="${devarray[0]}"
136typeset  sdev1="${devarray[1]}"
137
138set -A my_keywords "mirror" "raidz2"
139for keyword in "${my_keywords[@]}" ; do
140	setup_hotspares "$keyword"
141	verify_assertion $sdev
142done
143
144log_pass
145