1#!/usr/local/bin/ksh93 -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23# $FreeBSD$
24
25#
26# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
27# Copyright 2014 Spectra Logic Corp.  All rights reserved.
28# Use is subject to license terms.
29#
30. $STF_SUITE/tests/hotspare/hotspare.kshlib
31
32################################################################################
33#
34# __stc_assertion_start
35#
36# ID: zfsd_hotspare_001_pos
37#
38# DESCRIPTION:
39#	If an active spare fails, it will be replaced by an available spare.
40#
41# STRATEGY:
42#	1. Create a storage pool with two hot spares
43#	2. Fail one vdev
44#	3. Verify that a spare gets activated
45#	4. Fail the spare
46#	5. Verify the failed spare was replaced by the other spare.
47#
48# TESTABILITY: explicit
49#
50# TEST_AUTOMATION_LEVEL: automated
51#
52# CODING STATUS: COMPLETED (2014-05-13)
53#
54# __stc_assertion_end
55#
56###############################################################################
57
58verify_runnable "global"
59
60function cleanup
61{
62	if poolexists $TESTPOOL ; then
63		destroy_pool $TESTPOOL
64	fi
65
66	partition_cleanup
67}
68
69
70log_onexit cleanup
71
72function verify_assertion # type
73{
74	typeset pool_type=$1
75
76	typeset err_dev=${devarray[3]}
77	typeset raidz2_dev="${devarray[4]}"
78	typeset mntp=$(get_prop mountpoint $TESTPOOL)
79
80	# fail a basic vdev
81	$ZINJECT -d $err_dev -A fault $TESTPOOL
82
83	# ZFSD can take up to 60 seconds to replace a failed device
84	# (though it's usually faster).
85	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
86		check_state $TESTPOOL "$fail_spare" "INUSE"
87		spare_inuse=$?
88		if [[ $spare_inuse == 0 ]]; then
89			break
90		fi
91		$SLEEP 6
92	done
93	log_must $ZPOOL status $TESTPOOL
94	log_must check_state $TESTPOOL "$fail_spare" "INUSE"
95
96	# The zpool history should log when a spare device becomes active
97	log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \
98	$GREP "spare in vdev=$fail_spare for vdev=$err_dev" > /dev/null
99
100	######################################################################
101	# Now fail the active hotspare, and check that the second comes online
102	######################################################################
103
104	# fail the spare vdev
105	$ZINJECT -d $fail_spare -A fault $TESTPOOL
106
107	# ZFSD can take up to 60 seconds to replace a failed device
108	# (though it's usually faster).
109	for ((timeout=0; $timeout<10; timeout=$timeout+1)); do
110		check_state $TESTPOOL "$standby_spare" "INUSE"
111		spare_inuse=$?
112		if [[ $spare_inuse == 0 ]]; then
113			break
114		fi
115		$SLEEP 6
116	done
117	log_must $ZPOOL status $TESTPOOL
118
119	# The standby spare should be in use, while the original spare should
120	# be faulted.
121	log_must check_state $TESTPOOL $standby_spare "online"
122	log_must check_state $TESTPOOL $standby_spare "INUSE"
123	log_mustnot check_state $TESTPOOL $fail_spare "online"
124
125	# The zpool history should log when a spare device becomes active
126	log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \
127		$GREP "spare in vdev=$standby_spare for vdev=$fail_spare" > /dev/null
128
129	# do cleanup
130	destroy_pool $TESTPOOL
131}
132
133log_onexit cleanup
134
135log_assert "An active damaged spare will be replaced by an available spare"
136
137ensure_zfsd_running
138set_devs
139
140typeset  fail_spare="${devarray[0]}"
141typeset  standby_spare="${devarray[1]}"
142typeset  spares="$fail_spare $standby_spare"
143
144set -A my_keywords "mirror" "raidz1" "raidz2"
145
146for keyword in "${my_keywords[@]}"; do
147	setup_hotspares "$keyword"
148	verify_assertion "$keyword"
149done
150
151log_pass "If one of the spare fail, the other available spare will be in use"
152