1#!/usr/local/bin/ksh93 -p 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# $FreeBSD$ 24 25# 26# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 27# Copyright 2014 Spectra Logic Corp. All rights reserved. 28# Use is subject to license terms. 29# 30. $STF_SUITE/tests/hotspare/hotspare.kshlib 31 32################################################################################ 33# 34# __stc_assertion_start 35# 36# ID: zfsd_hotspare_001_pos 37# 38# DESCRIPTION: 39# If an active spare fails, it will be replaced by an available spare. 40# 41# STRATEGY: 42# 1. Create a storage pool with two hot spares 43# 2. Fail one vdev 44# 3. Verify that a spare gets activated 45# 4. Fail the spare 46# 5. Verify the failed spare was replaced by the other spare. 47# 48# TESTABILITY: explicit 49# 50# TEST_AUTOMATION_LEVEL: automated 51# 52# CODING STATUS: COMPLETED (2014-05-13) 53# 54# __stc_assertion_end 55# 56############################################################################### 57 58verify_runnable "global" 59 60function cleanup 61{ 62 if poolexists $TESTPOOL ; then 63 destroy_pool $TESTPOOL 64 fi 65 66 partition_cleanup 67} 68 69 70log_onexit cleanup 71 72function verify_assertion # type 73{ 74 typeset pool_type=$1 75 76 typeset err_dev=${devarray[3]} 77 typeset raidz2_dev="${devarray[4]}" 78 typeset mntp=$(get_prop mountpoint $TESTPOOL) 79 80 # fail a basic vdev 81 $ZINJECT -d $err_dev -A fault $TESTPOOL 82 83 # ZFSD can take up to 60 seconds to replace a failed device 84 # (though it's usually faster). 85 for ((timeout=0; $timeout<10; timeout=$timeout+1)); do 86 check_state $TESTPOOL "$fail_spare" "INUSE" 87 spare_inuse=$? 88 if [[ $spare_inuse == 0 ]]; then 89 break 90 fi 91 $SLEEP 6 92 done 93 log_must $ZPOOL status $TESTPOOL 94 log_must check_state $TESTPOOL "$fail_spare" "INUSE" 95 96 # The zpool history should log when a spare device becomes active 97 log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \ 98 $GREP "spare in vdev=$fail_spare for vdev=$err_dev" > /dev/null 99 100 ###################################################################### 101 # Now fail the active hotspare, and check that the second comes online 102 ###################################################################### 103 104 # fail the spare vdev 105 $ZINJECT -d $fail_spare -A fault $TESTPOOL 106 107 # ZFSD can take up to 60 seconds to replace a failed device 108 # (though it's usually faster). 109 for ((timeout=0; $timeout<10; timeout=$timeout+1)); do 110 check_state $TESTPOOL "$standby_spare" "INUSE" 111 spare_inuse=$? 112 if [[ $spare_inuse == 0 ]]; then 113 break 114 fi 115 $SLEEP 6 116 done 117 log_must $ZPOOL status $TESTPOOL 118 119 # The standby spare should be in use, while the original spare should 120 # be faulted. 121 log_must check_state $TESTPOOL $standby_spare "online" 122 log_must check_state $TESTPOOL $standby_spare "INUSE" 123 log_mustnot check_state $TESTPOOL $fail_spare "online" 124 125 # The zpool history should log when a spare device becomes active 126 log_must $ZPOOL history -i $TESTPOOL | $GREP "internal vdev attach" | \ 127 $GREP "spare in vdev=$standby_spare for vdev=$fail_spare" > /dev/null 128 129 # do cleanup 130 destroy_pool $TESTPOOL 131} 132 133log_onexit cleanup 134 135log_assert "An active damaged spare will be replaced by an available spare" 136 137ensure_zfsd_running 138set_devs 139 140typeset fail_spare="${devarray[0]}" 141typeset standby_spare="${devarray[1]}" 142typeset spares="$fail_spare $standby_spare" 143 144set -A my_keywords "mirror" "raidz1" "raidz2" 145 146for keyword in "${my_keywords[@]}"; do 147 setup_hotspares "$keyword" 148 verify_assertion "$keyword" 149done 150 151log_pass "If one of the spare fail, the other available spare will be in use" 152