1#!/bin/ksh -p 2 3# 4# CDDL HEADER START 5# 6# This file and its contents are supplied under the terms of the 7# Common Development and Distribution License ("CDDL"), version 1.0. 8# You may only use this file in accordance with the terms of version 9# 1.0 of the CDDL. 10# 11# A full copy of the text of the CDDL should have accompanied this 12# source. A copy of the CDDL is also available via the Internet at 13# http://www.illumos.org/license/CDDL. 14# 15# CDDL HEADER END 16# 17 18# 19# Copyright (c) 2017 by Intel Corporation. All rights reserved. 20# 21 22. $STF_SUITE/include/libtest.shlib 23. $STF_SUITE/tests/functional/fault/fault.cfg 24 25# 26# DESCRIPTION: 27# Testing Fault Management Agent ZED Logic - Automated Auto-Spare Test when 28# drive is faulted due to IO ERRORS. 29# 30# STRATEGY: 31# 1. Create a pool with hot spares 32# 2. Create a filesystem with the primary cache disable to force reads 33# 3. Write a file to the pool to be read back 34# 4. Inject IO ERRORS on read with a zinject error handler 35# 5. Verify the ZED kicks in a hot spare and expected pool/device status 36# 6. Clear the fault 37# 7. Verify the hot spare is available and expected pool/device status 38# 39 40verify_runnable "both" 41 42function cleanup 43{ 44 log_must zinject -c all 45 destroy_pool $TESTPOOL 46 rm -f $VDEV_FILES $SPARE_FILE 47} 48 49log_assert "Testing automated auto-spare FMA test" 50 51log_onexit cleanup 52 53# Clear events from previous runs 54zed_events_drain 55 56TESTFILE="/$TESTPOOL/$TESTFS/testfile" 57 58for type in "mirror" "raidz" "raidz2" "draid:1s"; do 59 if [ "$type" = "draid:1s" ]; then 60 # 1. Create a dRAID pool with a distributed hot spare 61 # 62 # Corruption is injected in the file-2 instead of file-1 63 # vdev since the dRAID permutation at these offsets maps 64 # to distributed spare space and not data devices. 65 # 66 log_must truncate -s $MINVDEVSIZE $VDEV_FILES 67 log_must zpool create -f $TESTPOOL $type $VDEV_FILES 68 SPARE="draid1-0-0" 69 FAULT="$TEST_BASE_DIR/file-2" 70 else 71 # 1. Create a pool with hot spares 72 log_must truncate -s $MINVDEVSIZE $VDEV_FILES $SPARE_FILE 73 log_must zpool create -f $TESTPOOL $type $VDEV_FILES \ 74 spare $SPARE_FILE 75 SPARE=$SPARE_FILE 76 FAULT=$FAULT_FILE 77 fi 78 79 # 2. Create a filesystem with the primary cache disable to force reads 80 log_must zfs create -o primarycache=none $TESTPOOL/$TESTFS 81 log_must zfs set recordsize=16k $TESTPOOL/$TESTFS 82 83 # 3. Write a file to the pool to be read back 84 log_must dd if=/dev/urandom of=$TESTFILE bs=1M count=64 85 86 # 4. Inject IO ERRORS on read with a zinject error handler 87 log_must zinject -d $FAULT -e io -T read $TESTPOOL 88 log_must cp $TESTFILE /dev/null 89 90 # 5. Verify the ZED kicks in a hot spare and expected pool/device status 91 log_note "Wait for ZED to auto-spare" 92 log_must wait_vdev_state $TESTPOOL $FAULT "FAULTED" 60 93 log_must wait_vdev_state $TESTPOOL $SPARE "ONLINE" 60 94 log_must wait_hotspare_state $TESTPOOL $SPARE "INUSE" 95 log_must check_state $TESTPOOL "" "DEGRADED" 96 97 # The ZED will use a sequential resilver for dRAID. Wait for the 98 # resilver and subsequent scrub to complete before moving on. 99 if [ "$type" = "draid:1s" ]; then 100 log_must wait_scrubbed $TESTPOOL 101 fi 102 103 # 6. Clear the fault 104 log_must zinject -c all 105 log_must zpool clear $TESTPOOL $FAULT 106 107 # 7. Verify the hot spare is available and expected pool/device status 108 log_must wait_vdev_state $TESTPOOL $FAULT "ONLINE" 60 109 log_must wait_hotspare_state $TESTPOOL $SPARE "AVAIL" 110 111 log_must is_pool_resilvered $TESTPOOL 112 log_must check_state $TESTPOOL "" "ONLINE" 113 114 cleanup 115done 116 117log_pass "Auto-spare test successful" 118