1#!/bin/ksh -p 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or https://opensource.org/licenses/CDDL-1.0. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright (c) 2023, Klara Inc. 25# 26 27# DESCRIPTION: 28# Verify that vdev properties, slow_io_n and slow_io_t, work with ZED. 29# 30# STRATEGY: 31# 1. Create a pool with single vdev 32# 2. Set slow_io_n/slow_io_t to non-default values 33# 3. Inject slow io errors 34# 4. Verify that ZED degrades vdev 35# 36 37. $STF_SUITE/include/libtest.shlib 38 39TESTDIR="$TEST_BASE_DIR/zed_slow_io" 40VDEV="$TEST_BASE_DIR/vdevfile.$$" 41TESTPOOL="slow_io_pool" 42FILEPATH="$TESTDIR/slow_io.testfile" 43 44OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS) 45OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND) 46 47verify_runnable "both" 48 49function do_setup 50{ 51 log_must truncate -s 1G $VDEV 52 default_setup_noexit $VDEV 53 zed_events_drain 54 log_must zfs set compression=off $TESTPOOL 55 log_must zfs set primarycache=none $TESTPOOL 56 log_must zfs set prefetch=none $TESTPOOL 57 log_must zfs set recordsize=512 $TESTPOOL 58 for i in {1..10}; do 59 dd if=/dev/urandom of=${FILEPATH}$i bs=512 count=1 2>/dev/null 60 done 61 zpool sync 62} 63 64# intermediate cleanup 65function do_clean 66{ 67 log_must zinject -c all 68 log_must zpool destroy $TESTPOOL 69 log_must rm -f $VDEV 70} 71 72# final cleanup 73function cleanup 74{ 75 log_must zinject -c all 76 77 # if pool still exists then something failed so log additional info 78 if poolexists $TESTPOOL ; then 79 log_note "$(zpool status -s $TESTPOOL)" 80 echo "=================== zed log search ===================" 81 grep "Diagnosis Engine" $ZEDLET_DIR/zed.log 82 destroy_pool $TESTPOOL 83 fi 84 log_must zed_stop 85 86 log_must rm -f $VDEV 87 88 log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO 89 log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS 90} 91 92function start_slow_io 93{ 94 zpool sync 95 log_must set_tunable64 ZIO_SLOW_IO_MS 10 96 log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000 97 98 log_must zinject -d $VDEV -D10:1 -T read $TESTPOOL 99 zpool sync 100} 101 102function stop_slow_io 103{ 104 log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO 105 log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS 106 107 log_must zinject -c all 108} 109 110# Test default ZED settings: 111# inject 10 events over 2.5 seconds, should not degrade. 112function default_degrade 113{ 114 do_setup 115 116 start_slow_io 117 for i in {1..10}; do 118 dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null 119 sleep 0.25 120 done 121 stop_slow_io 122 log_note "$(zpool status -s $TESTPOOL)" 123 124 # give slow ZED a chance to process the delay events 125 sleep 18 126 log_note "$(zpool status -s $TESTPOOL)" 127 128 degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log | wc -l) 129 log_note $degrades vdev degrades in ZED log 130 [ $degrades -eq "0" ] || \ 131 log_fail "expecting no degrade events, found $degrades" 132 133 do_clean 134} 135 136# change slow_io_n, slow_io_t to 5 events in 60 seconds 137# fire more than 5 events, should degrade 138function slow_io_degrade 139{ 140 do_setup 141 142 zpool set slow_io_n=5 $TESTPOOL $VDEV 143 zpool set slow_io_t=60 $TESTPOOL $VDEV 144 145 start_slow_io 146 for i in {1..16}; do 147 dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null 148 sleep 0.5 149 done 150 stop_slow_io 151 zpool sync 152 153 # 154 # wait up to 60 seconds for kernel to produce at least 5 delay events 155 # 156 typeset -i i=0 157 typeset -i events=0 158 while [[ $i -lt 60 ]]; do 159 events=$(zpool events | grep "ereport\.fs\.zfs.delay" | wc -l) 160 [[ $events -ge "5" ]] && break 161 i=$((i+1)) 162 sleep 1 163 done 164 log_note "$events delay events found" 165 166 if [[ $events -ge "5" ]]; then 167 log_must wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 10 168 fi 169 170 do_clean 171} 172 173# change slow_io_n, slow_io_t to 10 events in 1 second 174# inject events spaced 0.5 seconds apart, should not degrade 175function slow_io_no_degrade 176{ 177 do_setup 178 179 zpool set slow_io_n=10 $TESTPOOL $VDEV 180 zpool set slow_io_t=1 $TESTPOOL $VDEV 181 182 start_slow_io 183 for i in {1..16}; do 184 dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null 185 sleep 0.5 186 done 187 stop_slow_io 188 zpool sync 189 190 log_mustnot wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 45 191 192 do_clean 193} 194 195log_assert "Test ZED slow io configurability" 196log_onexit cleanup 197 198log_must zed_events_drain 199log_must zed_start 200 201default_degrade 202slow_io_degrade 203slow_io_no_degrade 204 205log_pass "Test ZED slow io configurability" 206