1#!/bin/ksh -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or https://opensource.org/licenses/CDDL-1.0.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright (c) 2023, Klara Inc.
25#
26
27# DESCRIPTION:
28#	Verify that vdev properties, slow_io_n and slow_io_t, work with ZED.
29#
30# STRATEGY:
31#	1. Create a pool with single vdev
32#	2. Set slow_io_n/slow_io_t to non-default values
33#	3. Inject slow io errors
34#	4. Verify that ZED degrades vdev
35#
36
37. $STF_SUITE/include/libtest.shlib
38
39TESTDIR="$TEST_BASE_DIR/zed_slow_io"
40VDEV="$TEST_BASE_DIR/vdevfile.$$"
41TESTPOOL="slow_io_pool"
42FILEPATH="$TESTDIR/slow_io.testfile"
43
44OLD_SLOW_IO=$(get_tunable ZIO_SLOW_IO_MS)
45OLD_SLOW_IO_EVENTS=$(get_tunable SLOW_IO_EVENTS_PER_SECOND)
46
47verify_runnable "both"
48
49function do_setup
50{
51	log_must truncate -s 1G $VDEV
52	default_setup_noexit $VDEV
53	zed_events_drain
54	log_must zfs set compression=off $TESTPOOL
55	log_must zfs set primarycache=none $TESTPOOL
56	log_must zfs set prefetch=none $TESTPOOL
57	log_must zfs set recordsize=512 $TESTPOOL
58	for i in {1..10}; do
59		dd if=/dev/urandom of=${FILEPATH}$i bs=512 count=1 2>/dev/null
60	done
61	zpool sync
62}
63
64# intermediate cleanup
65function do_clean
66{
67	log_must zinject -c all
68	log_must zpool destroy $TESTPOOL
69	log_must rm -f $VDEV
70}
71
72# final cleanup
73function cleanup
74{
75	log_must zinject -c all
76
77	# if pool still exists then something failed so log additional info
78	if poolexists $TESTPOOL ; then
79		log_note "$(zpool status -s $TESTPOOL)"
80		echo "=================== zed log search ==================="
81		grep "Diagnosis Engine" $ZEDLET_DIR/zed.log
82		destroy_pool $TESTPOOL
83	fi
84	log_must zed_stop
85
86	log_must rm -f $VDEV
87
88	log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
89	log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
90}
91
92function start_slow_io
93{
94	zpool sync
95	log_must set_tunable64 ZIO_SLOW_IO_MS 10
96	log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND 1000
97
98	log_must zinject -d $VDEV -D10:1 -T read $TESTPOOL
99	zpool sync
100}
101
102function stop_slow_io
103{
104	log_must set_tunable64 ZIO_SLOW_IO_MS $OLD_SLOW_IO
105	log_must set_tunable64 SLOW_IO_EVENTS_PER_SECOND $OLD_SLOW_IO_EVENTS
106
107	log_must zinject -c all
108}
109
110# Test default ZED settings:
111#    inject 10 events over 2.5 seconds, should not degrade.
112function default_degrade
113{
114	do_setup
115
116	start_slow_io
117	for i in {1..10}; do
118		dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
119		sleep 0.25
120	done
121	stop_slow_io
122	log_note "$(zpool status -s $TESTPOOL)"
123
124	# give slow ZED a chance to process the delay events
125	sleep 18
126	log_note "$(zpool status -s $TESTPOOL)"
127
128	degrades=$(grep "zpool_vdev_degrade" $ZEDLET_DIR/zed.log | wc -l)
129	log_note $degrades vdev degrades in ZED log
130	[ $degrades -eq "0" ] || \
131		log_fail "expecting no degrade events, found $degrades"
132
133	do_clean
134}
135
136# change slow_io_n, slow_io_t to 5 events in 60 seconds
137# fire more than 5 events, should degrade
138function slow_io_degrade
139{
140	do_setup
141
142	zpool set slow_io_n=5 $TESTPOOL $VDEV
143	zpool set slow_io_t=60 $TESTPOOL $VDEV
144
145	start_slow_io
146	for i in {1..16}; do
147		dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
148		sleep 0.5
149	done
150	stop_slow_io
151	zpool sync
152
153	#
154	# wait up to 60 seconds for kernel to produce at least 5 delay events
155	#
156	typeset -i i=0
157	typeset -i events=0
158	while [[ $i -lt 60 ]]; do
159		events=$(zpool events | grep "ereport\.fs\.zfs.delay" | wc -l)
160		[[ $events -ge "5" ]] && break
161		i=$((i+1))
162		sleep 1
163	done
164	log_note "$events delay events found"
165
166	if [[ $events -ge "5" ]]; then
167		log_must wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 10
168	fi
169
170	do_clean
171}
172
173# change slow_io_n, slow_io_t to 10 events in 1 second
174# inject events spaced 0.5 seconds apart, should not degrade
175function slow_io_no_degrade
176{
177	do_setup
178
179	zpool set slow_io_n=10 $TESTPOOL $VDEV
180	zpool set slow_io_t=1 $TESTPOOL $VDEV
181
182	start_slow_io
183	for i in {1..16}; do
184		dd if=${FILEPATH}$i of=/dev/null count=1 bs=512 2>/dev/null
185		sleep 0.5
186	done
187	stop_slow_io
188	zpool sync
189
190	log_mustnot wait_vdev_state $TESTPOOL $VDEV "DEGRADED" 45
191
192	do_clean
193}
194
195log_assert "Test ZED slow io configurability"
196log_onexit cleanup
197
198log_must zed_events_drain
199log_must zed_start
200
201default_degrade
202slow_io_degrade
203slow_io_no_degrade
204
205log_pass "Test ZED slow io configurability"
206