1#!/bin/ksh -p
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
25# Use is subject to license terms.
26#
27
28# DESCRIPTION:
29#	Verify zio deadman detects a hung zio
30#
31# STRATEGY:
32#	1. Reduce the zfs_deadman_ziotime_ms to 5s.
33#	2. Reduce the zfs_deadman_checktime_ms to 1s.
34#	3. Inject a 10s zio delay to force long IOs.
35#	4. Read an uncached file in the background.
36#	5. Verify a "deadman" event is posted.
37#	6. Inject a 100ms zio delay which is under the 5s allowed.
38#	7. Read an uncached file in the background.
39#	8. Verify a "deadman" event is not posted.
40#
41
42. $STF_SUITE/include/libtest.shlib
43. $STF_SUITE/tests/functional/deadman/deadman.cfg
44
45verify_runnable "both"
46
47function cleanup
48{
49	log_must zinject -c all
50	default_cleanup_noexit
51
52	log_must set_tunable64 DEADMAN_ZIOTIME_MS $ZIOTIME_DEFAULT
53	log_must set_tunable64 DEADMAN_CHECKTIME_MS $CHECKTIME_DEFAULT
54	log_must set_tunable64 DEADMAN_FAILMODE $FAILMODE_DEFAULT
55}
56
57log_assert "Verify zio deadman detects a hung zio"
58log_onexit cleanup
59
60# 1. Reduce the zfs_deadman_ziotime_ms to 5s.
61log_must set_tunable64 DEADMAN_ZIOTIME_MS 5000
62# 2. Reduce the zfs_deadman_checktime_ms to 1s.
63log_must set_tunable64 DEADMAN_CHECKTIME_MS 1000
64log_must set_tunable64 DEADMAN_FAILMODE "wait"
65
66# Create a new pool in order to use the updated deadman settings.
67default_setup_noexit $DISK1
68
69# Write a file and export/import the pool to clear the ARC.
70mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
71log_must file_write -b 1048576 -c 8 -o create -d 0 -f $mntpnt/file1
72log_must file_write -b 1048576 -c 8 -o create -d 0 -f $mntpnt/file2
73log_must zpool export $TESTPOOL
74log_must zpool import $TESTPOOL
75log_must stat -t /$mntpnt/file1
76log_must stat -t /$mntpnt/file2
77
78# 3. Inject a 10s zio delay to force long IOs and serialize them..
79log_must zpool events -c
80log_must zinject -d $DISK1 -D10000:1 $TESTPOOL
81
82# 4. Read an uncached file in the background, it's expected to hang.
83log_must eval "dd if=/$mntpnt/file1 of=/dev/null bs=1048576 &"
84sleep 10
85log_must zinject -c all
86log_must zpool sync
87wait
88
89# 5. Verify a "deadman" event is posted.  The first appears after 5
90# seconds, and another each second thereafter until the delay is cleared.
91events=$(zpool events | grep -c ereport.fs.zfs.deadman)
92if [ "$events" -lt 1 ]; then
93	log_fail "Expect >=1 deadman events, $events found"
94fi
95
96# 6. Inject a 100ms zio delay which is under the 5s allowed, allow them
97# to run concurrently so they don't get starved in the queue.
98log_must zpool events -c
99log_must zinject -d $DISK1 -D100:10 $TESTPOOL
100
101# 7. Read an uncached file in the background.
102log_must eval "dd if=/$mntpnt/file2 of=/dev/null bs=1048576 &"
103sleep 10
104log_must zinject -c all
105wait
106
107# 8. Verify a "deadman" event is not posted.
108events=$(zpool events | grep -c ereport.fs.zfs.deadman)
109if [ "$events" -ne 0 ]; then
110	log_fail "Expect 0 deadman events, $events found"
111fi
112
113log_pass "Verify zio deadman detected a hung zio and $events deadman events"
114