1eda14cbcSMatt Macy#!/bin/ksh -p 2eda14cbcSMatt Macy 3eda14cbcSMatt Macy# 4eda14cbcSMatt Macy# CDDL HEADER START 5eda14cbcSMatt Macy# 6eda14cbcSMatt Macy# This file and its contents are supplied under the terms of the 7eda14cbcSMatt Macy# Common Development and Distribution License ("CDDL"), version 1.0. 8eda14cbcSMatt Macy# You may only use this file in accordance with the terms of version 9eda14cbcSMatt Macy# 1.0 of the CDDL. 10eda14cbcSMatt Macy# 11eda14cbcSMatt Macy# A full copy of the text of the CDDL should have accompanied this 12eda14cbcSMatt Macy# source. A copy of the CDDL is also available via the Internet at 13eda14cbcSMatt Macy# http://www.illumos.org/license/CDDL. 14eda14cbcSMatt Macy# 15eda14cbcSMatt Macy# CDDL HEADER END 16eda14cbcSMatt Macy# 17eda14cbcSMatt Macy 18eda14cbcSMatt Macy# 19eda14cbcSMatt Macy# Copyright (c) 2019, Datto Inc. All rights reserved. 20eda14cbcSMatt Macy# 21eda14cbcSMatt Macy 22eda14cbcSMatt Macy. $STF_SUITE/include/libtest.shlib 23eda14cbcSMatt Macy. $STF_SUITE/tests/functional/replacement/replacement.cfg 24eda14cbcSMatt Macy 25eda14cbcSMatt Macy# 26eda14cbcSMatt Macy# DESCRIPTION: 27eda14cbcSMatt Macy# Testing resilver restart logic both with and without the deferred resilver 28eda14cbcSMatt Macy# feature enabled, verifying that resilver is not restarted when it is 29eda14cbcSMatt Macy# unnecessary. 30eda14cbcSMatt Macy# 31eda14cbcSMatt Macy# STRATEGY: 32eda14cbcSMatt Macy# 1. Create a pool 33eda14cbcSMatt Macy# 2. Create four filesystems with the primary cache disable to force reads 34eda14cbcSMatt Macy# 3. Write four files simultaneously, one to each filesystem 35eda14cbcSMatt Macy# 4. Do with and without deferred resilvers enabled 36eda14cbcSMatt Macy# a. Replace a vdev with a spare & suspend resilver immediately 37eda14cbcSMatt Macy# b. Verify resilver starts properly 38eda14cbcSMatt Macy# c. Offline / online another vdev to introduce a new DTL range 3916038816SMartin Matuska# d. Verify resilver restart or defer 40eda14cbcSMatt Macy# e. Inject read errors on vdev that was offlined / onlned 41eda14cbcSMatt Macy# f. Verify that resilver did not restart 42eda14cbcSMatt Macy# g. Unsuspend resilver and wait for it to finish 43eda14cbcSMatt Macy# h. Verify that there are two resilvers and nothing is deferred 44eda14cbcSMatt Macy# 45eda14cbcSMatt Macy 46eda14cbcSMatt Macyfunction cleanup 47eda14cbcSMatt Macy{ 48eda14cbcSMatt Macy log_must set_tunable32 RESILVER_MIN_TIME_MS $ORIG_RESILVER_MIN_TIME 49eda14cbcSMatt Macy log_must set_tunable32 SCAN_SUSPEND_PROGRESS \ 50eda14cbcSMatt Macy $ORIG_SCAN_SUSPEND_PROGRESS 51eda14cbcSMatt Macy log_must set_tunable32 ZEVENT_LEN_MAX $ORIG_ZFS_ZEVENT_LEN_MAX 52eda14cbcSMatt Macy log_must zinject -c all 53eda14cbcSMatt Macy destroy_pool $TESTPOOL1 54eda14cbcSMatt Macy rm -f ${VDEV_FILES[@]} $SPARE_VDEV_FILE 55eda14cbcSMatt Macy} 56eda14cbcSMatt Macy 57eda14cbcSMatt Macy# count resilver events in zpool and number of deferred rsilvers on vdevs 58eda14cbcSMatt Macyfunction verify_restarts # <msg> <cnt> <defer> 59eda14cbcSMatt Macy{ 60eda14cbcSMatt Macy msg=$1 61eda14cbcSMatt Macy cnt=$2 62eda14cbcSMatt Macy defer=$3 63eda14cbcSMatt Macy 64eda14cbcSMatt Macy # check the number of resilver start in events log 65eda14cbcSMatt Macy RESILVERS=$(zpool events | grep -c sysevent.fs.zfs.resilver_start) 66eda14cbcSMatt Macy log_note "expected $cnt resilver start(s)$msg, found $RESILVERS" 67eda14cbcSMatt Macy [[ "$RESILVERS" -ne "$cnt" ]] && 68eda14cbcSMatt Macy log_fail "expected $cnt resilver start(s)$msg, found $RESILVERS" 69eda14cbcSMatt Macy 70eda14cbcSMatt Macy [[ -z "$defer" ]] && return 71eda14cbcSMatt Macy 72eda14cbcSMatt Macy # use zdb to find which vdevs have the resilver defer flag 73eda14cbcSMatt Macy VDEV_DEFERS=$(zdb -C $TESTPOOL1 | awk ' 74eda14cbcSMatt Macy /children/ { gsub(/[^0-9]/, ""); child = $0 } 75eda14cbcSMatt Macy /com\.datto:resilver_defer$/ { print child } 76eda14cbcSMatt Macy ') 77eda14cbcSMatt Macy 78eda14cbcSMatt Macy if [[ "$defer" == "-" ]] 79eda14cbcSMatt Macy then 80eda14cbcSMatt Macy [[ -n $VDEV_DEFERS ]] && 81eda14cbcSMatt Macy log_fail "didn't expect any vdevs to have resilver deferred" 82eda14cbcSMatt Macy return 83eda14cbcSMatt Macy fi 84eda14cbcSMatt Macy 85eda14cbcSMatt Macy [[ $VDEV_DEFERS -eq $defer ]] || 86eda14cbcSMatt Macy log_fail "resilver deferred set on unexpected vdev: $VDEV_DEFERS" 87eda14cbcSMatt Macy} 88eda14cbcSMatt Macy 89eda14cbcSMatt Macylog_assert "Check for unnecessary resilver restarts" 90eda14cbcSMatt Macy 91eda14cbcSMatt MacyORIG_RESILVER_MIN_TIME=$(get_tunable RESILVER_MIN_TIME_MS) 92eda14cbcSMatt MacyORIG_SCAN_SUSPEND_PROGRESS=$(get_tunable SCAN_SUSPEND_PROGRESS) 93eda14cbcSMatt MacyORIG_ZFS_ZEVENT_LEN_MAX=$(get_tunable ZEVENT_LEN_MAX) 94eda14cbcSMatt Macy 95eda14cbcSMatt Macyset -A RESTARTS -- '1' '2' '2' '2' 96eda14cbcSMatt Macyset -A VDEVS -- '' '' '' '' 97eda14cbcSMatt Macyset -A DEFER_RESTARTS -- '1' '1' '1' '2' 98eda14cbcSMatt Macyset -A DEFER_VDEVS -- '-' '2' '2' '-' 99eda14cbcSMatt Macy 100eda14cbcSMatt MacyVDEV_REPLACE="${VDEV_FILES[1]} $SPARE_VDEV_FILE" 101eda14cbcSMatt Macy 102eda14cbcSMatt Macylog_onexit cleanup 103eda14cbcSMatt Macy 104eda14cbcSMatt Macy# ensure that enough events will be saved 105eda14cbcSMatt Macylog_must set_tunable32 ZEVENT_LEN_MAX 512 106eda14cbcSMatt Macy 107eda14cbcSMatt Macylog_must truncate -s $VDEV_FILE_SIZE ${VDEV_FILES[@]} $SPARE_VDEV_FILE 108eda14cbcSMatt Macy 109eda14cbcSMatt Macylog_must zpool create -f -o feature@resilver_defer=disabled $TESTPOOL1 \ 110eda14cbcSMatt Macy raidz ${VDEV_FILES[@]} 111eda14cbcSMatt Macy 112eda14cbcSMatt Macy# create 4 filesystems 113eda14cbcSMatt Macyfor fs in fs{0..3} 114eda14cbcSMatt Macydo 115eda14cbcSMatt Macy log_must zfs create -o primarycache=none -o recordsize=1k $TESTPOOL1/$fs 116eda14cbcSMatt Macydone 117eda14cbcSMatt Macy 118eda14cbcSMatt Macy# simultaneously write 16M to each of them 119eda14cbcSMatt Macyset -A DATAPATHS /$TESTPOOL1/fs{0..3}/dat.0 120eda14cbcSMatt Macylog_note "Writing data files" 121eda14cbcSMatt Macyfor path in ${DATAPATHS[@]} 122eda14cbcSMatt Macydo 123eda14cbcSMatt Macy dd if=/dev/urandom of=$path bs=1M count=16 > /dev/null 2>&1 & 124eda14cbcSMatt Macydone 125eda14cbcSMatt Macywait 126eda14cbcSMatt Macy 127eda14cbcSMatt Macy# test without and with deferred resilve feature enabled 128eda14cbcSMatt Macyfor test in "without" "with" 129eda14cbcSMatt Macydo 130eda14cbcSMatt Macy log_note "Testing $test deferred resilvers" 131eda14cbcSMatt Macy 132eda14cbcSMatt Macy if [[ $test == "with" ]] 133eda14cbcSMatt Macy then 134eda14cbcSMatt Macy log_must zpool set feature@resilver_defer=enabled $TESTPOOL1 135eda14cbcSMatt Macy RESTARTS=( "${DEFER_RESTARTS[@]}" ) 136eda14cbcSMatt Macy VDEVS=( "${DEFER_VDEVS[@]}" ) 137eda14cbcSMatt Macy VDEV_REPLACE="$SPARE_VDEV_FILE ${VDEV_FILES[1]}" 138eda14cbcSMatt Macy fi 139eda14cbcSMatt Macy 140eda14cbcSMatt Macy # clear the events 141eda14cbcSMatt Macy log_must zpool events -c 142eda14cbcSMatt Macy 143eda14cbcSMatt Macy # limit scanning time 144eda14cbcSMatt Macy log_must set_tunable32 RESILVER_MIN_TIME_MS 50 145eda14cbcSMatt Macy 146eda14cbcSMatt Macy # initiate a resilver and suspend the scan as soon as possible 147eda14cbcSMatt Macy log_must zpool replace $TESTPOOL1 $VDEV_REPLACE 148eda14cbcSMatt Macy log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 149eda14cbcSMatt Macy 150eda14cbcSMatt Macy # there should only be 1 resilver start 151eda14cbcSMatt Macy verify_restarts '' "${RESTARTS[0]}" "${VDEVS[0]}" 152eda14cbcSMatt Macy 153eda14cbcSMatt Macy # offline then online a vdev to introduce a new DTL range after current 154eda14cbcSMatt Macy # scan, which should restart (or defer) the resilver 155eda14cbcSMatt Macy log_must zpool offline $TESTPOOL1 ${VDEV_FILES[2]} 156e92ffd9bSMartin Matuska sync_pool $TESTPOOL1 157eda14cbcSMatt Macy log_must zpool online $TESTPOOL1 ${VDEV_FILES[2]} 158e92ffd9bSMartin Matuska sync_pool $TESTPOOL1 159eda14cbcSMatt Macy 160eda14cbcSMatt Macy # there should now be 2 resilver starts w/o defer, 1 with defer 161eda14cbcSMatt Macy verify_restarts ' after offline/online' "${RESTARTS[1]}" "${VDEVS[1]}" 162eda14cbcSMatt Macy 163eda14cbcSMatt Macy # inject read io errors on vdev and verify resilver does not restart 164eda14cbcSMatt Macy log_must zinject -a -d ${VDEV_FILES[2]} -e io -T read -f 0.25 $TESTPOOL1 165*716fd348SMartin Matuska log_must cp ${DATAPATHS[1]} /dev/null 166eda14cbcSMatt Macy log_must zinject -c all 167eda14cbcSMatt Macy 168eda14cbcSMatt Macy # there should still be 2 resilver starts w/o defer, 1 with defer 169eda14cbcSMatt Macy verify_restarts ' after zinject' "${RESTARTS[2]}" "${VDEVS[2]}" 170eda14cbcSMatt Macy 171eda14cbcSMatt Macy # unsuspend resilver 172eda14cbcSMatt Macy log_must set_tunable32 SCAN_SUSPEND_PROGRESS 0 173eda14cbcSMatt Macy log_must set_tunable32 RESILVER_MIN_TIME_MS 3000 174eda14cbcSMatt Macy 175eda14cbcSMatt Macy # wait for resilver to finish 176eda14cbcSMatt Macy log_must zpool wait -t resilver $TESTPOOL1 177eda14cbcSMatt Macy log_must is_pool_resilvered $TESTPOOL1 178eda14cbcSMatt Macy 179eda14cbcSMatt Macy # wait for a few txg's to see if a resilver happens 180e92ffd9bSMartin Matuska sync_pool $TESTPOOL1 181e92ffd9bSMartin Matuska sync_pool $TESTPOOL1 182eda14cbcSMatt Macy 183eda14cbcSMatt Macy # there should now be 2 resilver starts 184eda14cbcSMatt Macy verify_restarts ' after resilver' "${RESTARTS[3]}" "${VDEVS[3]}" 185eda14cbcSMatt Macydone 186eda14cbcSMatt Macy 187eda14cbcSMatt Macylog_pass "Resilver did not restart unnecessarily" 188