1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or https://opensource.org/licenses/CDDL-1.0. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. 24# Use is subject to license terms. 25# 26 27. $STF_SUITE/include/libtest.shlib 28. $STF_SUITE/tests/functional/mmp/mmp.cfg 29 30 31function check_pool_import # pool opts token keyword 32{ 33 typeset pool=${1:-$MMP_POOL} 34 typeset opts=$2 35 typeset token=$3 36 typeset keyword=$4 37 38 zpool import $opts 2>&1 | \ 39 awk -v token="$token:" '($1==token) {print}' | \ 40 grep -iq "$keyword" 41} 42 43function is_pool_imported # pool opts 44{ 45 typeset pool=${1:-$MMP_POOL} 46 typeset opts=$2 47 48 check_pool_import "$pool" "$opts" "status" \ 49 "The pool is currently imported" 50} 51 52function wait_pool_imported # pool opts 53{ 54 typeset pool=${1:-$MMP_POOL} 55 typeset opts=$2 56 57 while is_pool_imported "$pool" "$opts"; do 58 log_must sleep 5 59 done 60} 61 62function try_pool_import # pool opts message 63{ 64 typeset pool=${1:-$MMP_POOL} 65 typeset opts=$2 66 typeset msg=$3 67 68 zpool import $opts $pool 2>&1 | grep -i "$msg" 69} 70 71function mmp_set_hostid 72{ 73 typeset hostid=$1 74 75 zgenhostid $1 76 77 [ $(hostid) = "$hostid" ] 78} 79 80function mmp_clear_hostid 81{ 82 rm -f $HOSTID_FILE 83} 84 85function mmp_pool_create_simple # pool dir 86{ 87 typeset pool=${1:-$MMP_POOL} 88 typeset dir=${2:-$MMP_DIR} 89 90 log_must mkdir -p $dir 91 log_must rm -f $dir/* 92 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 93 94 log_must mmp_clear_hostid 95 log_must mmp_set_hostid $HOSTID1 96 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ 97 mirror $dir/vdev1 $dir/vdev2 98 log_must zpool set multihost=on $pool 99} 100 101function mmp_pool_create # pool dir 102{ 103 typeset pool=${1:-$MMP_POOL} 104 typeset dir=${2:-$MMP_DIR} 105 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" 106 107 mmp_pool_create_simple $pool $dir 108 109 log_must mv $MMP_CACHE ${MMP_CACHE}.stale 110 log_must zpool export $pool 111 log_must mmp_clear_hostid 112 log_must mmp_set_hostid $HOSTID2 113 114 log_note "Starting ztest in the background as hostid $HOSTID1" 115 log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &" 116 117 while ! is_pool_imported "$pool" "-d $dir"; do 118 log_must pgrep ztest 119 log_must sleep 5 120 done 121} 122 123function mmp_pool_destroy # pool dir 124{ 125 typeset pool=${1:-$MMP_POOL} 126 typeset dir=${2:-$MMP_DIR} 127 128 ZTESTPID=$(pgrep ztest) 129 if [ -n "$ZTESTPID" ]; then 130 log_must kill $ZTESTPID 131 wait $ZTESTPID 132 fi 133 134 if poolexists $pool; then 135 destroy_pool $pool 136 fi 137 138 log_must rm -f $dir/* 139 mmp_clear_hostid 140} 141 142function mmp_pool_set_hostid # pool hostid 143{ 144 typeset pool=$1 145 typeset hostid=$2 146 147 log_must mmp_clear_hostid 148 log_must mmp_set_hostid $hostid 149 log_must zpool export $pool 150 log_must zpool import $pool 151 152 return 0 153} 154 155function import_no_activity_check # pool opts 156{ 157 typeset pool=$1 158 typeset opts=$2 159 160 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) 161 162 SECONDS=0 163 zpool import $opts $pool 164 typeset rc=$? 165 166 if [[ $SECONDS -gt $max_duration ]]; then 167 log_fail "ERROR: import_no_activity_check unexpected activity \ 168check (${SECONDS}s gt $max_duration)" 169 fi 170 171 return $rc 172} 173 174function import_activity_check # pool opts act_test_duration 175{ 176 typeset pool=$1 177 typeset opts=$2 178 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} 179 180 SECONDS=0 181 zpool import $opts $pool 182 typeset rc=$? 183 184 if [[ $SECONDS -le $min_duration ]]; then 185 log_fail "ERROR: import_activity_check expected activity check \ 186(${SECONDS}s le min_duration $min_duration)" 187 fi 188 189 return $rc 190} 191 192function clear_mmp_history 193{ 194 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF 195 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY 196} 197 198function count_skipped_mmp_writes # pool duration 199{ 200 typeset pool=$1 201 typeset -i duration=$2 202 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 203 204 sleep $duration 205 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path" 206} 207 208function count_mmp_writes # pool duration 209{ 210 typeset pool=$1 211 typeset -i duration=$2 212 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 213 214 sleep $duration 215 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path" 216} 217 218function summarize_uberblock_mmp # device 219{ 220 typeset device=$1 221 222 zdb -luuuu $device | awk ' 223 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;} 224 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};}; 225 /mmp_fail/ {fail=$3}; 226 /mmp_seq/ {seq=$3}; 227 /mmp_write/ {write=$3}; 228 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};}; 229 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++}; 230 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++}; 231 /mmp_valid/ && delay>0 && write<=0 {write_missing++}; 232 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++}; 233 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++}; 234 END { 235 print "total_uberblocks " total; 236 print "delay_zero " delay_zero; 237 print "write_fail_present " write_fail_present; 238 print "write_fail_missing " write_fail_missing; 239 print "write_missing " write_missing; 240 print "fail_missing " fail_missing; 241 print "seq_nonzero " seq_nonzero; 242 print "uberblock_invalid " uber_invalid; 243 }' 244} 245 246function count_mmp_write_fail_present # device 247{ 248 typeset device=$1 249 250 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}' 251} 252 253function count_mmp_write_fail_missing # device 254{ 255 typeset device=$1 256 257 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}' 258} 259 260function verify_mmp_write_fail_present # device 261{ 262 typeset device=$1 263 264 count=$(count_mmp_write_fail_present $device) 265 log_note "present count: $count" 266 if [ $count -eq 0 ]; then 267 summarize_uberblock_mmp $device 268 log_note "----- snip -----" 269 zdb -luuuu $device 270 log_note "----- snip -----" 271 log_fail "No Uberblocks contain valid mmp_write and fail values" 272 fi 273 274 count=$(count_mmp_write_fail_missing $device) 275 log_note "missing count: $count" 276 if [ $count -gt 0 ]; then 277 summarize_uberblock_mmp $device 278 log_note "----- snip -----" 279 zdb -luuuu $device 280 log_note "----- snip -----" 281 log_fail "Uberblocks missing mmp_write or mmp_fail" 282 fi 283} 284