1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. 24# Use is subject to license terms. 25# 26 27. $STF_SUITE/include/libtest.shlib 28. $STF_SUITE/tests/functional/mmp/mmp.cfg 29 30 31function check_pool_import # pool opts token keyword 32{ 33 typeset pool=${1:-$MMP_POOL} 34 typeset opts=$2 35 typeset token=$3 36 typeset keyword=$4 37 38 zpool import $opts 2>&1 | \ 39 nawk -v token="$token:" '($1==token) {print $0}' | \ 40 grep -i "$keyword" > /dev/null 2>&1 41 42 return $? 43} 44 45function is_pool_imported # pool opts 46{ 47 typeset pool=${1:-$MMP_POOL} 48 typeset opts=$2 49 50 check_pool_import "$pool" "$opts" "status" \ 51 "The pool is currently imported" 52 return $? 53} 54 55function wait_pool_imported # pool opts 56{ 57 typeset pool=${1:-$MMP_POOL} 58 typeset opts=$2 59 60 while is_pool_imported "$pool" "$opts"; do 61 log_must sleep 5 62 done 63 64 return 0 65} 66 67function try_pool_import # pool opts message 68{ 69 typeset pool=${1:-$MMP_POOL} 70 typeset opts=$2 71 typeset msg=$3 72 73 zpool import $opts $pool 2>&1 | grep -i "$msg" 74 75 return $? 76} 77 78function mmp_set_hostid 79{ 80 typeset hostid=$1 81 82 zgenhostid $1 83 84 if [ $(hostid) != "$hostid" ]; then 85 return 1 86 fi 87 88 return 0 89} 90 91function mmp_clear_hostid 92{ 93 rm -f $HOSTID_FILE 94} 95 96function mmp_pool_create_simple # pool dir 97{ 98 typeset pool=${1:-$MMP_POOL} 99 typeset dir=${2:-$MMP_DIR} 100 101 log_must mkdir -p $dir 102 log_must rm -f $dir/* 103 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 104 105 log_must mmp_clear_hostid 106 log_must mmp_set_hostid $HOSTID1 107 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ 108 mirror $dir/vdev1 $dir/vdev2 109 log_must zpool set multihost=on $pool 110} 111 112function mmp_pool_create # pool dir 113{ 114 typeset pool=${1:-$MMP_POOL} 115 typeset dir=${2:-$MMP_DIR} 116 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" 117 118 mmp_pool_create_simple $pool $dir 119 120 log_must mv $MMP_CACHE ${MMP_CACHE}.stale 121 log_must zpool export $pool 122 log_must mmp_clear_hostid 123 log_must mmp_set_hostid $HOSTID2 124 125 log_note "Starting ztest in the background as hostid $HOSTID1" 126 log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &" 127 128 while ! is_pool_imported "$pool" "-d $dir"; do 129 log_must pgrep ztest 130 log_must sleep 5 131 done 132} 133 134function mmp_pool_destroy # pool dir 135{ 136 typeset pool=${1:-$MMP_POOL} 137 typeset dir=${2:-$MMP_DIR} 138 139 ZTESTPID=$(pgrep ztest) 140 if [ -n "$ZTESTPID" ]; then 141 log_must kill $ZTESTPID 142 wait $ZTESTPID 143 fi 144 145 if poolexists $pool; then 146 destroy_pool $pool 147 fi 148 149 log_must rm -f $dir/* 150 mmp_clear_hostid 151} 152 153function mmp_pool_set_hostid # pool hostid 154{ 155 typeset pool=$1 156 typeset hostid=$2 157 158 log_must mmp_clear_hostid 159 log_must mmp_set_hostid $hostid 160 log_must zpool export $pool 161 log_must zpool import $pool 162 163 return 0 164} 165# Return the number of seconds the activity check portion of the import process 166# will take. Does not include the time to find devices and assemble a config. 167# Note that the activity check may be skipped, e.g. if the pool and host 168# hostid's match, but this will return non-zero because mmp_* are populated. 169function seconds_mmp_waits_for_activity 170{ 171 typeset pool=$1 172 typeset devpath=$2 173 174 typeset seconds=0 175 typeset devices=${#DISK[@]} 176 typeset import_intervals=$(get_tunable MULTIHOST_IMPORT_INTERVALS) 177 typeset import_interval=$(get_tunable MULTIHOST_INTERVAL) 178 typeset tmpfile=$(mktemp) 179 typeset mmp_fail 180 typeset mmp_write 181 typeset mmp_delay 182 183 log_must zdb -e -p $devpath $pool >$tmpfile 2>/dev/null 184 mmp_fail=$(awk '/mmp_fail/ {print $NF}' $tmpfile) 185 mmp_write=$(awk '/mmp_write/ {print $NF}' $tmpfile) 186 mmp_delay=$(awk '/mmp_delay/ {print $NF}' $tmpfile) 187 if [ -f $tmpfile ]; then 188 rm $tmpfile 189 fi 190 191 # In order of preference: 192 if [ -n $mmp_fail -a -n $mmp_write ]; then 193 seconds=$((2*mmp_fail*mmp_write/1000)) 194 elif [ -n $mmp_delay ]; then 195 # MMP V0: Based on mmp_delay from the best Uberblock 196 seconds=$((import_intervals*devices*mmp_delay/1000000000)) 197 else 198 # Non-MMP aware: Based on zfs_multihost_interval and import_intervals 199 seconds=$((import_intervals*import_interval/1000)) 200 fi 201 202 echo $seconds 203} 204 205function import_no_activity_check # pool opts 206{ 207 typeset pool=$1 208 typeset opts=$2 209 210 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) 211 212 SECONDS=0 213 zpool import $opts $pool 214 typeset rc=$? 215 216 if [[ $SECONDS -gt $max_duration ]]; then 217 log_fail "ERROR: import_no_activity_check unexpected activity \ 218check (${SECONDS}s gt $max_duration)" 219 fi 220 221 return $rc 222} 223 224function import_activity_check # pool opts act_test_duration 225{ 226 typeset pool=$1 227 typeset opts=$2 228 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} 229 230 SECONDS=0 231 zpool import $opts $pool 232 typeset rc=$? 233 234 if [[ $SECONDS -le $min_duration ]]; then 235 log_fail "ERROR: import_activity_check expected activity check \ 236(${SECONDS}s le min_duration $min_duration)" 237 fi 238 239 return $rc 240} 241 242function clear_mmp_history 243{ 244 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF 245 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY 246} 247 248function count_skipped_mmp_writes # pool duration 249{ 250 typeset pool=$1 251 typeset -i duration=$2 252 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 253 254 sleep $duration 255 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path" 256} 257 258function count_mmp_writes # pool duration 259{ 260 typeset pool=$1 261 typeset -i duration=$2 262 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 263 264 sleep $duration 265 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path" 266} 267 268function summarize_uberblock_mmp # device 269{ 270 typeset device=$1 271 272 zdb -luuuu $device | awk ' 273 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;} 274 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};}; 275 /mmp_fail/ {fail=$3}; 276 /mmp_seq/ {seq=$3}; 277 /mmp_write/ {write=$3}; 278 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};}; 279 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++}; 280 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++}; 281 /mmp_valid/ && delay>0 && write<=0 {write_missing++}; 282 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++}; 283 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++}; 284 END { 285 print "total_uberblocks " total; 286 print "delay_zero " delay_zero; 287 print "write_fail_present " write_fail_present; 288 print "write_fail_missing " write_fail_missing; 289 print "write_missing " write_missing; 290 print "fail_missing " fail_missing; 291 print "seq_nonzero " seq_nonzero; 292 print "uberblock_invalid " uber_invalid; 293 }' 294} 295 296function count_mmp_write_fail_present # device 297{ 298 typeset device=$1 299 300 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}' 301} 302 303function count_mmp_write_fail_missing # device 304{ 305 typeset device=$1 306 307 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}' 308} 309 310function verify_mmp_write_fail_present # device 311{ 312 typeset device=$1 313 314 count=$(count_mmp_write_fail_present $device) 315 log_note "present count: $count" 316 if [ $count -eq 0 ]; then 317 summarize_uberblock_mmp $device 318 log_note "----- snip -----" 319 zdb -luuuu $device 320 log_note "----- snip -----" 321 log_fail "No Uberblocks contain valid mmp_write and fail values" 322 fi 323 324 count=$(count_mmp_write_fail_missing $device) 325 log_note "missing count: $count" 326 if [ $count -gt 0 ]; then 327 summarize_uberblock_mmp $device 328 log_note "----- snip -----" 329 zdb -luuuu $device 330 log_note "----- snip -----" 331 log_fail "Uberblocks missing mmp_write or mmp_fail" 332 fi 333} 334