1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must rm -rf $dir 43 fi 44 done 45} 46 47# 48# Get random number between min and max number. 49# 50# $1 Minimal value 51# $2 Maximal value 52# 53function random 54{ 55 typeset -i min=$1 56 typeset -i max=$2 57 typeset -i value 58 59 while true; do 60 ((value = RANDOM % (max + 1))) 61 if ((value >= min)); then 62 break 63 fi 64 done 65 66 echo $value 67} 68 69# 70# Get the number of checksum errors for the pool. 71# 72# $1 Pool 73# 74function cksum_pool 75{ 76 typeset -i cksum=$(zpool status $1 | awk ' 77 !NF { isvdev = 0 } 78 isvdev { errors += $NF } 79 /CKSUM$/ { isvdev = 1 } 80 END { print errors } 81 ') 82 83 echo $cksum 84} 85 86# 87# Record the directories construction and checksum all the files which reside 88# within the specified pool 89# 90# $1 The specified pool 91# $2 The file which save the record. 92# 93function record_data 94{ 95 typeset pool=$1 96 typeset recordfile=$2 97 98 [[ -z $pool ]] && log_fail "No specified pool." 99 [[ -f $recordfile ]] && log_must rm -f $recordfile 100 101 sync_pool $pool 102 typeset mntpnt 103 mntpnt=$(get_prop mountpoint $pool) 104 log_must eval "du -a $mntpnt > $recordfile 2>&1" 105 # 106 # When the data was damaged, checksum is failing and return 1 107 # So, will not use log_must 108 # 109 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 110} 111 112# 113# Create test pool and fill with files and directories. 114# 115# $1 pool name 116# $2 pool type 117# $3 virtual devices number 118# 119function setup_test_env 120{ 121 typeset pool=$1 122 typeset keyword=$2 123 typeset -i vdev_cnt=$3 124 typeset vdevs 125 126 typeset -i i=0 127 while (( i < vdev_cnt )); do 128 vdevs="$vdevs $BASEDIR/vdev$i" 129 ((i += 1)) 130 done 131 132 if [[ ! -d $BASEDIR ]]; then 133 log_must mkdir $BASEDIR 134 fi 135 136 if poolexists $pool ; then 137 destroy_pool $pool 138 fi 139 140 log_must truncate -s $MINVDEVSIZE $vdevs 141 142 log_must zpool create -f -m $TESTDIR $pool $keyword $vdevs 143 144 log_note "Filling up the filesystem ..." 145 typeset -i ret=0 146 typeset -i i=0 147 typeset file=$TESTDIR/file 148 typeset -i limit 149 (( limit = $(get_prop available $pool) / 2 )) 150 151 while true ; do 152 [[ $(get_prop available $pool) -lt $limit ]] && break 153 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES 154 ret=$? 155 (( $ret != 0 )) && break 156 (( i = i + 1 )) 157 done 158 159 record_data $TESTPOOL $PRE_RECORD_FILE 160} 161 162function refill_test_env 163{ 164 log_note "Re-filling the filesystem ..." 165 typeset pool=$1 166 typeset -i ret=0 167 typeset -i i=0 168 typeset mntpnt 169 mntpnt=$(get_prop mountpoint $pool) 170 typeset file=$mntpnt/file 171 while [[ -e $file.$i ]]; do 172 log_must rm -f $file.$i 173 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES 174 ret=$? 175 (( $ret != 0 )) && break 176 (( i = i + 1 )) 177 done 178 179 record_data $TESTPOOL $PRE_RECORD_FILE 180} 181 182# 183# Check pool status is healthy 184# 185# $1 pool 186# 187function is_healthy 188{ 189 typeset pool=$1 190 191 typeset healthy_output="pool '$pool' is healthy" 192 typeset real_output=$(zpool status -x $pool) 193 194 if [[ "$real_output" == "$healthy_output" ]]; then 195 return 0 196 else 197 typeset -i ret 198 zpool status -x $pool | grep "state:" | \ 199 grep "FAULTED" >/dev/null 2>&1 200 ret=$? 201 (( $ret == 0 )) && return 1 202 typeset l_scan 203 typeset errnum 204 l_scan=$(zpool status -x $pool | grep "scan:") 205 l_scan=${l_scan##*"with"} 206 errnum=$(echo $l_scan | awk '{print $1}') 207 208 return $errnum 209 fi 210} 211 212# 213# Check pool data is valid 214# 215# $1 pool 216# 217function is_data_valid 218{ 219 typeset pool=$1 220 221 log_must zpool scrub -w $pool 222 223 record_data $pool $PST_RECORD_FILE 224 if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 225 log_must cat $PRE_RECORD_FILE 226 log_must cat $PST_RECORD_FILE 227 diff -u $PRE_RECORD_FILE $PST_RECORD_FILE 228 return 1 229 fi 230 231 return 0 232} 233 234# 235# Get the specified count devices name 236# 237# $1 pool name 238# $2 devices count 239# 240function get_vdevs #pool cnt 241{ 242 typeset pool=$1 243 typeset -i cnt=$2 244 245 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ 246 egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|^raidz3$|^draid1.*|^draid2.*|^draid3.*|---" | \ 247 egrep -v "/old$|^$pool$") 248 typeset -i i=0 249 typeset vdevs 250 while ((i < cnt)); do 251 typeset dev=$(echo $all_devs | awk '{print $1}') 252 eval all_devs=\${all_devs##*$dev} 253 254 vdevs="$dev $vdevs" 255 ((i += 1)) 256 done 257 258 echo "$vdevs" 259} 260 261# 262# Create and replace the same name virtual device files 263# 264# $1 pool name 265# $2-n virtual device files 266# 267function replace_missing_devs 268{ 269 typeset pool=$1 270 shift 271 272 typeset vdev 273 for vdev in $@; do 274 log_must dd if=/dev/zero of=$vdev \ 275 bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \ 276 conv=fdatasync 277 log_must zpool replace -wf $pool $vdev $vdev 278 done 279} 280 281# 282# Damage the pool's virtual device files. 283# 284# $1 pool name 285# $2 Failing devices count 286# $3 damage vdevs method, if not null, we keep 287# the label for the vdevs 288# 289function damage_devs 290{ 291 typeset pool=$1 292 typeset -i cnt=$2 293 typeset label="$3" 294 typeset vdevs 295 typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096)) 296 297 vdevs=$(get_vdevs $pool $cnt) 298 typeset dev 299 if [[ -n $label ]]; then 300 for dev in $vdevs; do 301 log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \ 302 count=$bs_count conv=notrunc >/dev/null 2>&1 303 done 304 else 305 for dev in $vdevs; do 306 log_must dd if=/dev/zero of=$dev bs=1024 \ 307 count=$bs_count conv=notrunc >/dev/null 2>&1 308 done 309 fi 310 311 sync_pool $pool 312} 313 314# 315# Clear errors in the pool caused by data corruptions 316# 317# $1 pool name 318# 319function clear_errors 320{ 321 typeset pool=$1 322 323 log_must zpool clear $pool 324 325 if ! is_healthy $pool ; then 326 log_note "$pool should be healthy." 327 return 1 328 fi 329 if ! is_data_valid $pool ; then 330 log_note "Data should be valid in $pool." 331 return 1 332 fi 333 334 return 0 335} 336 337# 338# Remove the specified pool's virtual device files 339# 340# $1 Pool name 341# $2 Missing devices count 342# 343function remove_devs 344{ 345 typeset pool=$1 346 typeset -i cnt=$2 347 typeset vdevs 348 349 vdevs=$(get_vdevs $pool $cnt) 350 log_must rm -f $vdevs 351 352 sync_pool $pool 353} 354 355# 356# Recover the bad or missing device files in the pool 357# 358# $1 Pool name 359# $2 Missing devices count 360# 361function recover_bad_missing_devs 362{ 363 typeset pool=$1 364 typeset -i cnt=$2 365 typeset vdevs 366 367 vdevs=$(get_vdevs $pool $cnt) 368 replace_missing_devs $pool $vdevs 369 370 if ! is_healthy $pool ; then 371 log_note "$pool should be healthy." 372 return 1 373 fi 374 if ! is_data_valid $pool ; then 375 log_note "Data should be valid in $pool." 376 return 1 377 fi 378 379 return 0 380} 381