1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or https://opensource.org/licenses/CDDL-1.0. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must rm -rf $dir 43 fi 44 done 45} 46 47# 48# Get random number between min and max number. 49# 50# $1 Minimal value 51# $2 Maximal value 52# 53function random 54{ 55 typeset -i min=$1 56 typeset -i max=$2 57 typeset -i value 58 59 while true; do 60 ((value = RANDOM % (max + 1))) 61 if ((value >= min)); then 62 break 63 fi 64 done 65 66 echo $value 67} 68 69# 70# Get the number of checksum errors for the pool. 71# 72# $1 Pool 73# 74function cksum_pool 75{ 76 typeset -i cksum=$(zpool status $1 | awk ' 77 !NF { isvdev = 0 } 78 isvdev { errors += $NF } 79 /CKSUM$/ { isvdev = 1 } 80 END { print errors } 81 ') 82 83 echo $cksum 84} 85 86# 87# Record the directories construction and checksum all the files which reside 88# within the specified pool 89# 90# $1 The specified pool 91# $2 The file which save the record. 92# 93function record_data 94{ 95 typeset pool=$1 96 typeset recordfile=$2 97 98 [[ -z $pool ]] && log_fail "No specified pool." 99 [[ -f $recordfile ]] && log_must rm -f $recordfile 100 101 sync_pool $pool 102 typeset mntpnt 103 mntpnt=$(get_prop mountpoint $pool) 104 log_must eval "du -a $mntpnt > $recordfile 2>&1" 105 # 106 # When the data was damaged, checksum is failing and return 1 107 # So, will not use log_must 108 # 109 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 110} 111 112# 113# Create test pool and fill with files and directories. 114# 115# $1 pool name 116# $2 pool type 117# $3 virtual devices number 118# 119function setup_test_env 120{ 121 typeset pool=$1 122 typeset keyword=$2 123 typeset -i vdev_cnt=$3 124 typeset vdevs 125 126 typeset -i i=0 127 while (( i < vdev_cnt )); do 128 vdevs="$vdevs $BASEDIR/vdev$i" 129 ((i += 1)) 130 done 131 132 if [[ ! -d $BASEDIR ]]; then 133 log_must mkdir $BASEDIR 134 fi 135 136 if poolexists $pool ; then 137 destroy_pool $pool 138 fi 139 140 log_must truncate -s $MINVDEVSIZE $vdevs 141 142 log_must zpool create -O compression=off -f -m $TESTDIR $pool $keyword $vdevs 143 144 log_note "Filling up the filesystem ..." 145 typeset -i i=0 146 typeset file=$TESTDIR/file 147 typeset -i limit 148 (( limit = $(get_prop available $pool) / 2 )) 149 150 while true ; do 151 [[ $(get_prop available $pool) -lt $limit ]] && break 152 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break 153 (( i = i + 1 )) 154 done 155 156 record_data $TESTPOOL $PRE_RECORD_FILE 157} 158 159function refill_test_env 160{ 161 log_note "Re-filling the filesystem ..." 162 typeset pool=$1 163 typeset -i i=0 164 typeset mntpnt 165 mntpnt=$(get_prop mountpoint $pool) 166 typeset file=$mntpnt/file 167 while [[ -e $file.$i ]]; do 168 log_must rm -f $file.$i 169 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break 170 (( i = i + 1 )) 171 done 172 173 record_data $TESTPOOL $PRE_RECORD_FILE 174} 175 176# 177# Check pool status is healthy 178# 179# $1 pool 180# 181function is_healthy 182{ 183 typeset pool=$1 184 185 typeset healthy_output="pool '$pool' is healthy" 186 typeset real_output=$(zpool status -x $pool) 187 188 if [[ "$real_output" == "$healthy_output" ]]; then 189 return 0 190 else 191 typeset -i ret 192 zpool status -x $pool | grep "state:" | grep -q "FAULTED" && return 1 193 typeset l_scan 194 typeset errnum _ 195 l_scan=$(zpool status -x $pool | grep "scan:") 196 l_scan=${l_scan##*"with"} 197 read -r errnum _ <<<"$l_scan" 198 199 return $errnum 200 fi 201} 202 203# 204# Check pool data is valid 205# 206# $1 pool 207# 208function is_data_valid 209{ 210 typeset pool=$1 211 212 log_must zpool scrub -w $pool 213 214 record_data $pool $PST_RECORD_FILE 215 if ! cmp $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null; then 216 log_must cat $PRE_RECORD_FILE 217 log_must cat $PST_RECORD_FILE 218 diff -u $PRE_RECORD_FILE $PST_RECORD_FILE 219 return 1 220 fi 221 222 return 0 223} 224 225# 226# Get the specified count devices name 227# 228# $1 pool name 229# $2 devices count 230# 231function get_vdevs #pool cnt 232{ 233 typeset pool=$1 234 typeset -i cnt=$2 235 236 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}' | \ 237 grep -vEe "^pool$|^capacity$|^mirror\-[0-9]$|^raidz[1-3]\-[0-9]$|^draid[1-3].*\-[0-9]$|---" \ 238 -e "/old$|^$pool$") 239 typeset -i i=0 240 typeset vdevs 241 while ((i < cnt)); do 242 typeset dev _ 243 read -r dev _ <<<"$all_devs" 244 eval all_devs=\${all_devs##*$dev} 245 246 vdevs="$dev $vdevs" 247 ((i += 1)) 248 done 249 250 echo "$vdevs" 251} 252 253# 254# Create and replace the same name virtual device files 255# 256# $1 pool name 257# $2-n virtual device files 258# 259function replace_missing_devs 260{ 261 typeset pool=$1 262 shift 263 264 typeset vdev 265 for vdev in $@; do 266 log_must dd if=/dev/zero of=$vdev \ 267 bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \ 268 conv=fdatasync 269 log_must zpool replace -wf $pool $vdev $vdev 270 done 271} 272 273# 274# Damage the pool's virtual device files. 275# 276# $1 pool name 277# $2 Failing devices count 278# $3 damage vdevs method, if not null, we keep 279# the label for the vdevs 280# 281function damage_devs 282{ 283 typeset pool=$1 284 typeset -i cnt=$2 285 typeset label="$3" 286 typeset vdevs 287 typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096)) 288 289 vdevs=$(get_vdevs $pool $cnt) 290 typeset dev 291 if [[ -n $label ]]; then 292 for dev in $vdevs; do 293 log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \ 294 count=$bs_count conv=notrunc >/dev/null 2>&1 295 done 296 else 297 for dev in $vdevs; do 298 log_must dd if=/dev/zero of=$dev bs=1024 \ 299 count=$bs_count conv=notrunc >/dev/null 2>&1 300 done 301 fi 302 303 sync_pool $pool 304} 305 306# 307# Clear errors in the pool caused by data corruptions 308# 309# $1 pool name 310# 311function clear_errors 312{ 313 typeset pool=$1 314 315 log_must zpool clear $pool 316 317 if ! is_healthy $pool ; then 318 log_note "$pool should be healthy." 319 return 1 320 fi 321 if ! is_data_valid $pool ; then 322 log_note "Data should be valid in $pool." 323 return 1 324 fi 325 326 return 0 327} 328 329# 330# Remove the specified pool's virtual device files 331# 332# $1 Pool name 333# $2 Missing devices count 334# 335function remove_devs 336{ 337 typeset pool=$1 338 typeset -i cnt=$2 339 typeset vdevs 340 341 vdevs=$(get_vdevs $pool $cnt) 342 log_must rm -f $vdevs 343 344 sync_pool $pool 345} 346 347# 348# Recover the bad or missing device files in the pool 349# 350# $1 Pool name 351# $2 Missing devices count 352# 353function recover_bad_missing_devs 354{ 355 typeset pool=$1 356 typeset -i cnt=$2 357 typeset vdevs 358 359 vdevs=$(get_vdevs $pool $cnt) 360 replace_missing_devs $pool $vdevs 361 362 if ! is_healthy $pool ; then 363 log_note "$pool should be healthy." 364 return 1 365 fi 366 if ! is_data_valid $pool ; then 367 log_note "Data should be valid in $pool." 368 return 1 369 fi 370 371 return 0 372} 373