1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or https://opensource.org/licenses/CDDL-1.0. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must rm -rf $dir 43 fi 44 done 45} 46 47# 48# Get the number of checksum errors for the pool. 49# 50# $1 Pool 51# 52function cksum_pool 53{ 54 typeset -i cksum=$(zpool status $1 | awk ' 55 !NF { isvdev = 0 } 56 isvdev { errors += $NF } 57 /CKSUM$/ { isvdev = 1 } 58 END { print errors } 59 ') 60 61 echo $cksum 62} 63 64# 65# Record the directories construction and checksum all the files which reside 66# within the specified pool 67# 68# $1 The specified pool 69# $2 The file which save the record. 70# 71function record_data 72{ 73 typeset pool=$1 74 typeset recordfile=$2 75 76 [[ -z $pool ]] && log_fail "No specified pool." 77 [[ -f $recordfile ]] && log_must rm -f $recordfile 78 79 sync_pool $pool 80 typeset mntpnt 81 mntpnt=$(get_prop mountpoint $pool) 82 log_must eval "du -a $mntpnt > $recordfile 2>&1" 83 # 84 # When the data was damaged, checksum is failing and return 1 85 # So, will not use log_must 86 # 87 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 88} 89 90# 91# Create test pool and fill with files and directories. 92# 93# $1 pool name 94# $2 pool type 95# $3 virtual devices number 96# 97function setup_test_env 98{ 99 typeset pool=$1 100 typeset keyword=$2 101 typeset -i vdev_cnt=$3 102 typeset vdevs 103 104 typeset -i i=0 105 while (( i < vdev_cnt )); do 106 vdevs="$vdevs $BASEDIR/vdev$i" 107 ((i += 1)) 108 done 109 110 if [[ ! -d $BASEDIR ]]; then 111 log_must mkdir $BASEDIR 112 fi 113 114 if poolexists $pool ; then 115 destroy_pool $pool 116 fi 117 118 log_must truncate -s $MINVDEVSIZE $vdevs 119 120 log_must zpool create -O compression=off -f -m $TESTDIR $pool $keyword $vdevs 121 122 log_note "Filling up the filesystem ..." 123 typeset -i i=0 124 typeset file=$TESTDIR/file 125 typeset -i limit 126 (( limit = $(get_prop available $pool) / 2 )) 127 128 while true ; do 129 [[ $(get_prop available $pool) -lt $limit ]] && break 130 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break 131 (( i = i + 1 )) 132 done 133 134 record_data $TESTPOOL $PRE_RECORD_FILE 135} 136 137function refill_test_env 138{ 139 log_note "Re-filling the filesystem ..." 140 typeset pool=$1 141 typeset -i i=0 142 typeset mntpnt 143 mntpnt=$(get_prop mountpoint $pool) 144 typeset file=$mntpnt/file 145 while [[ -e $file.$i ]]; do 146 log_must rm -f $file.$i 147 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES || break 148 (( i = i + 1 )) 149 done 150 151 record_data $TESTPOOL $PRE_RECORD_FILE 152} 153 154# 155# Check pool status is healthy 156# 157# $1 pool 158# 159function is_healthy 160{ 161 typeset pool=$1 162 163 typeset healthy_output="pool '$pool' is healthy" 164 typeset real_output=$(zpool status -x $pool) 165 166 if [[ "$real_output" == "$healthy_output" ]]; then 167 return 0 168 else 169 typeset -i ret 170 zpool status -x $pool | grep "state:" | grep -q "FAULTED" && return 1 171 typeset l_scan 172 typeset errnum _ 173 l_scan=$(zpool status -x $pool | grep "scan:") 174 l_scan=${l_scan##*"with"} 175 read -r errnum _ <<<"$l_scan" 176 177 return $errnum 178 fi 179} 180 181# 182# Check pool data is valid 183# 184# $1 pool 185# 186function is_data_valid 187{ 188 typeset pool=$1 189 190 log_must zpool scrub -w $pool 191 192 record_data $pool $PST_RECORD_FILE 193 if ! cmp $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null; then 194 log_must cat $PRE_RECORD_FILE 195 log_must cat $PST_RECORD_FILE 196 diff -u $PRE_RECORD_FILE $PST_RECORD_FILE 197 return 1 198 fi 199 200 return 0 201} 202 203# 204# Get the specified count devices name 205# 206# $1 pool name 207# $2 devices count 208# 209function get_vdevs #pool cnt 210{ 211 typeset pool=$1 212 typeset -i cnt=$2 213 214 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}' | \ 215 grep -vEe "^pool$|^capacity$|^mirror\-[0-9]$|^raidz[1-3]\-[0-9]$|^draid[1-3].*\-[0-9]$|---" \ 216 -e "/old$|^$pool$") 217 typeset -i i=0 218 typeset vdevs 219 while ((i < cnt)); do 220 typeset dev _ 221 read -r dev _ <<<"$all_devs" 222 eval all_devs=\${all_devs##*$dev} 223 224 vdevs="$dev $vdevs" 225 ((i += 1)) 226 done 227 228 echo "$vdevs" 229} 230 231# 232# Create and replace the same name virtual device files 233# 234# $1 pool name 235# $2-n virtual device files 236# 237function replace_missing_devs 238{ 239 typeset pool=$1 240 shift 241 242 typeset vdev 243 for vdev in $@; do 244 log_must dd if=/dev/zero of=$vdev \ 245 bs=1024k count=$((MINVDEVSIZE / (1024 * 1024))) \ 246 conv=fdatasync 247 log_must zpool replace -wf $pool $vdev $vdev 248 done 249} 250 251# 252# Damage the pool's virtual device files. 253# 254# $1 pool name 255# $2 Failing devices count 256# $3 damage vdevs method, if not null, we keep 257# the label for the vdevs 258# 259function damage_devs 260{ 261 typeset pool=$1 262 typeset -i cnt=$2 263 typeset label="$3" 264 typeset vdevs 265 typeset -i bs_count=$(((MINVDEVSIZE / 1024) - 4096)) 266 267 vdevs=$(get_vdevs $pool $cnt) 268 typeset dev 269 if [[ -n $label ]]; then 270 for dev in $vdevs; do 271 log_must dd if=/dev/zero of=$dev seek=512 bs=1024 \ 272 count=$bs_count conv=notrunc >/dev/null 2>&1 273 done 274 else 275 for dev in $vdevs; do 276 log_must dd if=/dev/zero of=$dev bs=1024 \ 277 count=$bs_count conv=notrunc >/dev/null 2>&1 278 done 279 fi 280 281 sync_pool $pool 282} 283 284# 285# Clear errors in the pool caused by data corruptions 286# 287# $1 pool name 288# 289function clear_errors 290{ 291 typeset pool=$1 292 293 log_must zpool clear $pool 294 295 if ! is_healthy $pool ; then 296 log_note "$pool should be healthy." 297 return 1 298 fi 299 if ! is_data_valid $pool ; then 300 log_note "Data should be valid in $pool." 301 return 1 302 fi 303 304 return 0 305} 306 307# 308# Remove the specified pool's virtual device files 309# 310# $1 Pool name 311# $2 Missing devices count 312# 313function remove_devs 314{ 315 typeset pool=$1 316 typeset -i cnt=$2 317 typeset vdevs 318 319 vdevs=$(get_vdevs $pool $cnt) 320 log_must rm -f $vdevs 321 322 sync_pool $pool 323} 324 325# 326# Recover the bad or missing device files in the pool 327# 328# $1 Pool name 329# $2 Missing devices count 330# 331function recover_bad_missing_devs 332{ 333 typeset pool=$1 334 typeset -i cnt=$2 335 typeset vdevs 336 337 vdevs=$(get_vdevs $pool $cnt) 338 replace_missing_devs $pool $vdevs 339 340 if ! is_healthy $pool ; then 341 log_note "$pool should be healthy." 342 return 1 343 fi 344 if ! is_data_valid $pool ; then 345 log_note "Data should be valid in $pool." 346 return 1 347 fi 348 349 return 0 350} 351