1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved. 24# Copyright (c) 2012, 2020, Delphix. All rights reserved. 25# Copyright (c) 2017, Tim Chase. All rights reserved. 26# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved. 27# Copyright (c) 2017, Lawrence Livermore National Security LLC. 28# Copyright (c) 2017, Datto Inc. All rights reserved. 29# Copyright (c) 2017, Open-E Inc. All rights reserved. 30# Copyright (c) 2021, The FreeBSD Foundation. 31# Use is subject to license terms. 32# 33 34. ${STF_TOOLS}/include/logapi.shlib 35. ${STF_SUITE}/include/math.shlib 36. ${STF_SUITE}/include/blkdev.shlib 37 38. ${STF_SUITE}/include/tunables.cfg 39 40# 41# Apply constrained path when available. This is required since the 42# PATH may have been modified by sudo's secure_path behavior. 43# 44if [ -n "$STF_PATH" ]; then 45 export PATH="$STF_PATH" 46fi 47 48# 49# Generic dot version comparison function 50# 51# Returns success when version $1 is greater than or equal to $2. 52# 53function compare_version_gte 54{ 55 if [[ "$(printf "$1\n$2" | sort -V | tail -n1)" == "$1" ]]; then 56 return 0 57 else 58 return 1 59 fi 60} 61 62# Linux kernel version comparison function 63# 64# $1 Linux version ("4.10", "2.6.32") or blank for installed Linux version 65# 66# Used for comparison: if [ $(linux_version) -ge $(linux_version "2.6.32") ] 67# 68function linux_version 69{ 70 typeset ver="$1" 71 72 [[ -z "$ver" ]] && ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+") 73 74 typeset version=$(echo $ver | cut -d '.' -f 1) 75 typeset major=$(echo $ver | cut -d '.' -f 2) 76 typeset minor=$(echo $ver | cut -d '.' -f 3) 77 78 [[ -z "$version" ]] && version=0 79 [[ -z "$major" ]] && major=0 80 [[ -z "$minor" ]] && minor=0 81 82 echo $((version * 10000 + major * 100 + minor)) 83} 84 85# Determine if this is a Linux test system 86# 87# Return 0 if platform Linux, 1 if otherwise 88 89function is_linux 90{ 91 if [[ $(uname -o) == "GNU/Linux" ]]; then 92 return 0 93 else 94 return 1 95 fi 96} 97 98# Determine if this is an illumos test system 99# 100# Return 0 if platform illumos, 1 if otherwise 101function is_illumos 102{ 103 if [[ $(uname -o) == "illumos" ]]; then 104 return 0 105 else 106 return 1 107 fi 108} 109 110# Determine if this is a FreeBSD test system 111# 112# Return 0 if platform FreeBSD, 1 if otherwise 113 114function is_freebsd 115{ 116 if [[ $(uname -o) == "FreeBSD" ]]; then 117 return 0 118 else 119 return 1 120 fi 121} 122 123# Determine if this is a DilOS test system 124# 125# Return 0 if platform DilOS, 1 if otherwise 126 127function is_dilos 128{ 129 typeset ID="" 130 [[ -f /etc/os-release ]] && . /etc/os-release 131 if [[ $ID == "dilos" ]]; then 132 return 0 133 else 134 return 1 135 fi 136} 137 138# Determine if this is a 32-bit system 139# 140# Return 0 if platform is 32-bit, 1 if otherwise 141 142function is_32bit 143{ 144 if [[ $(getconf LONG_BIT) == "32" ]]; then 145 return 0 146 else 147 return 1 148 fi 149} 150 151# Determine if kmemleak is enabled 152# 153# Return 0 if kmemleak is enabled, 1 if otherwise 154 155function is_kmemleak 156{ 157 if is_linux && [[ -e /sys/kernel/debug/kmemleak ]]; then 158 return 0 159 else 160 return 1 161 fi 162} 163 164# Determine whether a dataset is mounted 165# 166# $1 dataset name 167# $2 filesystem type; optional - defaulted to zfs 168# 169# Return 0 if dataset is mounted; 1 if unmounted; 2 on error 170 171function ismounted 172{ 173 typeset fstype=$2 174 [[ -z $fstype ]] && fstype=zfs 175 typeset out dir name ret 176 177 case $fstype in 178 zfs) 179 if [[ "$1" == "/"* ]] ; then 180 for out in $(zfs mount | awk '{print $2}'); do 181 [[ $1 == $out ]] && return 0 182 done 183 else 184 for out in $(zfs mount | awk '{print $1}'); do 185 [[ $1 == $out ]] && return 0 186 done 187 fi 188 ;; 189 ufs|nfs) 190 if is_freebsd; then 191 mount -pt $fstype | while read dev dir _t _flags; do 192 [[ "$1" == "$dev" || "$1" == "$dir" ]] && return 0 193 done 194 else 195 out=$(df -F $fstype $1 2>/dev/null) 196 ret=$? 197 (($ret != 0)) && return $ret 198 199 dir=${out%%\(*} 200 dir=${dir%% *} 201 name=${out##*\(} 202 name=${name%%\)*} 203 name=${name%% *} 204 205 [[ "$1" == "$dir" || "$1" == "$name" ]] && return 0 206 fi 207 ;; 208 ext*) 209 out=$(df -t $fstype $1 2>/dev/null) 210 return $? 211 ;; 212 zvol) 213 if [[ -L "$ZVOL_DEVDIR/$1" ]]; then 214 link=$(readlink -f $ZVOL_DEVDIR/$1) 215 [[ -n "$link" ]] && \ 216 mount | grep -q "^$link" && \ 217 return 0 218 fi 219 ;; 220 esac 221 222 return 1 223} 224 225# Return 0 if a dataset is mounted; 1 otherwise 226# 227# $1 dataset name 228# $2 filesystem type; optional - defaulted to zfs 229 230function mounted 231{ 232 ismounted $1 $2 233 (($? == 0)) && return 0 234 return 1 235} 236 237# Return 0 if a dataset is unmounted; 1 otherwise 238# 239# $1 dataset name 240# $2 filesystem type; optional - defaulted to zfs 241 242function unmounted 243{ 244 ismounted $1 $2 245 (($? == 1)) && return 0 246 return 1 247} 248 249# split line on "," 250# 251# $1 - line to split 252 253function splitline 254{ 255 echo $1 | sed "s/,/ /g" 256} 257 258function default_setup 259{ 260 default_setup_noexit "$@" 261 262 log_pass 263} 264 265function default_setup_no_mountpoint 266{ 267 default_setup_noexit "$1" "$2" "$3" "yes" 268 269 log_pass 270} 271 272# 273# Given a list of disks, setup storage pools and datasets. 274# 275function default_setup_noexit 276{ 277 typeset disklist=$1 278 typeset container=$2 279 typeset volume=$3 280 typeset no_mountpoint=$4 281 log_note begin default_setup_noexit 282 283 if is_global_zone; then 284 if poolexists $TESTPOOL ; then 285 destroy_pool $TESTPOOL 286 fi 287 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 288 log_must zpool create -f $TESTPOOL $disklist 289 else 290 reexport_pool 291 fi 292 293 rm -rf $TESTDIR || log_unresolved Could not remove $TESTDIR 294 mkdir -p $TESTDIR || log_unresolved Could not create $TESTDIR 295 296 log_must zfs create $TESTPOOL/$TESTFS 297 if [[ -z $no_mountpoint ]]; then 298 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 299 fi 300 301 if [[ -n $container ]]; then 302 rm -rf $TESTDIR1 || \ 303 log_unresolved Could not remove $TESTDIR1 304 mkdir -p $TESTDIR1 || \ 305 log_unresolved Could not create $TESTDIR1 306 307 log_must zfs create $TESTPOOL/$TESTCTR 308 log_must zfs set canmount=off $TESTPOOL/$TESTCTR 309 log_must zfs create $TESTPOOL/$TESTCTR/$TESTFS1 310 if [[ -z $no_mountpoint ]]; then 311 log_must zfs set mountpoint=$TESTDIR1 \ 312 $TESTPOOL/$TESTCTR/$TESTFS1 313 fi 314 fi 315 316 if [[ -n $volume ]]; then 317 if is_global_zone ; then 318 log_must zfs create -V $VOLSIZE $TESTPOOL/$TESTVOL 319 block_device_wait 320 else 321 log_must zfs create $TESTPOOL/$TESTVOL 322 fi 323 fi 324} 325 326# 327# Given a list of disks, setup a storage pool, file system and 328# a container. 329# 330function default_container_setup 331{ 332 typeset disklist=$1 333 334 default_setup "$disklist" "true" 335} 336 337# 338# Given a list of disks, setup a storage pool,file system 339# and a volume. 340# 341function default_volume_setup 342{ 343 typeset disklist=$1 344 345 default_setup "$disklist" "" "true" 346} 347 348# 349# Given a list of disks, setup a storage pool,file system, 350# a container and a volume. 351# 352function default_container_volume_setup 353{ 354 typeset disklist=$1 355 356 default_setup "$disklist" "true" "true" 357} 358 359# 360# Create a snapshot on a filesystem or volume. Defaultly create a snapshot on 361# filesystem 362# 363# $1 Existing filesystem or volume name. Default, $TESTPOOL/$TESTFS 364# $2 snapshot name. Default, $TESTSNAP 365# 366function create_snapshot 367{ 368 typeset fs_vol=${1:-$TESTPOOL/$TESTFS} 369 typeset snap=${2:-$TESTSNAP} 370 371 [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined." 372 [[ -z $snap ]] && log_fail "Snapshot's name is undefined." 373 374 if snapexists $fs_vol@$snap; then 375 log_fail "$fs_vol@$snap already exists." 376 fi 377 datasetexists $fs_vol || \ 378 log_fail "$fs_vol must exist." 379 380 log_must zfs snapshot $fs_vol@$snap 381} 382 383# 384# Create a clone from a snapshot, default clone name is $TESTCLONE. 385# 386# $1 Existing snapshot, $TESTPOOL/$TESTFS@$TESTSNAP is default. 387# $2 Clone name, $TESTPOOL/$TESTCLONE is default. 388# 389function create_clone # snapshot clone 390{ 391 typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 392 typeset clone=${2:-$TESTPOOL/$TESTCLONE} 393 394 [[ -z $snap ]] && \ 395 log_fail "Snapshot name is undefined." 396 [[ -z $clone ]] && \ 397 log_fail "Clone name is undefined." 398 399 log_must zfs clone $snap $clone 400} 401 402# 403# Create a bookmark of the given snapshot. Defaultly create a bookmark on 404# filesystem. 405# 406# $1 Existing filesystem or volume name. Default, $TESTFS 407# $2 Existing snapshot name. Default, $TESTSNAP 408# $3 bookmark name. Default, $TESTBKMARK 409# 410function create_bookmark 411{ 412 typeset fs_vol=${1:-$TESTFS} 413 typeset snap=${2:-$TESTSNAP} 414 typeset bkmark=${3:-$TESTBKMARK} 415 416 [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined." 417 [[ -z $snap ]] && log_fail "Snapshot's name is undefined." 418 [[ -z $bkmark ]] && log_fail "Bookmark's name is undefined." 419 420 if bkmarkexists $fs_vol#$bkmark; then 421 log_fail "$fs_vol#$bkmark already exists." 422 fi 423 datasetexists $fs_vol || \ 424 log_fail "$fs_vol must exist." 425 snapexists $fs_vol@$snap || \ 426 log_fail "$fs_vol@$snap must exist." 427 428 log_must zfs bookmark $fs_vol@$snap $fs_vol#$bkmark 429} 430 431# 432# Create a temporary clone result of an interrupted resumable 'zfs receive' 433# $1 Destination filesystem name. Must not exist, will be created as the result 434# of this function along with its %recv temporary clone 435# $2 Source filesystem name. Must not exist, will be created and destroyed 436# 437function create_recv_clone 438{ 439 typeset recvfs="$1" 440 typeset sendfs="${2:-$TESTPOOL/create_recv_clone}" 441 typeset snap="$sendfs@snap1" 442 typeset incr="$sendfs@snap2" 443 typeset mountpoint="$TESTDIR/create_recv_clone" 444 typeset sendfile="$TESTDIR/create_recv_clone.zsnap" 445 446 [[ -z $recvfs ]] && log_fail "Recv filesystem's name is undefined." 447 448 datasetexists $recvfs && log_fail "Recv filesystem must not exist." 449 datasetexists $sendfs && log_fail "Send filesystem must not exist." 450 451 log_must zfs create -o mountpoint="$mountpoint" $sendfs 452 log_must zfs snapshot $snap 453 log_must eval "zfs send $snap | zfs recv -u $recvfs" 454 log_must mkfile 1m "$mountpoint/data" 455 log_must zfs snapshot $incr 456 log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \ 457 iflag=fullblock > $sendfile" 458 log_mustnot eval "zfs recv -su $recvfs < $sendfile" 459 destroy_dataset "$sendfs" "-r" 460 log_must rm -f "$sendfile" 461 462 if [[ $(get_prop 'inconsistent' "$recvfs/%recv") -ne 1 ]]; then 463 log_fail "Error creating temporary $recvfs/%recv clone" 464 fi 465} 466 467function default_mirror_setup 468{ 469 default_mirror_setup_noexit $1 $2 $3 470 471 log_pass 472} 473 474# 475# Given a pair of disks, set up a storage pool and dataset for the mirror 476# @parameters: $1 the primary side of the mirror 477# $2 the secondary side of the mirror 478# @uses: ZPOOL ZFS TESTPOOL TESTFS 479function default_mirror_setup_noexit 480{ 481 readonly func="default_mirror_setup_noexit" 482 typeset primary=$1 483 typeset secondary=$2 484 485 [[ -z $primary ]] && \ 486 log_fail "$func: No parameters passed" 487 [[ -z $secondary ]] && \ 488 log_fail "$func: No secondary partition passed" 489 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 490 log_must zpool create -f $TESTPOOL mirror $@ 491 log_must zfs create $TESTPOOL/$TESTFS 492 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 493} 494 495# 496# create a number of mirrors. 497# We create a number($1) of 2 way mirrors using the pairs of disks named 498# on the command line. These mirrors are *not* mounted 499# @parameters: $1 the number of mirrors to create 500# $... the devices to use to create the mirrors on 501# @uses: ZPOOL ZFS TESTPOOL 502function setup_mirrors 503{ 504 typeset -i nmirrors=$1 505 506 shift 507 while ((nmirrors > 0)); do 508 log_must test -n "$1" -a -n "$2" 509 [[ -d /$TESTPOOL$nmirrors ]] && rm -rf /$TESTPOOL$nmirrors 510 log_must zpool create -f $TESTPOOL$nmirrors mirror $1 $2 511 shift 2 512 ((nmirrors = nmirrors - 1)) 513 done 514} 515 516# 517# create a number of raidz pools. 518# We create a number($1) of 2 raidz pools using the pairs of disks named 519# on the command line. These pools are *not* mounted 520# @parameters: $1 the number of pools to create 521# $... the devices to use to create the pools on 522# @uses: ZPOOL ZFS TESTPOOL 523function setup_raidzs 524{ 525 typeset -i nraidzs=$1 526 527 shift 528 while ((nraidzs > 0)); do 529 log_must test -n "$1" -a -n "$2" 530 [[ -d /$TESTPOOL$nraidzs ]] && rm -rf /$TESTPOOL$nraidzs 531 log_must zpool create -f $TESTPOOL$nraidzs raidz $1 $2 532 shift 2 533 ((nraidzs = nraidzs - 1)) 534 done 535} 536 537# 538# Destroy the configured testpool mirrors. 539# the mirrors are of the form ${TESTPOOL}{number} 540# @uses: ZPOOL ZFS TESTPOOL 541function destroy_mirrors 542{ 543 default_cleanup_noexit 544 545 log_pass 546} 547 548# 549# Given a minimum of two disks, set up a storage pool and dataset for the raid-z 550# $1 the list of disks 551# 552function default_raidz_setup 553{ 554 typeset disklist="$*" 555 disks=(${disklist[*]}) 556 557 if [[ ${#disks[*]} -lt 2 ]]; then 558 log_fail "A raid-z requires a minimum of two disks." 559 fi 560 561 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 562 log_must zpool create -f $TESTPOOL raidz $disklist 563 log_must zfs create $TESTPOOL/$TESTFS 564 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 565 566 log_pass 567} 568 569# 570# Common function used to cleanup storage pools and datasets. 571# 572# Invoked at the start of the test suite to ensure the system 573# is in a known state, and also at the end of each set of 574# sub-tests to ensure errors from one set of tests doesn't 575# impact the execution of the next set. 576 577function default_cleanup 578{ 579 default_cleanup_noexit 580 581 log_pass 582} 583 584# 585# Utility function used to list all available pool names. 586# 587# NOTE: $KEEP is a variable containing pool names, separated by a newline 588# character, that must be excluded from the returned list. 589# 590function get_all_pools 591{ 592 zpool list -H -o name | grep -Fvx "$KEEP" | grep -v "$NO_POOLS" 593} 594 595function default_cleanup_noexit 596{ 597 typeset pool="" 598 # 599 # Destroying the pool will also destroy any 600 # filesystems it contains. 601 # 602 if is_global_zone; then 603 zfs unmount -a > /dev/null 2>&1 604 ALL_POOLS=$(get_all_pools) 605 # Here, we loop through the pools we're allowed to 606 # destroy, only destroying them if it's safe to do 607 # so. 608 while [ ! -z ${ALL_POOLS} ] 609 do 610 for pool in ${ALL_POOLS} 611 do 612 if safe_to_destroy_pool $pool ; 613 then 614 destroy_pool $pool 615 fi 616 done 617 ALL_POOLS=$(get_all_pools) 618 done 619 620 zfs mount -a 621 else 622 typeset fs="" 623 for fs in $(zfs list -H -o name \ 624 | grep "^$ZONE_POOL/$ZONE_CTR[01234]/"); do 625 destroy_dataset "$fs" "-Rf" 626 done 627 628 # Need cleanup here to avoid garbage dir left. 629 for fs in $(zfs list -H -o name); do 630 [[ $fs == /$ZONE_POOL ]] && continue 631 [[ -d $fs ]] && log_must rm -rf $fs/* 632 done 633 634 # 635 # Reset the $ZONE_POOL/$ZONE_CTR[01234] file systems property to 636 # the default value 637 # 638 for fs in $(zfs list -H -o name); do 639 if [[ $fs == $ZONE_POOL/$ZONE_CTR[01234] ]]; then 640 log_must zfs set reservation=none $fs 641 log_must zfs set recordsize=128K $fs 642 log_must zfs set mountpoint=/$fs $fs 643 typeset enc="" 644 enc=$(get_prop encryption $fs) 645 if [[ $? -ne 0 ]] || [[ -z "$enc" ]] || \ 646 [[ "$enc" == "off" ]]; then 647 log_must zfs set checksum=on $fs 648 fi 649 log_must zfs set compression=off $fs 650 log_must zfs set atime=on $fs 651 log_must zfs set devices=off $fs 652 log_must zfs set exec=on $fs 653 log_must zfs set setuid=on $fs 654 log_must zfs set readonly=off $fs 655 log_must zfs set snapdir=hidden $fs 656 log_must zfs set aclmode=groupmask $fs 657 log_must zfs set aclinherit=secure $fs 658 fi 659 done 660 fi 661 662 [[ -d $TESTDIR ]] && \ 663 log_must rm -rf $TESTDIR 664 665 disk1=${DISKS%% *} 666 if is_mpath_device $disk1; then 667 delete_partitions 668 fi 669 670 rm -f $TEST_BASE_DIR/{err,out} 671} 672 673 674# 675# Common function used to cleanup storage pools, file systems 676# and containers. 677# 678function default_container_cleanup 679{ 680 if ! is_global_zone; then 681 reexport_pool 682 fi 683 684 ismounted $TESTPOOL/$TESTCTR/$TESTFS1 685 [[ $? -eq 0 ]] && \ 686 log_must zfs unmount $TESTPOOL/$TESTCTR/$TESTFS1 687 688 destroy_dataset "$TESTPOOL/$TESTCTR/$TESTFS1" "-R" 689 destroy_dataset "$TESTPOOL/$TESTCTR" "-Rf" 690 691 [[ -e $TESTDIR1 ]] && \ 692 log_must rm -rf $TESTDIR1 > /dev/null 2>&1 693 694 default_cleanup 695} 696 697# 698# Common function used to cleanup snapshot of file system or volume. Default to 699# delete the file system's snapshot 700# 701# $1 snapshot name 702# 703function destroy_snapshot 704{ 705 typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 706 707 if ! snapexists $snap; then 708 log_fail "'$snap' does not exist." 709 fi 710 711 # 712 # For the sake of the value which come from 'get_prop' is not equal 713 # to the really mountpoint when the snapshot is unmounted. So, firstly 714 # check and make sure this snapshot's been mounted in current system. 715 # 716 typeset mtpt="" 717 if ismounted $snap; then 718 mtpt=$(get_prop mountpoint $snap) 719 (($? != 0)) && \ 720 log_fail "get_prop mountpoint $snap failed." 721 fi 722 723 destroy_dataset "$snap" 724 [[ $mtpt != "" && -d $mtpt ]] && \ 725 log_must rm -rf $mtpt 726} 727 728# 729# Common function used to cleanup clone. 730# 731# $1 clone name 732# 733function destroy_clone 734{ 735 typeset clone=${1:-$TESTPOOL/$TESTCLONE} 736 737 if ! datasetexists $clone; then 738 log_fail "'$clone' does not existed." 739 fi 740 741 # With the same reason in destroy_snapshot 742 typeset mtpt="" 743 if ismounted $clone; then 744 mtpt=$(get_prop mountpoint $clone) 745 (($? != 0)) && \ 746 log_fail "get_prop mountpoint $clone failed." 747 fi 748 749 destroy_dataset "$clone" 750 [[ $mtpt != "" && -d $mtpt ]] && \ 751 log_must rm -rf $mtpt 752} 753 754# 755# Common function used to cleanup bookmark of file system or volume. Default 756# to delete the file system's bookmark. 757# 758# $1 bookmark name 759# 760function destroy_bookmark 761{ 762 typeset bkmark=${1:-$TESTPOOL/$TESTFS#$TESTBKMARK} 763 764 if ! bkmarkexists $bkmark; then 765 log_fail "'$bkmarkp' does not existed." 766 fi 767 768 destroy_dataset "$bkmark" 769} 770 771# Return 0 if a snapshot exists; $? otherwise 772# 773# $1 - snapshot name 774 775function snapexists 776{ 777 zfs list -H -t snapshot "$1" > /dev/null 2>&1 778 return $? 779} 780 781# 782# Return 0 if a bookmark exists; $? otherwise 783# 784# $1 - bookmark name 785# 786function bkmarkexists 787{ 788 zfs list -H -t bookmark "$1" > /dev/null 2>&1 789 return $? 790} 791 792# 793# Return 0 if a hold exists; $? otherwise 794# 795# $1 - hold tag 796# $2 - snapshot name 797# 798function holdexists 799{ 800 zfs holds "$2" | awk '{ print $2 }' | grep "$1" > /dev/null 2>&1 801 return $? 802} 803 804# 805# Set a property to a certain value on a dataset. 806# Sets a property of the dataset to the value as passed in. 807# @param: 808# $1 dataset who's property is being set 809# $2 property to set 810# $3 value to set property to 811# @return: 812# 0 if the property could be set. 813# non-zero otherwise. 814# @use: ZFS 815# 816function dataset_setprop 817{ 818 typeset fn=dataset_setprop 819 820 if (($# < 3)); then 821 log_note "$fn: Insufficient parameters (need 3, had $#)" 822 return 1 823 fi 824 typeset output= 825 output=$(zfs set $2=$3 $1 2>&1) 826 typeset rv=$? 827 if ((rv != 0)); then 828 log_note "Setting property on $1 failed." 829 log_note "property $2=$3" 830 log_note "Return Code: $rv" 831 log_note "Output: $output" 832 return $rv 833 fi 834 return 0 835} 836 837# 838# Assign suite defined dataset properties. 839# This function is used to apply the suite's defined default set of 840# properties to a dataset. 841# @parameters: $1 dataset to use 842# @uses: ZFS COMPRESSION_PROP CHECKSUM_PROP 843# @returns: 844# 0 if the dataset has been altered. 845# 1 if no pool name was passed in. 846# 2 if the dataset could not be found. 847# 3 if the dataset could not have it's properties set. 848# 849function dataset_set_defaultproperties 850{ 851 typeset dataset="$1" 852 853 [[ -z $dataset ]] && return 1 854 855 typeset confset= 856 typeset -i found=0 857 for confset in $(zfs list); do 858 if [[ $dataset = $confset ]]; then 859 found=1 860 break 861 fi 862 done 863 [[ $found -eq 0 ]] && return 2 864 if [[ -n $COMPRESSION_PROP ]]; then 865 dataset_setprop $dataset compression $COMPRESSION_PROP || \ 866 return 3 867 log_note "Compression set to '$COMPRESSION_PROP' on $dataset" 868 fi 869 if [[ -n $CHECKSUM_PROP ]]; then 870 dataset_setprop $dataset checksum $CHECKSUM_PROP || \ 871 return 3 872 log_note "Checksum set to '$CHECKSUM_PROP' on $dataset" 873 fi 874 return 0 875} 876 877# 878# Check a numeric assertion 879# @parameter: $@ the assertion to check 880# @output: big loud notice if assertion failed 881# @use: log_fail 882# 883function assert 884{ 885 (($@)) || log_fail "$@" 886} 887 888# 889# Function to format partition size of a disk 890# Given a disk cxtxdx reduces all partitions 891# to 0 size 892# 893function zero_partitions #<whole_disk_name> 894{ 895 typeset diskname=$1 896 typeset i 897 898 if is_freebsd; then 899 gpart destroy -F $diskname 900 elif is_linux; then 901 DSK=$DEV_DSKDIR/$diskname 902 DSK=$(echo $DSK | sed -e "s|//|/|g") 903 log_must parted $DSK -s -- mklabel gpt 904 blockdev --rereadpt $DSK 2>/dev/null 905 block_device_wait 906 else 907 for i in 0 1 3 4 5 6 7 908 do 909 log_must set_partition $i "" 0mb $diskname 910 done 911 fi 912 913 return 0 914} 915 916# 917# Given a slice, size and disk, this function 918# formats the slice to the specified size. 919# Size should be specified with units as per 920# the `format` command requirements eg. 100mb 3gb 921# 922# NOTE: This entire interface is problematic for the Linux parted utility 923# which requires the end of the partition to be specified. It would be 924# best to retire this interface and replace it with something more flexible. 925# At the moment a best effort is made. 926# 927# arguments: <slice_num> <slice_start> <size_plus_units> <whole_disk_name> 928function set_partition 929{ 930 typeset -i slicenum=$1 931 typeset start=$2 932 typeset size=$3 933 typeset disk=${4#$DEV_DSKDIR/} 934 disk=${disk#$DEV_RDSKDIR/} 935 936 case "$(uname)" in 937 Linux) 938 if [[ -z $size || -z $disk ]]; then 939 log_fail "The size or disk name is unspecified." 940 fi 941 disk=$DEV_DSKDIR/$disk 942 typeset size_mb=${size%%[mMgG]} 943 944 size_mb=${size_mb%%[mMgG][bB]} 945 if [[ ${size:1:1} == 'g' ]]; then 946 ((size_mb = size_mb * 1024)) 947 fi 948 949 # Create GPT partition table when setting slice 0 or 950 # when the device doesn't already contain a GPT label. 951 parted $disk -s -- print 1 >/dev/null 952 typeset ret_val=$? 953 if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then 954 parted $disk -s -- mklabel gpt 955 if [[ $? -ne 0 ]]; then 956 log_note "Failed to create GPT partition table on $disk" 957 return 1 958 fi 959 fi 960 961 # When no start is given align on the first cylinder. 962 if [[ -z "$start" ]]; then 963 start=1 964 fi 965 966 # Determine the cylinder size for the device and using 967 # that calculate the end offset in cylinders. 968 typeset -i cly_size_kb=0 969 cly_size_kb=$(parted -m $disk -s -- \ 970 unit cyl print | head -3 | tail -1 | \ 971 awk -F '[:k.]' '{print $4}') 972 ((end = (size_mb * 1024 / cly_size_kb) + start)) 973 974 parted $disk -s -- \ 975 mkpart part$slicenum ${start}cyl ${end}cyl 976 typeset ret_val=$? 977 if [[ $ret_val -ne 0 ]]; then 978 log_note "Failed to create partition $slicenum on $disk" 979 return 1 980 fi 981 982 blockdev --rereadpt $disk 2>/dev/null 983 block_device_wait $disk 984 ;; 985 FreeBSD) 986 if [[ -z $size || -z $disk ]]; then 987 log_fail "The size or disk name is unspecified." 988 fi 989 disk=$DEV_DSKDIR/$disk 990 991 if [[ $slicenum -eq 0 ]] || ! gpart show $disk >/dev/null 2>&1; then 992 gpart destroy -F $disk >/dev/null 2>&1 993 gpart create -s GPT $disk 994 if [[ $? -ne 0 ]]; then 995 log_note "Failed to create GPT partition table on $disk" 996 return 1 997 fi 998 fi 999 1000 typeset index=$((slicenum + 1)) 1001 1002 if [[ -n $start ]]; then 1003 start="-b $start" 1004 fi 1005 gpart add -t freebsd-zfs $start -s $size -i $index $disk 1006 if [[ $ret_val -ne 0 ]]; then 1007 log_note "Failed to create partition $slicenum on $disk" 1008 return 1 1009 fi 1010 1011 block_device_wait $disk 1012 ;; 1013 *) 1014 if [[ -z $slicenum || -z $size || -z $disk ]]; then 1015 log_fail "The slice, size or disk name is unspecified." 1016 fi 1017 1018 typeset format_file=/var/tmp/format_in.$$ 1019 1020 echo "partition" >$format_file 1021 echo "$slicenum" >> $format_file 1022 echo "" >> $format_file 1023 echo "" >> $format_file 1024 echo "$start" >> $format_file 1025 echo "$size" >> $format_file 1026 echo "label" >> $format_file 1027 echo "" >> $format_file 1028 echo "q" >> $format_file 1029 echo "q" >> $format_file 1030 1031 format -e -s -d $disk -f $format_file 1032 typeset ret_val=$? 1033 rm -f $format_file 1034 ;; 1035 esac 1036 1037 if [[ $ret_val -ne 0 ]]; then 1038 log_note "Unable to format $disk slice $slicenum to $size" 1039 return 1 1040 fi 1041 return 0 1042} 1043 1044# 1045# Delete all partitions on all disks - this is specifically for the use of multipath 1046# devices which currently can only be used in the test suite as raw/un-partitioned 1047# devices (ie a zpool cannot be created on a whole mpath device that has partitions) 1048# 1049function delete_partitions 1050{ 1051 typeset disk 1052 1053 if [[ -z $DISKSARRAY ]]; then 1054 DISKSARRAY=$DISKS 1055 fi 1056 1057 if is_linux; then 1058 typeset -i part 1059 for disk in $DISKSARRAY; do 1060 for (( part = 1; part < MAX_PARTITIONS; part++ )); do 1061 typeset partition=${disk}${SLICE_PREFIX}${part} 1062 parted $DEV_DSKDIR/$disk -s rm $part > /dev/null 2>&1 1063 if lsblk | grep -qF ${partition}; then 1064 log_fail "Partition ${partition} not deleted" 1065 else 1066 log_note "Partition ${partition} deleted" 1067 fi 1068 done 1069 done 1070 elif is_freebsd; then 1071 for disk in $DISKSARRAY; do 1072 if gpart destroy -F $disk; then 1073 log_note "Partitions for ${disk} deleted" 1074 else 1075 log_fail "Partitions for ${disk} not deleted" 1076 fi 1077 done 1078 fi 1079} 1080 1081# 1082# Get the end cyl of the given slice 1083# 1084function get_endslice #<disk> <slice> 1085{ 1086 typeset disk=$1 1087 typeset slice=$2 1088 if [[ -z $disk || -z $slice ]] ; then 1089 log_fail "The disk name or slice number is unspecified." 1090 fi 1091 1092 case "$(uname)" in 1093 Linux) 1094 endcyl=$(parted -s $DEV_DSKDIR/$disk -- unit cyl print | \ 1095 grep "part${slice}" | \ 1096 awk '{print $3}' | \ 1097 sed 's,cyl,,') 1098 ((endcyl = (endcyl + 1))) 1099 ;; 1100 FreeBSD) 1101 disk=${disk#/dev/zvol/} 1102 disk=${disk%p*} 1103 slice=$((slice + 1)) 1104 endcyl=$(gpart show $disk | \ 1105 awk -v slice=$slice '$3 == slice { print $1 + $2 }') 1106 ;; 1107 *) 1108 disk=${disk#/dev/dsk/} 1109 disk=${disk#/dev/rdsk/} 1110 disk=${disk%s*} 1111 1112 typeset -i ratio=0 1113 ratio=$(prtvtoc /dev/rdsk/${disk}s2 | \ 1114 grep "sectors\/cylinder" | \ 1115 awk '{print $2}') 1116 1117 if ((ratio == 0)); then 1118 return 1119 fi 1120 1121 typeset -i endcyl=$(prtvtoc -h /dev/rdsk/${disk}s2 | 1122 nawk -v token="$slice" '{if ($1==token) print $6}') 1123 1124 ((endcyl = (endcyl + 1) / ratio)) 1125 ;; 1126 esac 1127 1128 echo $endcyl 1129} 1130 1131 1132# 1133# Given a size,disk and total slice number, this function formats the 1134# disk slices from 0 to the total slice number with the same specified 1135# size. 1136# 1137function partition_disk #<slice_size> <whole_disk_name> <total_slices> 1138{ 1139 typeset -i i=0 1140 typeset slice_size=$1 1141 typeset disk_name=$2 1142 typeset total_slices=$3 1143 typeset cyl 1144 1145 zero_partitions $disk_name 1146 while ((i < $total_slices)); do 1147 if ! is_linux; then 1148 if ((i == 2)); then 1149 ((i = i + 1)) 1150 continue 1151 fi 1152 fi 1153 log_must set_partition $i "$cyl" $slice_size $disk_name 1154 cyl=$(get_endslice $disk_name $i) 1155 ((i = i+1)) 1156 done 1157} 1158 1159# 1160# This function continues to write to a filenum number of files into dirnum 1161# number of directories until either file_write returns an error or the 1162# maximum number of files per directory have been written. 1163# 1164# Usage: 1165# fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data] 1166# 1167# Return value: 0 on success 1168# non 0 on error 1169# 1170# Where : 1171# destdir: is the directory where everything is to be created under 1172# dirnum: the maximum number of subdirectories to use, -1 no limit 1173# filenum: the maximum number of files per subdirectory 1174# bytes: number of bytes to write 1175# num_writes: number of types to write out bytes 1176# data: the data that will be written 1177# 1178# E.g. 1179# fill_fs /testdir 20 25 1024 256 0 1180# 1181# Note: bytes * num_writes equals the size of the testfile 1182# 1183function fill_fs # destdir dirnum filenum bytes num_writes data 1184{ 1185 typeset destdir=${1:-$TESTDIR} 1186 typeset -i dirnum=${2:-50} 1187 typeset -i filenum=${3:-50} 1188 typeset -i bytes=${4:-8192} 1189 typeset -i num_writes=${5:-10240} 1190 typeset data=${6:-0} 1191 1192 mkdir -p $destdir/{1..$dirnum} 1193 for f in $destdir/{1..$dirnum}/$TESTFILE{1..$filenum}; do 1194 file_write -o create -f $f -b $bytes -c $num_writes -d $data \ 1195 || return $? 1196 done 1197 return 0 1198} 1199 1200# 1201# Simple function to get the specified property. If unable to 1202# get the property then exits. 1203# 1204# Note property is in 'parsable' format (-p) 1205# 1206function get_prop # property dataset 1207{ 1208 typeset prop_val 1209 typeset prop=$1 1210 typeset dataset=$2 1211 1212 prop_val=$(zfs get -pH -o value $prop $dataset 2>/dev/null) 1213 if [[ $? -ne 0 ]]; then 1214 log_note "Unable to get $prop property for dataset " \ 1215 "$dataset" 1216 return 1 1217 fi 1218 1219 echo "$prop_val" 1220 return 0 1221} 1222 1223# 1224# Simple function to get the specified property of pool. If unable to 1225# get the property then exits. 1226# 1227# Note property is in 'parsable' format (-p) 1228# 1229function get_pool_prop # property pool 1230{ 1231 typeset prop_val 1232 typeset prop=$1 1233 typeset pool=$2 1234 1235 if poolexists $pool ; then 1236 prop_val=$(zpool get -pH $prop $pool 2>/dev/null | tail -1 | \ 1237 awk '{print $3}') 1238 if [[ $? -ne 0 ]]; then 1239 log_note "Unable to get $prop property for pool " \ 1240 "$pool" 1241 return 1 1242 fi 1243 else 1244 log_note "Pool $pool not exists." 1245 return 1 1246 fi 1247 1248 echo "$prop_val" 1249 return 0 1250} 1251 1252# Return 0 if a pool exists; $? otherwise 1253# 1254# $1 - pool name 1255 1256function poolexists 1257{ 1258 typeset pool=$1 1259 1260 if [[ -z $pool ]]; then 1261 log_note "No pool name given." 1262 return 1 1263 fi 1264 1265 zpool get name "$pool" > /dev/null 2>&1 1266 return $? 1267} 1268 1269# Return 0 if all the specified datasets exist; $? otherwise 1270# 1271# $1-n dataset name 1272function datasetexists 1273{ 1274 if (($# == 0)); then 1275 log_note "No dataset name given." 1276 return 1 1277 fi 1278 1279 while (($# > 0)); do 1280 zfs get name $1 > /dev/null 2>&1 || \ 1281 return $? 1282 shift 1283 done 1284 1285 return 0 1286} 1287 1288# return 0 if none of the specified datasets exists, otherwise return 1. 1289# 1290# $1-n dataset name 1291function datasetnonexists 1292{ 1293 if (($# == 0)); then 1294 log_note "No dataset name given." 1295 return 1 1296 fi 1297 1298 while (($# > 0)); do 1299 zfs list -H -t filesystem,snapshot,volume $1 > /dev/null 2>&1 \ 1300 && return 1 1301 shift 1302 done 1303 1304 return 0 1305} 1306 1307function is_shared_freebsd 1308{ 1309 typeset fs=$1 1310 1311 pgrep -q mountd && showmount -E | grep -qx $fs 1312} 1313 1314function is_shared_illumos 1315{ 1316 typeset fs=$1 1317 typeset mtpt 1318 1319 for mtpt in `share | awk '{print $2}'` ; do 1320 if [[ $mtpt == $fs ]] ; then 1321 return 0 1322 fi 1323 done 1324 1325 typeset stat=$(svcs -H -o STA nfs/server:default) 1326 if [[ $stat != "ON" ]]; then 1327 log_note "Current nfs/server status: $stat" 1328 fi 1329 1330 return 1 1331} 1332 1333function is_shared_linux 1334{ 1335 typeset fs=$1 1336 typeset mtpt 1337 1338 for mtpt in `share | awk '{print $1}'` ; do 1339 if [[ $mtpt == $fs ]] ; then 1340 return 0 1341 fi 1342 done 1343 return 1 1344} 1345 1346# 1347# Given a mountpoint, or a dataset name, determine if it is shared via NFS. 1348# 1349# Returns 0 if shared, 1 otherwise. 1350# 1351function is_shared 1352{ 1353 typeset fs=$1 1354 typeset mtpt 1355 1356 if [[ $fs != "/"* ]] ; then 1357 if datasetnonexists "$fs" ; then 1358 return 1 1359 else 1360 mtpt=$(get_prop mountpoint "$fs") 1361 case $mtpt in 1362 none|legacy|-) return 1 1363 ;; 1364 *) fs=$mtpt 1365 ;; 1366 esac 1367 fi 1368 fi 1369 1370 case $(uname) in 1371 FreeBSD) is_shared_freebsd "$fs" ;; 1372 Linux) is_shared_linux "$fs" ;; 1373 *) is_shared_illumos "$fs" ;; 1374 esac 1375} 1376 1377function is_exported_illumos 1378{ 1379 typeset fs=$1 1380 typeset mtpt 1381 1382 for mtpt in `awk '{print $1}' /etc/dfs/sharetab` ; do 1383 if [[ $mtpt == $fs ]] ; then 1384 return 0 1385 fi 1386 done 1387 1388 return 1 1389} 1390 1391function is_exported_freebsd 1392{ 1393 typeset fs=$1 1394 typeset mtpt 1395 1396 for mtpt in `awk '{print $1}' /etc/zfs/exports` ; do 1397 if [[ $mtpt == $fs ]] ; then 1398 return 0 1399 fi 1400 done 1401 1402 return 1 1403} 1404 1405function is_exported_linux 1406{ 1407 typeset fs=$1 1408 typeset mtpt 1409 1410 for mtpt in `awk '{print $1}' /etc/exports.d/zfs.exports` ; do 1411 if [[ $mtpt == $fs ]] ; then 1412 return 0 1413 fi 1414 done 1415 1416 return 1 1417} 1418 1419# 1420# Given a mountpoint, or a dataset name, determine if it is exported via 1421# the os-specific NFS exports file. 1422# 1423# Returns 0 if exported, 1 otherwise. 1424# 1425function is_exported 1426{ 1427 typeset fs=$1 1428 typeset mtpt 1429 1430 if [[ $fs != "/"* ]] ; then 1431 if datasetnonexists "$fs" ; then 1432 return 1 1433 else 1434 mtpt=$(get_prop mountpoint "$fs") 1435 case $mtpt in 1436 none|legacy|-) return 1 1437 ;; 1438 *) fs=$mtpt 1439 ;; 1440 esac 1441 fi 1442 fi 1443 1444 case $(uname) in 1445 FreeBSD) is_exported_freebsd "$fs" ;; 1446 Linux) is_exported_linux "$fs" ;; 1447 *) is_exported_illumos "$fs" ;; 1448 esac 1449} 1450 1451# 1452# Given a dataset name determine if it is shared via SMB. 1453# 1454# Returns 0 if shared, 1 otherwise. 1455# 1456function is_shared_smb 1457{ 1458 typeset fs=$1 1459 typeset mtpt 1460 1461 if datasetnonexists "$fs" ; then 1462 return 1 1463 else 1464 fs=$(echo $fs | sed 's@/@_@g') 1465 fi 1466 1467 if is_linux; then 1468 for mtpt in `net usershare list | awk '{print $1}'` ; do 1469 if [[ $mtpt == $fs ]] ; then 1470 return 0 1471 fi 1472 done 1473 return 1 1474 else 1475 log_note "Currently unsupported by the test framework" 1476 return 1 1477 fi 1478} 1479 1480# 1481# Given a mountpoint, determine if it is not shared via NFS. 1482# 1483# Returns 0 if not shared, 1 otherwise. 1484# 1485function not_shared 1486{ 1487 typeset fs=$1 1488 1489 is_shared $fs 1490 if (($? == 0)); then 1491 return 1 1492 fi 1493 1494 return 0 1495} 1496 1497# 1498# Given a dataset determine if it is not shared via SMB. 1499# 1500# Returns 0 if not shared, 1 otherwise. 1501# 1502function not_shared_smb 1503{ 1504 typeset fs=$1 1505 1506 is_shared_smb $fs 1507 if (($? == 0)); then 1508 return 1 1509 fi 1510 1511 return 0 1512} 1513 1514# 1515# Helper function to unshare a mountpoint. 1516# 1517function unshare_fs #fs 1518{ 1519 typeset fs=$1 1520 1521 is_shared $fs || is_shared_smb $fs 1522 if (($? == 0)); then 1523 zfs unshare $fs || log_fail "zfs unshare $fs failed" 1524 fi 1525 1526 return 0 1527} 1528 1529# 1530# Helper function to share a NFS mountpoint. 1531# 1532function share_nfs #fs 1533{ 1534 typeset fs=$1 1535 1536 if is_linux; then 1537 is_shared $fs 1538 if (($? != 0)); then 1539 log_must share "*:$fs" 1540 fi 1541 else 1542 is_shared $fs 1543 if (($? != 0)); then 1544 log_must share -F nfs $fs 1545 fi 1546 fi 1547 1548 return 0 1549} 1550 1551# 1552# Helper function to unshare a NFS mountpoint. 1553# 1554function unshare_nfs #fs 1555{ 1556 typeset fs=$1 1557 1558 if is_linux; then 1559 is_shared $fs 1560 if (($? == 0)); then 1561 log_must unshare -u "*:$fs" 1562 fi 1563 else 1564 is_shared $fs 1565 if (($? == 0)); then 1566 log_must unshare -F nfs $fs 1567 fi 1568 fi 1569 1570 return 0 1571} 1572 1573# 1574# Helper function to show NFS shares. 1575# 1576function showshares_nfs 1577{ 1578 if is_linux; then 1579 share -v 1580 else 1581 share -F nfs 1582 fi 1583 1584 return 0 1585} 1586 1587# 1588# Helper function to show SMB shares. 1589# 1590function showshares_smb 1591{ 1592 if is_linux; then 1593 net usershare list 1594 else 1595 share -F smb 1596 fi 1597 1598 return 0 1599} 1600 1601function check_nfs 1602{ 1603 if is_linux; then 1604 share -s 1605 elif is_freebsd; then 1606 showmount -e 1607 else 1608 log_unsupported "Unknown platform" 1609 fi 1610 1611 if [[ $? -ne 0 ]]; then 1612 log_unsupported "The NFS utilities are not installed" 1613 fi 1614} 1615 1616# 1617# Check NFS server status and trigger it online. 1618# 1619function setup_nfs_server 1620{ 1621 # Cannot share directory in non-global zone. 1622 # 1623 if ! is_global_zone; then 1624 log_note "Cannot trigger NFS server by sharing in LZ." 1625 return 1626 fi 1627 1628 if is_linux; then 1629 # 1630 # Re-synchronize /var/lib/nfs/etab with /etc/exports and 1631 # /etc/exports.d./* to provide a clean test environment. 1632 # 1633 log_must share -r 1634 1635 log_note "NFS server must be started prior to running ZTS." 1636 return 1637 elif is_freebsd; then 1638 kill -s HUP $(cat /var/run/mountd.pid) 1639 1640 log_note "NFS server must be started prior to running ZTS." 1641 return 1642 fi 1643 1644 typeset nfs_fmri="svc:/network/nfs/server:default" 1645 if [[ $(svcs -Ho STA $nfs_fmri) != "ON" ]]; then 1646 # 1647 # Only really sharing operation can enable NFS server 1648 # to online permanently. 1649 # 1650 typeset dummy=/tmp/dummy 1651 1652 if [[ -d $dummy ]]; then 1653 log_must rm -rf $dummy 1654 fi 1655 1656 log_must mkdir $dummy 1657 log_must share $dummy 1658 1659 # 1660 # Waiting for fmri's status to be the final status. 1661 # Otherwise, in transition, an asterisk (*) is appended for 1662 # instances, unshare will reverse status to 'DIS' again. 1663 # 1664 # Waiting for 1's at least. 1665 # 1666 log_must sleep 1 1667 timeout=10 1668 while [[ timeout -ne 0 && $(svcs -Ho STA $nfs_fmri) == *'*' ]] 1669 do 1670 log_must sleep 1 1671 1672 ((timeout -= 1)) 1673 done 1674 1675 log_must unshare $dummy 1676 log_must rm -rf $dummy 1677 fi 1678 1679 log_note "Current NFS status: '$(svcs -Ho STA,FMRI $nfs_fmri)'" 1680} 1681 1682# 1683# To verify whether calling process is in global zone 1684# 1685# Return 0 if in global zone, 1 in non-global zone 1686# 1687function is_global_zone 1688{ 1689 if is_linux || is_freebsd; then 1690 return 0 1691 else 1692 typeset cur_zone=$(zonename 2>/dev/null) 1693 if [[ $cur_zone != "global" ]]; then 1694 return 1 1695 fi 1696 return 0 1697 fi 1698} 1699 1700# 1701# Verify whether test is permitted to run from 1702# global zone, local zone, or both 1703# 1704# $1 zone limit, could be "global", "local", or "both"(no limit) 1705# 1706# Return 0 if permitted, otherwise exit with log_unsupported 1707# 1708function verify_runnable # zone limit 1709{ 1710 typeset limit=$1 1711 1712 [[ -z $limit ]] && return 0 1713 1714 if is_global_zone ; then 1715 case $limit in 1716 global|both) 1717 ;; 1718 local) log_unsupported "Test is unable to run from "\ 1719 "global zone." 1720 ;; 1721 *) log_note "Warning: unknown limit $limit - " \ 1722 "use both." 1723 ;; 1724 esac 1725 else 1726 case $limit in 1727 local|both) 1728 ;; 1729 global) log_unsupported "Test is unable to run from "\ 1730 "local zone." 1731 ;; 1732 *) log_note "Warning: unknown limit $limit - " \ 1733 "use both." 1734 ;; 1735 esac 1736 1737 reexport_pool 1738 fi 1739 1740 return 0 1741} 1742 1743# Return 0 if create successfully or the pool exists; $? otherwise 1744# Note: In local zones, this function should return 0 silently. 1745# 1746# $1 - pool name 1747# $2-n - [keyword] devs_list 1748 1749function create_pool #pool devs_list 1750{ 1751 typeset pool=${1%%/*} 1752 1753 shift 1754 1755 if [[ -z $pool ]]; then 1756 log_note "Missing pool name." 1757 return 1 1758 fi 1759 1760 if poolexists $pool ; then 1761 destroy_pool $pool 1762 fi 1763 1764 if is_global_zone ; then 1765 [[ -d /$pool ]] && rm -rf /$pool 1766 log_must zpool create -f $pool $@ 1767 fi 1768 1769 return 0 1770} 1771 1772# Return 0 if destroy successfully or the pool exists; $? otherwise 1773# Note: In local zones, this function should return 0 silently. 1774# 1775# $1 - pool name 1776# Destroy pool with the given parameters. 1777 1778function destroy_pool #pool 1779{ 1780 typeset pool=${1%%/*} 1781 typeset mtpt 1782 1783 if [[ -z $pool ]]; then 1784 log_note "No pool name given." 1785 return 1 1786 fi 1787 1788 if is_global_zone ; then 1789 if poolexists "$pool" ; then 1790 mtpt=$(get_prop mountpoint "$pool") 1791 1792 # At times, syseventd/udev activity can cause attempts 1793 # to destroy a pool to fail with EBUSY. We retry a few 1794 # times allowing failures before requiring the destroy 1795 # to succeed. 1796 log_must_busy zpool destroy -f $pool 1797 1798 [[ -d $mtpt ]] && \ 1799 log_must rm -rf $mtpt 1800 else 1801 log_note "Pool does not exist. ($pool)" 1802 return 1 1803 fi 1804 fi 1805 1806 return 0 1807} 1808 1809# Return 0 if created successfully; $? otherwise 1810# 1811# $1 - dataset name 1812# $2-n - dataset options 1813 1814function create_dataset #dataset dataset_options 1815{ 1816 typeset dataset=$1 1817 1818 shift 1819 1820 if [[ -z $dataset ]]; then 1821 log_note "Missing dataset name." 1822 return 1 1823 fi 1824 1825 if datasetexists $dataset ; then 1826 destroy_dataset $dataset 1827 fi 1828 1829 log_must zfs create $@ $dataset 1830 1831 return 0 1832} 1833 1834# Return 0 if destroy successfully or the dataset exists; $? otherwise 1835# Note: In local zones, this function should return 0 silently. 1836# 1837# $1 - dataset name 1838# $2 - custom arguments for zfs destroy 1839# Destroy dataset with the given parameters. 1840 1841function destroy_dataset #dataset #args 1842{ 1843 typeset dataset=$1 1844 typeset mtpt 1845 typeset args=${2:-""} 1846 1847 if [[ -z $dataset ]]; then 1848 log_note "No dataset name given." 1849 return 1 1850 fi 1851 1852 if is_global_zone ; then 1853 if datasetexists "$dataset" ; then 1854 mtpt=$(get_prop mountpoint "$dataset") 1855 log_must_busy zfs destroy $args $dataset 1856 1857 [[ -d $mtpt ]] && \ 1858 log_must rm -rf $mtpt 1859 else 1860 log_note "Dataset does not exist. ($dataset)" 1861 return 1 1862 fi 1863 fi 1864 1865 return 0 1866} 1867 1868# 1869# Firstly, create a pool with 5 datasets. Then, create a single zone and 1870# export the 5 datasets to it. In addition, we also add a ZFS filesystem 1871# and a zvol device to the zone. 1872# 1873# $1 zone name 1874# $2 zone root directory prefix 1875# $3 zone ip 1876# 1877function zfs_zones_setup #zone_name zone_root zone_ip 1878{ 1879 typeset zone_name=${1:-$(hostname)-z} 1880 typeset zone_root=${2:-"/zone_root"} 1881 typeset zone_ip=${3:-"10.1.1.10"} 1882 typeset prefix_ctr=$ZONE_CTR 1883 typeset pool_name=$ZONE_POOL 1884 typeset -i cntctr=5 1885 typeset -i i=0 1886 1887 # Create pool and 5 container within it 1888 # 1889 [[ -d /$pool_name ]] && rm -rf /$pool_name 1890 log_must zpool create -f $pool_name $DISKS 1891 while ((i < cntctr)); do 1892 log_must zfs create $pool_name/$prefix_ctr$i 1893 ((i += 1)) 1894 done 1895 1896 # create a zvol 1897 log_must zfs create -V 1g $pool_name/zone_zvol 1898 block_device_wait 1899 1900 # 1901 # Add slog device for pool 1902 # 1903 typeset sdevs="$TEST_BASE_DIR/sdev1 $TEST_BASE_DIR/sdev2" 1904 log_must mkfile $MINVDEVSIZE $sdevs 1905 log_must zpool add $pool_name log mirror $sdevs 1906 1907 # this isn't supported just yet. 1908 # Create a filesystem. In order to add this to 1909 # the zone, it must have it's mountpoint set to 'legacy' 1910 # log_must zfs create $pool_name/zfs_filesystem 1911 # log_must zfs set mountpoint=legacy $pool_name/zfs_filesystem 1912 1913 [[ -d $zone_root ]] && \ 1914 log_must rm -rf $zone_root/$zone_name 1915 [[ ! -d $zone_root ]] && \ 1916 log_must mkdir -p -m 0700 $zone_root/$zone_name 1917 1918 # Create zone configure file and configure the zone 1919 # 1920 typeset zone_conf=/tmp/zone_conf.$$ 1921 echo "create" > $zone_conf 1922 echo "set zonepath=$zone_root/$zone_name" >> $zone_conf 1923 echo "set autoboot=true" >> $zone_conf 1924 i=0 1925 while ((i < cntctr)); do 1926 echo "add dataset" >> $zone_conf 1927 echo "set name=$pool_name/$prefix_ctr$i" >> \ 1928 $zone_conf 1929 echo "end" >> $zone_conf 1930 ((i += 1)) 1931 done 1932 1933 # add our zvol to the zone 1934 echo "add device" >> $zone_conf 1935 echo "set match=/dev/zvol/dsk/$pool_name/zone_zvol" >> $zone_conf 1936 echo "end" >> $zone_conf 1937 1938 # add a corresponding zvol rdsk to the zone 1939 echo "add device" >> $zone_conf 1940 echo "set match=$ZVOL_RDEVDIR/$pool_name/zone_zvol" >> $zone_conf 1941 echo "end" >> $zone_conf 1942 1943 # once it's supported, we'll add our filesystem to the zone 1944 # echo "add fs" >> $zone_conf 1945 # echo "set type=zfs" >> $zone_conf 1946 # echo "set special=$pool_name/zfs_filesystem" >> $zone_conf 1947 # echo "set dir=/export/zfs_filesystem" >> $zone_conf 1948 # echo "end" >> $zone_conf 1949 1950 echo "verify" >> $zone_conf 1951 echo "commit" >> $zone_conf 1952 log_must zonecfg -z $zone_name -f $zone_conf 1953 log_must rm -f $zone_conf 1954 1955 # Install the zone 1956 zoneadm -z $zone_name install 1957 if (($? == 0)); then 1958 log_note "SUCCESS: zoneadm -z $zone_name install" 1959 else 1960 log_fail "FAIL: zoneadm -z $zone_name install" 1961 fi 1962 1963 # Install sysidcfg file 1964 # 1965 typeset sysidcfg=$zone_root/$zone_name/root/etc/sysidcfg 1966 echo "system_locale=C" > $sysidcfg 1967 echo "terminal=dtterm" >> $sysidcfg 1968 echo "network_interface=primary {" >> $sysidcfg 1969 echo "hostname=$zone_name" >> $sysidcfg 1970 echo "}" >> $sysidcfg 1971 echo "name_service=NONE" >> $sysidcfg 1972 echo "root_password=mo791xfZ/SFiw" >> $sysidcfg 1973 echo "security_policy=NONE" >> $sysidcfg 1974 echo "timezone=US/Eastern" >> $sysidcfg 1975 1976 # Boot this zone 1977 log_must zoneadm -z $zone_name boot 1978} 1979 1980# 1981# Reexport TESTPOOL & TESTPOOL(1-4) 1982# 1983function reexport_pool 1984{ 1985 typeset -i cntctr=5 1986 typeset -i i=0 1987 1988 while ((i < cntctr)); do 1989 if ((i == 0)); then 1990 TESTPOOL=$ZONE_POOL/$ZONE_CTR$i 1991 if ! ismounted $TESTPOOL; then 1992 log_must zfs mount $TESTPOOL 1993 fi 1994 else 1995 eval TESTPOOL$i=$ZONE_POOL/$ZONE_CTR$i 1996 if eval ! ismounted \$TESTPOOL$i; then 1997 log_must eval zfs mount \$TESTPOOL$i 1998 fi 1999 fi 2000 ((i += 1)) 2001 done 2002} 2003 2004# 2005# Verify a given disk or pool state 2006# 2007# Return 0 is pool/disk matches expected state, 1 otherwise 2008# 2009function check_state # pool disk state{online,offline,degraded} 2010{ 2011 typeset pool=$1 2012 typeset disk=${2#$DEV_DSKDIR/} 2013 typeset state=$3 2014 2015 [[ -z $pool ]] || [[ -z $state ]] \ 2016 && log_fail "Arguments invalid or missing" 2017 2018 if [[ -z $disk ]]; then 2019 #check pool state only 2020 zpool get -H -o value health $pool \ 2021 | grep -i "$state" > /dev/null 2>&1 2022 else 2023 zpool status -v $pool | grep "$disk" \ 2024 | grep -i "$state" > /dev/null 2>&1 2025 fi 2026 2027 return $? 2028} 2029 2030# 2031# Get the mountpoint of snapshot 2032# For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap> 2033# as its mountpoint 2034# 2035function snapshot_mountpoint 2036{ 2037 typeset dataset=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 2038 2039 if [[ $dataset != *@* ]]; then 2040 log_fail "Error name of snapshot '$dataset'." 2041 fi 2042 2043 typeset fs=${dataset%@*} 2044 typeset snap=${dataset#*@} 2045 2046 if [[ -z $fs || -z $snap ]]; then 2047 log_fail "Error name of snapshot '$dataset'." 2048 fi 2049 2050 echo $(get_prop mountpoint $fs)/.zfs/snapshot/$snap 2051} 2052 2053# 2054# Given a device and 'ashift' value verify it's correctly set on every label 2055# 2056function verify_ashift # device ashift 2057{ 2058 typeset device="$1" 2059 typeset ashift="$2" 2060 2061 zdb -e -lll $device | awk -v ashift=$ashift '/ashift: / { 2062 if (ashift != $2) 2063 exit 1; 2064 else 2065 count++; 2066 } END { 2067 if (count != 4) 2068 exit 1; 2069 else 2070 exit 0; 2071 }' 2072 2073 return $? 2074} 2075 2076# 2077# Given a pool and file system, this function will verify the file system 2078# using the zdb internal tool. Note that the pool is exported and imported 2079# to ensure it has consistent state. 2080# 2081function verify_filesys # pool filesystem dir 2082{ 2083 typeset pool="$1" 2084 typeset filesys="$2" 2085 typeset zdbout="/tmp/zdbout.$$" 2086 2087 shift 2088 shift 2089 typeset dirs=$@ 2090 typeset search_path="" 2091 2092 log_note "Calling zdb to verify filesystem '$filesys'" 2093 zfs unmount -a > /dev/null 2>&1 2094 log_must zpool export $pool 2095 2096 if [[ -n $dirs ]] ; then 2097 for dir in $dirs ; do 2098 search_path="$search_path -d $dir" 2099 done 2100 fi 2101 2102 log_must zpool import $search_path $pool 2103 2104 zdb -cudi $filesys > $zdbout 2>&1 2105 if [[ $? != 0 ]]; then 2106 log_note "Output: zdb -cudi $filesys" 2107 cat $zdbout 2108 log_fail "zdb detected errors with: '$filesys'" 2109 fi 2110 2111 log_must zfs mount -a 2112 log_must rm -rf $zdbout 2113} 2114 2115# 2116# Given a pool issue a scrub and verify that no checksum errors are reported. 2117# 2118function verify_pool 2119{ 2120 typeset pool=${1:-$TESTPOOL} 2121 2122 log_must zpool scrub $pool 2123 log_must wait_scrubbed $pool 2124 2125 typeset -i cksum=$(zpool status $pool | awk ' 2126 !NF { isvdev = 0 } 2127 isvdev { errors += $NF } 2128 /CKSUM$/ { isvdev = 1 } 2129 END { print errors } 2130 ') 2131 if [[ $cksum != 0 ]]; then 2132 log_must zpool status -v 2133 log_fail "Unexpected CKSUM errors found on $pool ($cksum)" 2134 fi 2135} 2136 2137# 2138# Given a pool, and this function list all disks in the pool 2139# 2140function get_disklist # pool 2141{ 2142 typeset disklist="" 2143 2144 disklist=$(zpool iostat -v $1 | nawk '(NR >4) {print $1}' | \ 2145 grep -v "\-\-\-\-\-" | \ 2146 egrep -v -e "^(mirror|raidz[1-3]|spare|log|cache|special|dedup)$") 2147 2148 echo $disklist 2149} 2150 2151# 2152# Given a pool, and this function list all disks in the pool with their full 2153# path (like "/dev/sda" instead of "sda"). 2154# 2155function get_disklist_fullpath # pool 2156{ 2157 args="-P $1" 2158 get_disklist $args 2159} 2160 2161 2162 2163# /** 2164# This function kills a given list of processes after a time period. We use 2165# this in the stress tests instead of STF_TIMEOUT so that we can have processes 2166# run for a fixed amount of time, yet still pass. Tests that hit STF_TIMEOUT 2167# would be listed as FAIL, which we don't want : we're happy with stress tests 2168# running for a certain amount of time, then finishing. 2169# 2170# @param $1 the time in seconds after which we should terminate these processes 2171# @param $2..$n the processes we wish to terminate. 2172# */ 2173function stress_timeout 2174{ 2175 typeset -i TIMEOUT=$1 2176 shift 2177 typeset cpids="$@" 2178 2179 log_note "Waiting for child processes($cpids). " \ 2180 "It could last dozens of minutes, please be patient ..." 2181 log_must sleep $TIMEOUT 2182 2183 log_note "Killing child processes after ${TIMEOUT} stress timeout." 2184 typeset pid 2185 for pid in $cpids; do 2186 ps -p $pid > /dev/null 2>&1 2187 if (($? == 0)); then 2188 log_must kill -USR1 $pid 2189 fi 2190 done 2191} 2192 2193# 2194# Verify a given hotspare disk is inuse or avail 2195# 2196# Return 0 is pool/disk matches expected state, 1 otherwise 2197# 2198function check_hotspare_state # pool disk state{inuse,avail} 2199{ 2200 typeset pool=$1 2201 typeset disk=${2#$DEV_DSKDIR/} 2202 typeset state=$3 2203 2204 cur_state=$(get_device_state $pool $disk "spares") 2205 2206 if [[ $state != ${cur_state} ]]; then 2207 return 1 2208 fi 2209 return 0 2210} 2211 2212# 2213# Wait until a hotspare transitions to a given state or times out. 2214# 2215# Return 0 when pool/disk matches expected state, 1 on timeout. 2216# 2217function wait_hotspare_state # pool disk state timeout 2218{ 2219 typeset pool=$1 2220 typeset disk=${2#*$DEV_DSKDIR/} 2221 typeset state=$3 2222 typeset timeout=${4:-60} 2223 typeset -i i=0 2224 2225 while [[ $i -lt $timeout ]]; do 2226 if check_hotspare_state $pool $disk $state; then 2227 return 0 2228 fi 2229 2230 i=$((i+1)) 2231 sleep 1 2232 done 2233 2234 return 1 2235} 2236 2237# 2238# Verify a given slog disk is inuse or avail 2239# 2240# Return 0 is pool/disk matches expected state, 1 otherwise 2241# 2242function check_slog_state # pool disk state{online,offline,unavail} 2243{ 2244 typeset pool=$1 2245 typeset disk=${2#$DEV_DSKDIR/} 2246 typeset state=$3 2247 2248 cur_state=$(get_device_state $pool $disk "logs") 2249 2250 if [[ $state != ${cur_state} ]]; then 2251 return 1 2252 fi 2253 return 0 2254} 2255 2256# 2257# Verify a given vdev disk is inuse or avail 2258# 2259# Return 0 is pool/disk matches expected state, 1 otherwise 2260# 2261function check_vdev_state # pool disk state{online,offline,unavail} 2262{ 2263 typeset pool=$1 2264 typeset disk=${2#*$DEV_DSKDIR/} 2265 typeset state=$3 2266 2267 cur_state=$(get_device_state $pool $disk) 2268 2269 if [[ $state != ${cur_state} ]]; then 2270 return 1 2271 fi 2272 return 0 2273} 2274 2275# 2276# Wait until a vdev transitions to a given state or times out. 2277# 2278# Return 0 when pool/disk matches expected state, 1 on timeout. 2279# 2280function wait_vdev_state # pool disk state timeout 2281{ 2282 typeset pool=$1 2283 typeset disk=${2#*$DEV_DSKDIR/} 2284 typeset state=$3 2285 typeset timeout=${4:-60} 2286 typeset -i i=0 2287 2288 while [[ $i -lt $timeout ]]; do 2289 if check_vdev_state $pool $disk $state; then 2290 return 0 2291 fi 2292 2293 i=$((i+1)) 2294 sleep 1 2295 done 2296 2297 return 1 2298} 2299 2300# 2301# Check the output of 'zpool status -v <pool>', 2302# and to see if the content of <token> contain the <keyword> specified. 2303# 2304# Return 0 is contain, 1 otherwise 2305# 2306function check_pool_status # pool token keyword <verbose> 2307{ 2308 typeset pool=$1 2309 typeset token=$2 2310 typeset keyword=$3 2311 typeset verbose=${4:-false} 2312 2313 scan=$(zpool status -v "$pool" 2>/dev/null | nawk -v token="$token:" ' 2314 ($1==token) {print $0}') 2315 if [[ $verbose == true ]]; then 2316 log_note $scan 2317 fi 2318 echo $scan | egrep -i "$keyword" > /dev/null 2>&1 2319 2320 return $? 2321} 2322 2323# 2324# The following functions are instance of check_pool_status() 2325# is_pool_resilvering - to check if the pool resilver is in progress 2326# is_pool_resilvered - to check if the pool resilver is completed 2327# is_pool_scrubbing - to check if the pool scrub is in progress 2328# is_pool_scrubbed - to check if the pool scrub is completed 2329# is_pool_scrub_stopped - to check if the pool scrub is stopped 2330# is_pool_scrub_paused - to check if the pool scrub has paused 2331# is_pool_removing - to check if the pool removing is a vdev 2332# is_pool_removed - to check if the pool remove is completed 2333# is_pool_discarding - to check if the pool checkpoint is being discarded 2334# 2335function is_pool_resilvering #pool <verbose> 2336{ 2337 check_pool_status "$1" "scan" \ 2338 "resilver[ ()0-9A-Za-z:_-]* in progress since" $2 2339 return $? 2340} 2341 2342function is_pool_resilvered #pool <verbose> 2343{ 2344 check_pool_status "$1" "scan" "resilvered " $2 2345 return $? 2346} 2347 2348function is_pool_scrubbing #pool <verbose> 2349{ 2350 check_pool_status "$1" "scan" "scrub in progress since " $2 2351 return $? 2352} 2353 2354function is_pool_scrubbed #pool <verbose> 2355{ 2356 check_pool_status "$1" "scan" "scrub repaired" $2 2357 return $? 2358} 2359 2360function is_pool_scrub_stopped #pool <verbose> 2361{ 2362 check_pool_status "$1" "scan" "scrub canceled" $2 2363 return $? 2364} 2365 2366function is_pool_scrub_paused #pool <verbose> 2367{ 2368 check_pool_status "$1" "scan" "scrub paused since " $2 2369 return $? 2370} 2371 2372function is_pool_removing #pool 2373{ 2374 check_pool_status "$1" "remove" "in progress since " 2375 return $? 2376} 2377 2378function is_pool_removed #pool 2379{ 2380 check_pool_status "$1" "remove" "completed on" 2381 return $? 2382} 2383 2384function is_pool_discarding #pool 2385{ 2386 check_pool_status "$1" "checkpoint" "discarding" 2387 return $? 2388} 2389 2390function wait_for_degraded 2391{ 2392 typeset pool=$1 2393 typeset timeout=${2:-30} 2394 typeset t0=$SECONDS 2395 2396 while :; do 2397 [[ $(get_pool_prop health $pool) == "DEGRADED" ]] && break 2398 log_note "$pool is not yet degraded." 2399 sleep 1 2400 if ((SECONDS - t0 > $timeout)); then 2401 log_note "$pool not degraded after $timeout seconds." 2402 return 1 2403 fi 2404 done 2405 2406 return 0 2407} 2408 2409# 2410# Use create_pool()/destroy_pool() to clean up the information in 2411# in the given disk to avoid slice overlapping. 2412# 2413function cleanup_devices #vdevs 2414{ 2415 typeset pool="foopool$$" 2416 2417 for vdev in $@; do 2418 zero_partitions $vdev 2419 done 2420 2421 poolexists $pool && destroy_pool $pool 2422 create_pool $pool $@ 2423 destroy_pool $pool 2424 2425 return 0 2426} 2427 2428#/** 2429# A function to find and locate free disks on a system or from given 2430# disks as the parameter. It works by locating disks that are in use 2431# as swap devices and dump devices, and also disks listed in /etc/vfstab 2432# 2433# $@ given disks to find which are free, default is all disks in 2434# the test system 2435# 2436# @return a string containing the list of available disks 2437#*/ 2438function find_disks 2439{ 2440 # Trust provided list, no attempt is made to locate unused devices. 2441 if is_linux || is_freebsd; then 2442 echo "$@" 2443 return 2444 fi 2445 2446 2447 sfi=/tmp/swaplist.$$ 2448 dmpi=/tmp/dumpdev.$$ 2449 max_finddisksnum=${MAX_FINDDISKSNUM:-6} 2450 2451 swap -l > $sfi 2452 dumpadm > $dmpi 2>/dev/null 2453 2454# write an awk script that can process the output of format 2455# to produce a list of disks we know about. Note that we have 2456# to escape "$2" so that the shell doesn't interpret it while 2457# we're creating the awk script. 2458# ------------------- 2459 cat > /tmp/find_disks.awk <<EOF 2460#!/bin/nawk -f 2461 BEGIN { FS="."; } 2462 2463 /^Specify disk/{ 2464 searchdisks=0; 2465 } 2466 2467 { 2468 if (searchdisks && \$2 !~ "^$"){ 2469 split(\$2,arr," "); 2470 print arr[1]; 2471 } 2472 } 2473 2474 /^AVAILABLE DISK SELECTIONS:/{ 2475 searchdisks=1; 2476 } 2477EOF 2478#--------------------- 2479 2480 chmod 755 /tmp/find_disks.awk 2481 disks=${@:-$(echo "" | format -e 2>/dev/null | /tmp/find_disks.awk)} 2482 rm /tmp/find_disks.awk 2483 2484 unused="" 2485 for disk in $disks; do 2486 # Check for mounted 2487 grep "${disk}[sp]" /etc/mnttab >/dev/null 2488 (($? == 0)) && continue 2489 # Check for swap 2490 grep "${disk}[sp]" $sfi >/dev/null 2491 (($? == 0)) && continue 2492 # check for dump device 2493 grep "${disk}[sp]" $dmpi >/dev/null 2494 (($? == 0)) && continue 2495 # check to see if this disk hasn't been explicitly excluded 2496 # by a user-set environment variable 2497 echo "${ZFS_HOST_DEVICES_IGNORE}" | grep "${disk}" > /dev/null 2498 (($? == 0)) && continue 2499 unused_candidates="$unused_candidates $disk" 2500 done 2501 rm $sfi 2502 rm $dmpi 2503 2504# now just check to see if those disks do actually exist 2505# by looking for a device pointing to the first slice in 2506# each case. limit the number to max_finddisksnum 2507 count=0 2508 for disk in $unused_candidates; do 2509 if is_disk_device $DEV_DSKDIR/${disk}s0 && \ 2510 [ $count -lt $max_finddisksnum ]; then 2511 unused="$unused $disk" 2512 # do not impose limit if $@ is provided 2513 [[ -z $@ ]] && ((count = count + 1)) 2514 fi 2515 done 2516 2517# finally, return our disk list 2518 echo $unused 2519} 2520 2521function add_user_freebsd #<group_name> <user_name> <basedir> 2522{ 2523 typeset group=$1 2524 typeset user=$2 2525 typeset basedir=$3 2526 2527 # Check to see if the user exists. 2528 if id $user > /dev/null 2>&1; then 2529 return 0 2530 fi 2531 2532 # Assign 1000 as the base uid 2533 typeset -i uid=1000 2534 while true; do 2535 typeset -i ret 2536 pw useradd -u $uid -g $group -d $basedir/$user -m -n $user 2537 ret=$? 2538 case $ret in 2539 0) break ;; 2540 # The uid is not unique 2541 65) ((uid += 1)) ;; 2542 *) return 1 ;; 2543 esac 2544 if [[ $uid == 65000 ]]; then 2545 log_fail "No user id available under 65000 for $user" 2546 fi 2547 done 2548 2549 # Silence MOTD 2550 touch $basedir/$user/.hushlogin 2551 2552 return 0 2553} 2554 2555# 2556# Delete the specified user. 2557# 2558# $1 login name 2559# 2560function del_user_freebsd #<logname> 2561{ 2562 typeset user=$1 2563 2564 if id $user > /dev/null 2>&1; then 2565 log_must pw userdel $user 2566 fi 2567 2568 return 0 2569} 2570 2571# 2572# Select valid gid and create specified group. 2573# 2574# $1 group name 2575# 2576function add_group_freebsd #<group_name> 2577{ 2578 typeset group=$1 2579 2580 # See if the group already exists. 2581 if pw groupshow $group >/dev/null 2>&1; then 2582 return 0 2583 fi 2584 2585 # Assign 1000 as the base gid 2586 typeset -i gid=1000 2587 while true; do 2588 pw groupadd -g $gid -n $group > /dev/null 2>&1 2589 typeset -i ret=$? 2590 case $ret in 2591 0) return 0 ;; 2592 # The gid is not unique 2593 65) ((gid += 1)) ;; 2594 *) return 1 ;; 2595 esac 2596 if [[ $gid == 65000 ]]; then 2597 log_fail "No user id available under 65000 for $group" 2598 fi 2599 done 2600} 2601 2602# 2603# Delete the specified group. 2604# 2605# $1 group name 2606# 2607function del_group_freebsd #<group_name> 2608{ 2609 typeset group=$1 2610 2611 pw groupdel -n $group > /dev/null 2>&1 2612 typeset -i ret=$? 2613 case $ret in 2614 # Group does not exist, or was deleted successfully. 2615 0|6|65) return 0 ;; 2616 # Name already exists as a group name 2617 9) log_must pw groupdel $group ;; 2618 *) return 1 ;; 2619 esac 2620 2621 return 0 2622} 2623 2624function add_user_illumos #<group_name> <user_name> <basedir> 2625{ 2626 typeset group=$1 2627 typeset user=$2 2628 typeset basedir=$3 2629 2630 log_must useradd -g $group -d $basedir/$user -m $user 2631 2632 return 0 2633} 2634 2635function del_user_illumos #<user_name> 2636{ 2637 typeset user=$1 2638 2639 if id $user > /dev/null 2>&1; then 2640 log_must_retry "currently used" 6 userdel $user 2641 fi 2642 2643 return 0 2644} 2645 2646function add_group_illumos #<group_name> 2647{ 2648 typeset group=$1 2649 2650 typeset -i gid=100 2651 while true; do 2652 groupadd -g $gid $group > /dev/null 2>&1 2653 typeset -i ret=$? 2654 case $ret in 2655 0) return 0 ;; 2656 # The gid is not unique 2657 4) ((gid += 1)) ;; 2658 *) return 1 ;; 2659 esac 2660 done 2661} 2662 2663function del_group_illumos #<group_name> 2664{ 2665 typeset group=$1 2666 2667 groupmod -n $grp $grp > /dev/null 2>&1 2668 typeset -i ret=$? 2669 case $ret in 2670 # Group does not exist. 2671 6) return 0 ;; 2672 # Name already exists as a group name 2673 9) log_must groupdel $grp ;; 2674 *) return 1 ;; 2675 esac 2676} 2677 2678function add_user_linux #<group_name> <user_name> <basedir> 2679{ 2680 typeset group=$1 2681 typeset user=$2 2682 typeset basedir=$3 2683 2684 log_must useradd -g $group -d $basedir/$user -m $user 2685 2686 # Add new users to the same group and the command line utils. 2687 # This allows them to be run out of the original users home 2688 # directory as long as it permissioned to be group readable. 2689 cmd_group=$(stat --format="%G" $(which zfs)) 2690 log_must usermod -a -G $cmd_group $user 2691 2692 return 0 2693} 2694 2695function del_user_linux #<user_name> 2696{ 2697 typeset user=$1 2698 2699 if id $user > /dev/null 2>&1; then 2700 log_must_retry "currently used" 6 userdel $user 2701 fi 2702 2703 return 0 2704} 2705 2706function add_group_linux #<group_name> 2707{ 2708 typeset group=$1 2709 2710 # Assign 100 as the base gid, a larger value is selected for 2711 # Linux because for many distributions 1000 and under are reserved. 2712 while true; do 2713 groupadd $group > /dev/null 2>&1 2714 typeset -i ret=$? 2715 case $ret in 2716 0) return 0 ;; 2717 *) return 1 ;; 2718 esac 2719 done 2720} 2721 2722function del_group_linux #<group_name> 2723{ 2724 typeset group=$1 2725 2726 getent group $group > /dev/null 2>&1 2727 typeset -i ret=$? 2728 case $ret in 2729 # Group does not exist. 2730 2) return 0 ;; 2731 # Name already exists as a group name 2732 0) log_must groupdel $group ;; 2733 *) return 1 ;; 2734 esac 2735 2736 return 0 2737} 2738 2739# 2740# Add specified user to specified group 2741# 2742# $1 group name 2743# $2 user name 2744# $3 base of the homedir (optional) 2745# 2746function add_user #<group_name> <user_name> <basedir> 2747{ 2748 typeset group=$1 2749 typeset user=$2 2750 typeset basedir=${3:-"/var/tmp"} 2751 2752 if ((${#group} == 0 || ${#user} == 0)); then 2753 log_fail "group name or user name are not defined." 2754 fi 2755 2756 case $(uname) in 2757 FreeBSD) 2758 add_user_freebsd "$group" "$user" "$basedir" 2759 ;; 2760 Linux) 2761 add_user_linux "$group" "$user" "$basedir" 2762 ;; 2763 *) 2764 add_user_illumos "$group" "$user" "$basedir" 2765 ;; 2766 esac 2767 2768 return 0 2769} 2770 2771# 2772# Delete the specified user. 2773# 2774# $1 login name 2775# $2 base of the homedir (optional) 2776# 2777function del_user #<logname> <basedir> 2778{ 2779 typeset user=$1 2780 typeset basedir=${2:-"/var/tmp"} 2781 2782 if ((${#user} == 0)); then 2783 log_fail "login name is necessary." 2784 fi 2785 2786 case $(uname) in 2787 FreeBSD) 2788 del_user_freebsd "$user" 2789 ;; 2790 Linux) 2791 del_user_linux "$user" 2792 ;; 2793 *) 2794 del_user_illumos "$user" 2795 ;; 2796 esac 2797 2798 [[ -d $basedir/$user ]] && rm -fr $basedir/$user 2799 2800 return 0 2801} 2802 2803# 2804# Select valid gid and create specified group. 2805# 2806# $1 group name 2807# 2808function add_group #<group_name> 2809{ 2810 typeset group=$1 2811 2812 if ((${#group} == 0)); then 2813 log_fail "group name is necessary." 2814 fi 2815 2816 case $(uname) in 2817 FreeBSD) 2818 add_group_freebsd "$group" 2819 ;; 2820 Linux) 2821 add_group_linux "$group" 2822 ;; 2823 *) 2824 add_group_illumos "$group" 2825 ;; 2826 esac 2827 2828 return 0 2829} 2830 2831# 2832# Delete the specified group. 2833# 2834# $1 group name 2835# 2836function del_group #<group_name> 2837{ 2838 typeset group=$1 2839 2840 if ((${#group} == 0)); then 2841 log_fail "group name is necessary." 2842 fi 2843 2844 case $(uname) in 2845 FreeBSD) 2846 del_group_freebsd "$group" 2847 ;; 2848 Linux) 2849 del_group_linux "$group" 2850 ;; 2851 *) 2852 del_group_illumos "$group" 2853 ;; 2854 esac 2855 2856 return 0 2857} 2858 2859# 2860# This function will return true if it's safe to destroy the pool passed 2861# as argument 1. It checks for pools based on zvols and files, and also 2862# files contained in a pool that may have a different mountpoint. 2863# 2864function safe_to_destroy_pool { # $1 the pool name 2865 2866 typeset pool="" 2867 typeset DONT_DESTROY="" 2868 2869 # We check that by deleting the $1 pool, we're not 2870 # going to pull the rug out from other pools. Do this 2871 # by looking at all other pools, ensuring that they 2872 # aren't built from files or zvols contained in this pool. 2873 2874 for pool in $(zpool list -H -o name) 2875 do 2876 ALTMOUNTPOOL="" 2877 2878 # this is a list of the top-level directories in each of the 2879 # files that make up the path to the files the pool is based on 2880 FILEPOOL=$(zpool status -v $pool | grep /$1/ | \ 2881 awk '{print $1}') 2882 2883 # this is a list of the zvols that make up the pool 2884 ZVOLPOOL=$(zpool status -v $pool | grep "$ZVOL_DEVDIR/$1$" \ 2885 | awk '{print $1}') 2886 2887 # also want to determine if it's a file-based pool using an 2888 # alternate mountpoint... 2889 POOL_FILE_DIRS=$(zpool status -v $pool | \ 2890 grep / | awk '{print $1}' | \ 2891 awk -F/ '{print $2}' | grep -v "dev") 2892 2893 for pooldir in $POOL_FILE_DIRS 2894 do 2895 OUTPUT=$(zfs list -H -r -o mountpoint $1 | \ 2896 grep "${pooldir}$" | awk '{print $1}') 2897 2898 ALTMOUNTPOOL="${ALTMOUNTPOOL}${OUTPUT}" 2899 done 2900 2901 2902 if [ ! -z "$ZVOLPOOL" ] 2903 then 2904 DONT_DESTROY="true" 2905 log_note "Pool $pool is built from $ZVOLPOOL on $1" 2906 fi 2907 2908 if [ ! -z "$FILEPOOL" ] 2909 then 2910 DONT_DESTROY="true" 2911 log_note "Pool $pool is built from $FILEPOOL on $1" 2912 fi 2913 2914 if [ ! -z "$ALTMOUNTPOOL" ] 2915 then 2916 DONT_DESTROY="true" 2917 log_note "Pool $pool is built from $ALTMOUNTPOOL on $1" 2918 fi 2919 done 2920 2921 if [ -z "${DONT_DESTROY}" ] 2922 then 2923 return 0 2924 else 2925 log_note "Warning: it is not safe to destroy $1!" 2926 return 1 2927 fi 2928} 2929 2930# 2931# Verify zfs operation with -p option work as expected 2932# $1 operation, value could be create, clone or rename 2933# $2 dataset type, value could be fs or vol 2934# $3 dataset name 2935# $4 new dataset name 2936# 2937function verify_opt_p_ops 2938{ 2939 typeset ops=$1 2940 typeset datatype=$2 2941 typeset dataset=$3 2942 typeset newdataset=$4 2943 2944 if [[ $datatype != "fs" && $datatype != "vol" ]]; then 2945 log_fail "$datatype is not supported." 2946 fi 2947 2948 # check parameters accordingly 2949 case $ops in 2950 create) 2951 newdataset=$dataset 2952 dataset="" 2953 if [[ $datatype == "vol" ]]; then 2954 ops="create -V $VOLSIZE" 2955 fi 2956 ;; 2957 clone) 2958 if [[ -z $newdataset ]]; then 2959 log_fail "newdataset should not be empty" \ 2960 "when ops is $ops." 2961 fi 2962 log_must datasetexists $dataset 2963 log_must snapexists $dataset 2964 ;; 2965 rename) 2966 if [[ -z $newdataset ]]; then 2967 log_fail "newdataset should not be empty" \ 2968 "when ops is $ops." 2969 fi 2970 log_must datasetexists $dataset 2971 ;; 2972 *) 2973 log_fail "$ops is not supported." 2974 ;; 2975 esac 2976 2977 # make sure the upper level filesystem does not exist 2978 destroy_dataset "${newdataset%/*}" "-rRf" 2979 2980 # without -p option, operation will fail 2981 log_mustnot zfs $ops $dataset $newdataset 2982 log_mustnot datasetexists $newdataset ${newdataset%/*} 2983 2984 # with -p option, operation should succeed 2985 log_must zfs $ops -p $dataset $newdataset 2986 block_device_wait 2987 2988 if ! datasetexists $newdataset ; then 2989 log_fail "-p option does not work for $ops" 2990 fi 2991 2992 # when $ops is create or clone, redo the operation still return zero 2993 if [[ $ops != "rename" ]]; then 2994 log_must zfs $ops -p $dataset $newdataset 2995 fi 2996 2997 return 0 2998} 2999 3000# 3001# Get configuration of pool 3002# $1 pool name 3003# $2 config name 3004# 3005function get_config 3006{ 3007 typeset pool=$1 3008 typeset config=$2 3009 typeset alt_root 3010 3011 if ! poolexists "$pool" ; then 3012 return 1 3013 fi 3014 alt_root=$(zpool list -H $pool | awk '{print $NF}') 3015 if [[ $alt_root == "-" ]]; then 3016 value=$(zdb -C $pool | grep "$config:" | awk -F: \ 3017 '{print $2}') 3018 else 3019 value=$(zdb -e $pool | grep "$config:" | awk -F: \ 3020 '{print $2}') 3021 fi 3022 if [[ -n $value ]] ; then 3023 value=${value#'} 3024 value=${value%'} 3025 fi 3026 echo $value 3027 3028 return 0 3029} 3030 3031# 3032# Privated function. Random select one of items from arguments. 3033# 3034# $1 count 3035# $2-n string 3036# 3037function _random_get 3038{ 3039 typeset cnt=$1 3040 shift 3041 3042 typeset str="$@" 3043 typeset -i ind 3044 ((ind = RANDOM % cnt + 1)) 3045 3046 typeset ret=$(echo "$str" | cut -f $ind -d ' ') 3047 echo $ret 3048} 3049 3050# 3051# Random select one of item from arguments which include NONE string 3052# 3053function random_get_with_non 3054{ 3055 typeset -i cnt=$# 3056 ((cnt =+ 1)) 3057 3058 _random_get "$cnt" "$@" 3059} 3060 3061# 3062# Random select one of item from arguments which doesn't include NONE string 3063# 3064function random_get 3065{ 3066 _random_get "$#" "$@" 3067} 3068 3069# 3070# The function will generate a dataset name with specific length 3071# $1, the length of the name 3072# $2, the base string to construct the name 3073# 3074function gen_dataset_name 3075{ 3076 typeset -i len=$1 3077 typeset basestr="$2" 3078 typeset -i baselen=${#basestr} 3079 typeset -i iter=0 3080 typeset l_name="" 3081 3082 if ((len % baselen == 0)); then 3083 ((iter = len / baselen)) 3084 else 3085 ((iter = len / baselen + 1)) 3086 fi 3087 while ((iter > 0)); do 3088 l_name="${l_name}$basestr" 3089 3090 ((iter -= 1)) 3091 done 3092 3093 echo $l_name 3094} 3095 3096# 3097# Get cksum tuple of dataset 3098# $1 dataset name 3099# 3100# sample zdb output: 3101# Dataset data/test [ZPL], ID 355, cr_txg 2413856, 31.0K, 7 objects, rootbp 3102# DVA[0]=<0:803046400:200> DVA[1]=<0:81199000:200> [L0 DMU objset] fletcher4 3103# lzjb LE contiguous unique double size=800L/200P birth=2413856L/2413856P 3104# fill=7 cksum=11ce125712:643a9c18ee2:125e25238fca0:254a3f74b59744 3105function datasetcksum 3106{ 3107 typeset cksum 3108 sync 3109 cksum=$(zdb -vvv $1 | grep "^Dataset $1 \[" | grep "cksum" \ 3110 | awk -F= '{print $7}') 3111 echo $cksum 3112} 3113 3114# 3115# Get cksum of file 3116# #1 file path 3117# 3118function checksum 3119{ 3120 typeset cksum 3121 cksum=$(cksum $1 | awk '{print $1}') 3122 echo $cksum 3123} 3124 3125# 3126# Get the given disk/slice state from the specific field of the pool 3127# 3128function get_device_state #pool disk field("", "spares","logs") 3129{ 3130 typeset pool=$1 3131 typeset disk=${2#$DEV_DSKDIR/} 3132 typeset field=${3:-$pool} 3133 3134 state=$(zpool status -v "$pool" 2>/dev/null | \ 3135 nawk -v device=$disk -v pool=$pool -v field=$field \ 3136 'BEGIN {startconfig=0; startfield=0; } 3137 /config:/ {startconfig=1} 3138 (startconfig==1) && ($1==field) {startfield=1; next;} 3139 (startfield==1) && ($1==device) {print $2; exit;} 3140 (startfield==1) && 3141 ($1==field || $1 ~ "^spares$" || $1 ~ "^logs$") {startfield=0}') 3142 echo $state 3143} 3144 3145 3146# 3147# print the given directory filesystem type 3148# 3149# $1 directory name 3150# 3151function get_fstype 3152{ 3153 typeset dir=$1 3154 3155 if [[ -z $dir ]]; then 3156 log_fail "Usage: get_fstype <directory>" 3157 fi 3158 3159 # 3160 # $ df -n / 3161 # / : ufs 3162 # 3163 df -n $dir | awk '{print $3}' 3164} 3165 3166# 3167# Given a disk, label it to VTOC regardless what label was on the disk 3168# $1 disk 3169# 3170function labelvtoc 3171{ 3172 typeset disk=$1 3173 if [[ -z $disk ]]; then 3174 log_fail "The disk name is unspecified." 3175 fi 3176 typeset label_file=/var/tmp/labelvtoc.$$ 3177 typeset arch=$(uname -p) 3178 3179 if is_linux || is_freebsd; then 3180 log_note "Currently unsupported by the test framework" 3181 return 1 3182 fi 3183 3184 if [[ $arch == "i386" ]]; then 3185 echo "label" > $label_file 3186 echo "0" >> $label_file 3187 echo "" >> $label_file 3188 echo "q" >> $label_file 3189 echo "q" >> $label_file 3190 3191 fdisk -B $disk >/dev/null 2>&1 3192 # wait a while for fdisk finishes 3193 sleep 60 3194 elif [[ $arch == "sparc" ]]; then 3195 echo "label" > $label_file 3196 echo "0" >> $label_file 3197 echo "" >> $label_file 3198 echo "" >> $label_file 3199 echo "" >> $label_file 3200 echo "q" >> $label_file 3201 else 3202 log_fail "unknown arch type" 3203 fi 3204 3205 format -e -s -d $disk -f $label_file 3206 typeset -i ret_val=$? 3207 rm -f $label_file 3208 # 3209 # wait the format to finish 3210 # 3211 sleep 60 3212 if ((ret_val != 0)); then 3213 log_fail "unable to label $disk as VTOC." 3214 fi 3215 3216 return 0 3217} 3218 3219# 3220# check if the system was installed as zfsroot or not 3221# return: 0 if zfsroot, non-zero if not 3222# 3223function is_zfsroot 3224{ 3225 df -n / | grep zfs > /dev/null 2>&1 3226 return $? 3227} 3228 3229# 3230# get the root filesystem name if it's zfsroot system. 3231# 3232# return: root filesystem name 3233function get_rootfs 3234{ 3235 typeset rootfs="" 3236 3237 if is_freebsd; then 3238 rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}') 3239 elif ! is_linux; then 3240 rootfs=$(awk '{if ($2 == "/" && $3 == "zfs") print $1}' \ 3241 /etc/mnttab) 3242 fi 3243 if [[ -z "$rootfs" ]]; then 3244 log_fail "Can not get rootfs" 3245 fi 3246 zfs list $rootfs > /dev/null 2>&1 3247 if (($? == 0)); then 3248 echo $rootfs 3249 else 3250 log_fail "This is not a zfsroot system." 3251 fi 3252} 3253 3254# 3255# get the rootfs's pool name 3256# return: 3257# rootpool name 3258# 3259function get_rootpool 3260{ 3261 typeset rootfs="" 3262 typeset rootpool="" 3263 3264 if is_freebsd; then 3265 rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}') 3266 elif ! is_linux; then 3267 rootfs=$(awk '{if ($2 == "/" && $3 =="zfs") print $1}' \ 3268 /etc/mnttab) 3269 fi 3270 if [[ -z "$rootfs" ]]; then 3271 log_fail "Can not get rootpool" 3272 fi 3273 zfs list $rootfs > /dev/null 2>&1 3274 if (($? == 0)); then 3275 echo ${rootfs%%/*} 3276 else 3277 log_fail "This is not a zfsroot system." 3278 fi 3279} 3280 3281# 3282# Get the word numbers from a string separated by white space 3283# 3284function get_word_count 3285{ 3286 echo $1 | wc -w 3287} 3288 3289# 3290# To verify if the require numbers of disks is given 3291# 3292function verify_disk_count 3293{ 3294 typeset -i min=${2:-1} 3295 3296 typeset -i count=$(get_word_count "$1") 3297 3298 if ((count < min)); then 3299 log_untested "A minimum of $min disks is required to run." \ 3300 " You specified $count disk(s)" 3301 fi 3302} 3303 3304function ds_is_volume 3305{ 3306 typeset type=$(get_prop type $1) 3307 [[ $type = "volume" ]] && return 0 3308 return 1 3309} 3310 3311function ds_is_filesystem 3312{ 3313 typeset type=$(get_prop type $1) 3314 [[ $type = "filesystem" ]] && return 0 3315 return 1 3316} 3317 3318function ds_is_snapshot 3319{ 3320 typeset type=$(get_prop type $1) 3321 [[ $type = "snapshot" ]] && return 0 3322 return 1 3323} 3324 3325# 3326# Check if Trusted Extensions are installed and enabled 3327# 3328function is_te_enabled 3329{ 3330 svcs -H -o state labeld 2>/dev/null | grep "enabled" 3331 if (($? != 0)); then 3332 return 1 3333 else 3334 return 0 3335 fi 3336} 3337 3338# Utility function to determine if a system has multiple cpus. 3339function is_mp 3340{ 3341 if is_linux; then 3342 (($(nproc) > 1)) 3343 elif is_freebsd; then 3344 sysctl -n kern.smp.cpus 3345 else 3346 (($(psrinfo | wc -l) > 1)) 3347 fi 3348 3349 return $? 3350} 3351 3352function get_cpu_freq 3353{ 3354 if is_linux; then 3355 lscpu | awk '/CPU MHz/ { print $3 }' 3356 elif is_freebsd; then 3357 sysctl -n hw.clockrate 3358 else 3359 psrinfo -v 0 | awk '/processor operates at/ {print $6}' 3360 fi 3361} 3362 3363# Run the given command as the user provided. 3364function user_run 3365{ 3366 typeset user=$1 3367 shift 3368 3369 log_note "user: $user" 3370 log_note "cmd: $*" 3371 3372 typeset out=$TEST_BASE_DIR/out 3373 typeset err=$TEST_BASE_DIR/err 3374 3375 sudo -Eu $user env PATH="$PATH" ksh <<<"$*" >$out 2>$err 3376 typeset res=$? 3377 log_note "out: $(<$out)" 3378 log_note "err: $(<$err)" 3379 return $res 3380} 3381 3382# 3383# Check if the pool contains the specified vdevs 3384# 3385# $1 pool 3386# $2..n <vdev> ... 3387# 3388# Return 0 if the vdevs are contained in the pool, 1 if any of the specified 3389# vdevs is not in the pool, and 2 if pool name is missing. 3390# 3391function vdevs_in_pool 3392{ 3393 typeset pool=$1 3394 typeset vdev 3395 3396 if [[ -z $pool ]]; then 3397 log_note "Missing pool name." 3398 return 2 3399 fi 3400 3401 shift 3402 3403 # We could use 'zpool list' to only get the vdevs of the pool but we 3404 # can't reference a mirror/raidz vdev using its ID (i.e mirror-0), 3405 # therefore we use the 'zpool status' output. 3406 typeset tmpfile=$(mktemp) 3407 zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile 3408 for vdev in $@; do 3409 grep -w ${vdev##*/} $tmpfile >/dev/null 2>&1 3410 [[ $? -ne 0 ]] && return 1 3411 done 3412 3413 rm -f $tmpfile 3414 3415 return 0; 3416} 3417 3418function get_max 3419{ 3420 typeset -l i max=$1 3421 shift 3422 3423 for i in "$@"; do 3424 max=$((max > i ? max : i)) 3425 done 3426 3427 echo $max 3428} 3429 3430function get_min 3431{ 3432 typeset -l i min=$1 3433 shift 3434 3435 for i in "$@"; do 3436 min=$((min < i ? min : i)) 3437 done 3438 3439 echo $min 3440} 3441 3442# Write data that can be compressed into a directory 3443function write_compressible 3444{ 3445 typeset dir=$1 3446 typeset megs=$2 3447 typeset nfiles=${3:-1} 3448 typeset bs=${4:-1024k} 3449 typeset fname=${5:-file} 3450 3451 [[ -d $dir ]] || log_fail "No directory: $dir" 3452 3453 # Under Linux fio is not currently used since its behavior can 3454 # differ significantly across versions. This includes missing 3455 # command line options and cases where the --buffer_compress_* 3456 # options fail to behave as expected. 3457 if is_linux; then 3458 typeset file_bytes=$(to_bytes $megs) 3459 typeset bs_bytes=4096 3460 typeset blocks=$(($file_bytes / $bs_bytes)) 3461 3462 for (( i = 0; i < $nfiles; i++ )); do 3463 truncate -s $file_bytes $dir/$fname.$i 3464 3465 # Write every third block to get 66% compression. 3466 for (( j = 0; j < $blocks; j += 3 )); do 3467 dd if=/dev/urandom of=$dir/$fname.$i \ 3468 seek=$j bs=$bs_bytes count=1 \ 3469 conv=notrunc >/dev/null 2>&1 3470 done 3471 done 3472 else 3473 log_must eval "fio \ 3474 --name=job \ 3475 --fallocate=0 \ 3476 --minimal \ 3477 --randrepeat=0 \ 3478 --buffer_compress_percentage=66 \ 3479 --buffer_compress_chunk=4096 \ 3480 --directory=$dir \ 3481 --numjobs=$nfiles \ 3482 --nrfiles=$nfiles \ 3483 --rw=write \ 3484 --bs=$bs \ 3485 --filesize=$megs \ 3486 --filename_format='$fname.\$jobnum' >/dev/null" 3487 fi 3488} 3489 3490function get_objnum 3491{ 3492 typeset pathname=$1 3493 typeset objnum 3494 3495 [[ -e $pathname ]] || log_fail "No such file or directory: $pathname" 3496 if is_freebsd; then 3497 objnum=$(stat -f "%i" $pathname) 3498 else 3499 objnum=$(stat -c %i $pathname) 3500 fi 3501 echo $objnum 3502} 3503 3504# 3505# Sync data to the pool 3506# 3507# $1 pool name 3508# $2 boolean to force uberblock (and config including zpool cache file) update 3509# 3510function sync_pool #pool <force> 3511{ 3512 typeset pool=${1:-$TESTPOOL} 3513 typeset force=${2:-false} 3514 3515 if [[ $force == true ]]; then 3516 log_must zpool sync -f $pool 3517 else 3518 log_must zpool sync $pool 3519 fi 3520 3521 return 0 3522} 3523 3524# 3525# Wait for zpool 'freeing' property drops to zero. 3526# 3527# $1 pool name 3528# 3529function wait_freeing #pool 3530{ 3531 typeset pool=${1:-$TESTPOOL} 3532 while true; do 3533 [[ "0" == "$(zpool list -Ho freeing $pool)" ]] && break 3534 log_must sleep 1 3535 done 3536} 3537 3538# 3539# Wait for every device replace operation to complete 3540# 3541# $1 pool name 3542# 3543function wait_replacing #pool 3544{ 3545 typeset pool=${1:-$TESTPOOL} 3546 while true; do 3547 [[ "" == "$(zpool status $pool | 3548 awk '/replacing-[0-9]+/ {print $1}')" ]] && break 3549 log_must sleep 1 3550 done 3551} 3552 3553# 3554# Wait for a pool to be scrubbed 3555# 3556# $1 pool name 3557# 3558function wait_scrubbed 3559{ 3560 typeset pool=${1:-$TESTPOOL} 3561 while ! is_pool_scrubbed $pool ; do 3562 sleep 1 3563 done 3564} 3565 3566# Backup the zed.rc in our test directory so that we can edit it for our test. 3567# 3568# Returns: Backup file name. You will need to pass this to zed_rc_restore(). 3569function zed_rc_backup 3570{ 3571 zedrc_backup="$(mktemp)" 3572 cp $ZEDLET_DIR/zed.rc $zedrc_backup 3573 echo $zedrc_backup 3574} 3575 3576function zed_rc_restore 3577{ 3578 mv $1 $ZEDLET_DIR/zed.rc 3579} 3580 3581# 3582# Setup custom environment for the ZED. 3583# 3584# $@ Optional list of zedlets to run under zed. 3585function zed_setup 3586{ 3587 if ! is_linux; then 3588 log_unsupported "No zed on $(uname)" 3589 fi 3590 3591 if [[ ! -d $ZEDLET_DIR ]]; then 3592 log_must mkdir $ZEDLET_DIR 3593 fi 3594 3595 if [[ ! -e $VDEVID_CONF ]]; then 3596 log_must touch $VDEVID_CONF 3597 fi 3598 3599 if [[ -e $VDEVID_CONF_ETC ]]; then 3600 log_fail "Must not have $VDEVID_CONF_ETC file present on system" 3601 fi 3602 EXTRA_ZEDLETS=$@ 3603 3604 # Create a symlink for /etc/zfs/vdev_id.conf file. 3605 log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC 3606 3607 # Setup minimal ZED configuration. Individual test cases should 3608 # add additional ZEDLETs as needed for their specific test. 3609 log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR 3610 log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR 3611 3612 # Scripts must only be user writable. 3613 if [[ -n "$EXTRA_ZEDLETS" ]] ; then 3614 saved_umask=$(umask) 3615 log_must umask 0022 3616 for i in $EXTRA_ZEDLETS ; do 3617 log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR 3618 done 3619 log_must umask $saved_umask 3620 fi 3621 3622 # Customize the zed.rc file to enable the full debug log. 3623 log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc 3624 echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc 3625 3626} 3627 3628# 3629# Cleanup custom ZED environment. 3630# 3631# $@ Optional list of zedlets to remove from our test zed.d directory. 3632function zed_cleanup 3633{ 3634 if ! is_linux; then 3635 return 3636 fi 3637 EXTRA_ZEDLETS=$@ 3638 3639 log_must rm -f ${ZEDLET_DIR}/zed.rc 3640 log_must rm -f ${ZEDLET_DIR}/zed-functions.sh 3641 log_must rm -f ${ZEDLET_DIR}/all-syslog.sh 3642 log_must rm -f ${ZEDLET_DIR}/all-debug.sh 3643 log_must rm -f ${ZEDLET_DIR}/state 3644 3645 if [[ -n "$EXTRA_ZEDLETS" ]] ; then 3646 for i in $EXTRA_ZEDLETS ; do 3647 log_must rm -f ${ZEDLET_DIR}/$i 3648 done 3649 fi 3650 log_must rm -f $ZED_LOG 3651 log_must rm -f $ZED_DEBUG_LOG 3652 log_must rm -f $VDEVID_CONF_ETC 3653 log_must rm -f $VDEVID_CONF 3654 rmdir $ZEDLET_DIR 3655} 3656 3657# 3658# Check if ZED is currently running, if not start ZED. 3659# 3660function zed_start 3661{ 3662 if ! is_linux; then 3663 return 3664 fi 3665 3666 # ZEDLET_DIR=/var/tmp/zed 3667 if [[ ! -d $ZEDLET_DIR ]]; then 3668 log_must mkdir $ZEDLET_DIR 3669 fi 3670 3671 # Verify the ZED is not already running. 3672 pgrep -x zed > /dev/null 3673 if (($? == 0)); then 3674 log_note "ZED already running" 3675 else 3676 log_note "Starting ZED" 3677 # run ZED in the background and redirect foreground logging 3678 # output to $ZED_LOG. 3679 log_must truncate -s 0 $ZED_DEBUG_LOG 3680 log_must eval "zed -vF -d $ZEDLET_DIR -P $PATH" \ 3681 "-s $ZEDLET_DIR/state -j 1 2>$ZED_LOG &" 3682 fi 3683 3684 return 0 3685} 3686 3687# 3688# Kill ZED process 3689# 3690function zed_stop 3691{ 3692 if ! is_linux; then 3693 return 3694 fi 3695 3696 log_note "Stopping ZED" 3697 while true; do 3698 zedpids="$(pgrep -x zed)" 3699 [ "$?" -ne 0 ] && break 3700 3701 log_must kill $zedpids 3702 sleep 1 3703 done 3704 return 0 3705} 3706 3707# 3708# Drain all zevents 3709# 3710function zed_events_drain 3711{ 3712 while [ $(zpool events -H | wc -l) -ne 0 ]; do 3713 sleep 1 3714 zpool events -c >/dev/null 3715 done 3716} 3717 3718# Set a variable in zed.rc to something, un-commenting it in the process. 3719# 3720# $1 variable 3721# $2 value 3722function zed_rc_set 3723{ 3724 var="$1" 3725 val="$2" 3726 # Remove the line 3727 cmd="'/$var/d'" 3728 eval sed -i $cmd $ZEDLET_DIR/zed.rc 3729 3730 # Add it at the end 3731 echo "$var=$val" >> $ZEDLET_DIR/zed.rc 3732} 3733 3734 3735# 3736# Check is provided device is being active used as a swap device. 3737# 3738function is_swap_inuse 3739{ 3740 typeset device=$1 3741 3742 if [[ -z $device ]] ; then 3743 log_note "No device specified." 3744 return 1 3745 fi 3746 3747 if is_linux; then 3748 swapon -s | grep -w $(readlink -f $device) > /dev/null 2>&1 3749 elif is_freebsd; then 3750 swapctl -l | grep -w $device 3751 else 3752 swap -l | grep -w $device > /dev/null 2>&1 3753 fi 3754 3755 return $? 3756} 3757 3758# 3759# Setup a swap device using the provided device. 3760# 3761function swap_setup 3762{ 3763 typeset swapdev=$1 3764 3765 if is_linux; then 3766 log_must eval "mkswap $swapdev > /dev/null 2>&1" 3767 log_must swapon $swapdev 3768 elif is_freebsd; then 3769 log_must swapctl -a $swapdev 3770 else 3771 log_must swap -a $swapdev 3772 fi 3773 3774 return 0 3775} 3776 3777# 3778# Cleanup a swap device on the provided device. 3779# 3780function swap_cleanup 3781{ 3782 typeset swapdev=$1 3783 3784 if is_swap_inuse $swapdev; then 3785 if is_linux; then 3786 log_must swapoff $swapdev 3787 elif is_freebsd; then 3788 log_must swapoff $swapdev 3789 else 3790 log_must swap -d $swapdev 3791 fi 3792 fi 3793 3794 return 0 3795} 3796 3797# 3798# Set a global system tunable (64-bit value) 3799# 3800# $1 tunable name (use a NAME defined in tunables.cfg) 3801# $2 tunable values 3802# 3803function set_tunable64 3804{ 3805 set_tunable_impl "$1" "$2" Z 3806} 3807 3808# 3809# Set a global system tunable (32-bit value) 3810# 3811# $1 tunable name (use a NAME defined in tunables.cfg) 3812# $2 tunable values 3813# 3814function set_tunable32 3815{ 3816 set_tunable_impl "$1" "$2" W 3817} 3818 3819function set_tunable_impl 3820{ 3821 typeset name="$1" 3822 typeset value="$2" 3823 typeset mdb_cmd="$3" 3824 typeset module="${4:-zfs}" 3825 3826 eval "typeset tunable=\$$name" 3827 case "$tunable" in 3828 UNSUPPORTED) 3829 log_unsupported "Tunable '$name' is unsupported on $(uname)" 3830 ;; 3831 "") 3832 log_fail "Tunable '$name' must be added to tunables.cfg" 3833 ;; 3834 *) 3835 ;; 3836 esac 3837 3838 [[ -z "$value" ]] && return 1 3839 [[ -z "$mdb_cmd" ]] && return 1 3840 3841 case "$(uname)" in 3842 Linux) 3843 typeset zfs_tunables="/sys/module/$module/parameters" 3844 [[ -w "$zfs_tunables/$tunable" ]] || return 1 3845 cat >"$zfs_tunables/$tunable" <<<"$value" 3846 return $? 3847 ;; 3848 FreeBSD) 3849 sysctl vfs.zfs.$tunable=$value 3850 return "$?" 3851 ;; 3852 SunOS) 3853 [[ "$module" -eq "zfs" ]] || return 1 3854 echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw 3855 return $? 3856 ;; 3857 esac 3858} 3859 3860# 3861# Get a global system tunable 3862# 3863# $1 tunable name (use a NAME defined in tunables.cfg) 3864# 3865function get_tunable 3866{ 3867 get_tunable_impl "$1" 3868} 3869 3870function get_tunable_impl 3871{ 3872 typeset name="$1" 3873 typeset module="${2:-zfs}" 3874 3875 eval "typeset tunable=\$$name" 3876 case "$tunable" in 3877 UNSUPPORTED) 3878 log_unsupported "Tunable '$name' is unsupported on $(uname)" 3879 ;; 3880 "") 3881 log_fail "Tunable '$name' must be added to tunables.cfg" 3882 ;; 3883 *) 3884 ;; 3885 esac 3886 3887 case "$(uname)" in 3888 Linux) 3889 typeset zfs_tunables="/sys/module/$module/parameters" 3890 [[ -f "$zfs_tunables/$tunable" ]] || return 1 3891 cat $zfs_tunables/$tunable 3892 return $? 3893 ;; 3894 FreeBSD) 3895 sysctl -n vfs.zfs.$tunable 3896 ;; 3897 SunOS) 3898 [[ "$module" -eq "zfs" ]] || return 1 3899 ;; 3900 esac 3901 3902 return 1 3903} 3904 3905# 3906# Prints the current time in seconds since UNIX Epoch. 3907# 3908function current_epoch 3909{ 3910 printf '%(%s)T' 3911} 3912 3913# 3914# Get decimal value of global uint32_t variable using mdb. 3915# 3916function mdb_get_uint32 3917{ 3918 typeset variable=$1 3919 typeset value 3920 3921 value=$(mdb -k -e "$variable/X | ::eval .=U") 3922 if [[ $? -ne 0 ]]; then 3923 log_fail "Failed to get value of '$variable' from mdb." 3924 return 1 3925 fi 3926 3927 echo $value 3928 return 0 3929} 3930 3931# 3932# Set global uint32_t variable to a decimal value using mdb. 3933# 3934function mdb_set_uint32 3935{ 3936 typeset variable=$1 3937 typeset value=$2 3938 3939 mdb -kw -e "$variable/W 0t$value" > /dev/null 3940 if [[ $? -ne 0 ]]; then 3941 echo "Failed to set '$variable' to '$value' in mdb." 3942 return 1 3943 fi 3944 3945 return 0 3946} 3947 3948# 3949# Set global scalar integer variable to a hex value using mdb. 3950# Note: Target should have CTF data loaded. 3951# 3952function mdb_ctf_set_int 3953{ 3954 typeset variable=$1 3955 typeset value=$2 3956 3957 mdb -kw -e "$variable/z $value" > /dev/null 3958 if [[ $? -ne 0 ]]; then 3959 echo "Failed to set '$variable' to '$value' in mdb." 3960 return 1 3961 fi 3962 3963 return 0 3964} 3965 3966# 3967# Compute MD5 digest for given file or stdin if no file given. 3968# Note: file path must not contain spaces 3969# 3970function md5digest 3971{ 3972 typeset file=$1 3973 3974 case $(uname) in 3975 FreeBSD) 3976 md5 -q $file 3977 ;; 3978 *) 3979 md5sum -b $file | awk '{ print $1 }' 3980 ;; 3981 esac 3982} 3983 3984# 3985# Compute SHA256 digest for given file or stdin if no file given. 3986# Note: file path must not contain spaces 3987# 3988function sha256digest 3989{ 3990 typeset file=$1 3991 3992 case $(uname) in 3993 FreeBSD) 3994 sha256 -q $file 3995 ;; 3996 *) 3997 sha256sum -b $file | awk '{ print $1 }' 3998 ;; 3999 esac 4000} 4001 4002function new_fs #<args> 4003{ 4004 case $(uname) in 4005 FreeBSD) 4006 newfs "$@" 4007 ;; 4008 *) 4009 echo y | newfs -v "$@" 4010 ;; 4011 esac 4012} 4013 4014function stat_size #<path> 4015{ 4016 typeset path=$1 4017 4018 case $(uname) in 4019 FreeBSD) 4020 stat -f %z "$path" 4021 ;; 4022 *) 4023 stat -c %s "$path" 4024 ;; 4025 esac 4026} 4027 4028function stat_ctime #<path> 4029{ 4030 typeset path=$1 4031 4032 case $(uname) in 4033 FreeBSD) 4034 stat -f %c "$path" 4035 ;; 4036 *) 4037 stat -c %Z "$path" 4038 ;; 4039 esac 4040} 4041 4042function stat_crtime #<path> 4043{ 4044 typeset path=$1 4045 4046 case $(uname) in 4047 FreeBSD) 4048 stat -f %B "$path" 4049 ;; 4050 *) 4051 stat -c %W "$path" 4052 ;; 4053 esac 4054} 4055 4056# Run a command as if it was being run in a TTY. 4057# 4058# Usage: 4059# 4060# faketty command 4061# 4062function faketty 4063{ 4064 if is_freebsd; then 4065 script -q /dev/null env "$@" 4066 else 4067 script --return --quiet -c "$*" /dev/null 4068 fi 4069} 4070 4071# 4072# Produce a random permutation of the integers in a given range (inclusive). 4073# 4074function range_shuffle # begin end 4075{ 4076 typeset -i begin=$1 4077 typeset -i end=$2 4078 4079 seq ${begin} ${end} | sort -R 4080} 4081 4082# 4083# Cross-platform xattr helpers 4084# 4085 4086function get_xattr # name path 4087{ 4088 typeset name=$1 4089 typeset path=$2 4090 4091 case $(uname) in 4092 FreeBSD) 4093 getextattr -qq user "${name}" "${path}" 4094 ;; 4095 *) 4096 attr -qg "${name}" "${path}" 4097 ;; 4098 esac 4099} 4100 4101function set_xattr # name value path 4102{ 4103 typeset name=$1 4104 typeset value=$2 4105 typeset path=$3 4106 4107 case $(uname) in 4108 FreeBSD) 4109 setextattr user "${name}" "${value}" "${path}" 4110 ;; 4111 *) 4112 attr -qs "${name}" -V "${value}" "${path}" 4113 ;; 4114 esac 4115} 4116 4117function set_xattr_stdin # name value 4118{ 4119 typeset name=$1 4120 typeset path=$2 4121 4122 case $(uname) in 4123 FreeBSD) 4124 setextattr -i user "${name}" "${path}" 4125 ;; 4126 *) 4127 attr -qs "${name}" "${path}" 4128 ;; 4129 esac 4130} 4131 4132function rm_xattr # name path 4133{ 4134 typeset name=$1 4135 typeset path=$2 4136 4137 case $(uname) in 4138 FreeBSD) 4139 rmextattr -q user "${name}" "${path}" 4140 ;; 4141 *) 4142 attr -qr "${name}" "${path}" 4143 ;; 4144 esac 4145} 4146 4147function ls_xattr # path 4148{ 4149 typeset path=$1 4150 4151 case $(uname) in 4152 FreeBSD) 4153 lsextattr -qq user "${path}" 4154 ;; 4155 *) 4156 attr -ql "${path}" 4157 ;; 4158 esac 4159} 4160 4161function kstat # stat flags? 4162{ 4163 typeset stat=$1 4164 typeset flags=${2-"-n"} 4165 4166 case $(uname) in 4167 FreeBSD) 4168 sysctl $flags kstat.zfs.misc.$stat 4169 ;; 4170 Linux) 4171 typeset zfs_kstat="/proc/spl/kstat/zfs/$stat" 4172 [[ -f "$zfs_kstat" ]] || return 1 4173 cat $zfs_kstat 4174 ;; 4175 *) 4176 false 4177 ;; 4178 esac 4179} 4180 4181function get_arcstat # stat 4182{ 4183 typeset stat=$1 4184 4185 case $(uname) in 4186 FreeBSD) 4187 kstat arcstats.$stat 4188 ;; 4189 Linux) 4190 kstat arcstats | awk "/$stat/ { print \$3 }" 4191 ;; 4192 *) 4193 false 4194 ;; 4195 esac 4196} 4197 4198function punch_hole # offset length file 4199{ 4200 typeset offset=$1 4201 typeset length=$2 4202 typeset file=$3 4203 4204 case $(uname) in 4205 FreeBSD) 4206 truncate -d -o $offset -l $length "$file" 4207 ;; 4208 Linux) 4209 fallocate --punch-hole --offset $offset --length $length "$file" 4210 ;; 4211 *) 4212 false 4213 ;; 4214 esac 4215} 4216 4217# 4218# Wait for the specified arcstat to reach non-zero quiescence. 4219# If echo is 1 echo the value after reaching quiescence, otherwise 4220# if echo is 0 print the arcstat we are waiting on. 4221# 4222function arcstat_quiescence # stat echo 4223{ 4224 typeset stat=$1 4225 typeset echo=$2 4226 typeset do_once=true 4227 4228 if [[ $echo -eq 0 ]]; then 4229 echo "Waiting for arcstat $1 quiescence." 4230 fi 4231 4232 while $do_once || [ $stat1 -ne $stat2 ] || [ $stat2 -eq 0 ]; do 4233 typeset stat1=$(get_arcstat $stat) 4234 sleep 2 4235 typeset stat2=$(get_arcstat $stat) 4236 do_once=false 4237 done 4238 4239 if [[ $echo -eq 1 ]]; then 4240 echo $stat2 4241 fi 4242} 4243 4244function arcstat_quiescence_noecho # stat 4245{ 4246 typeset stat=$1 4247 arcstat_quiescence $stat 0 4248} 4249 4250function arcstat_quiescence_echo # stat 4251{ 4252 typeset stat=$1 4253 arcstat_quiescence $stat 1 4254} 4255 4256# 4257# Given an array of pids, wait until all processes 4258# have completed and check their return status. 4259# 4260function wait_for_children #children 4261{ 4262 rv=0 4263 children=("$@") 4264 for child in "${children[@]}" 4265 do 4266 child_exit=0 4267 wait ${child} || child_exit=$? 4268 if [ $child_exit -ne 0 ]; then 4269 echo "child ${child} failed with ${child_exit}" 4270 rv=1 4271 fi 4272 done 4273 return $rv 4274} 4275