1#!/usr/bin/env bash 2# 3# Verify that an assortment of known good reference pools can be imported 4# using different versions of the ZoL code. 5# 6# By default references pools for the major ZFS implementation will be 7# checked against the most recent ZoL tags and the master development branch. 8# Alternate tags or branches may be verified with the '-s <src-tag> option. 9# Passing the keyword "installed" will instruct the script to test whatever 10# version is installed. 11# 12# Preferentially a reference pool is used for all tests. However, if one 13# does not exist and the pool-tag matches one of the src-tags then a new 14# reference pool will be created using binaries from that source build. 15# This is particularly useful when you need to test your changes before 16# opening a pull request. The keyword 'all' can be used as short hand 17# refer to all available reference pools. 18# 19# New reference pools may be added by placing a bzip2 compressed tarball 20# of the pool in the scripts/zfs-images directory and then passing 21# the -p <pool-tag> option. To increase the test coverage reference pools 22# should be collected for all the major ZFS implementations. Having these 23# pools easily available is also helpful to the developers. 24# 25# Care should be taken to run these tests with a kernel supported by all 26# the listed tags. Otherwise build failure will cause false positives. 27# 28# 29# EXAMPLES: 30# 31# The following example will verify the zfs-0.6.2 tag, the master branch, 32# and the installed zfs version can correctly import the listed pools. 33# Note there is no reference pool available for master and installed but 34# because binaries are available one is automatically constructed. The 35# working directory is also preserved between runs (-k) preventing the 36# need to rebuild from source for multiple runs. 37# 38# zimport.sh -k -f /var/tmp/zimport \ 39# -s "zfs-0.6.2 master installed" \ 40# -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed" 41# 42# ------------------------ OpenZFS Source Versions ---------------- 43# zfs-0.6.2 master 0.6.2-175_g36eb554 44# ----------------------------------------------------------------- 45# Clone ZFS Local Local Skip 46# Build ZFS Pass Pass Skip 47# ----------------------------------------------------------------- 48# zevo-1.1.1 Pass Pass Pass 49# zol-0.6.2 Pass Pass Pass 50# zol-0.6.2-173 Fail Pass Pass 51# master Pass Pass Pass 52# installed Pass Pass Pass 53# 54 55BASE_DIR=$(dirname "$0") 56SCRIPT_COMMON=common.sh 57if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then 58 . "${BASE_DIR}/${SCRIPT_COMMON}" 59else 60 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 61fi 62 63PROG=zimport.sh 64SRC_TAGS="zfs-0.6.5.11 master" 65POOL_TAGS="all master" 66POOL_CREATE_OPTIONS= 67TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX) 68KEEP="no" 69VERBOSE="no" 70COLOR="yes" 71REPO="https://github.com/openzfs" 72IMAGES_DIR="$SCRIPTDIR/zfs-images/" 73IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master" 74ERROR=0 75 76CONFIG_LOG="configure.log" 77CONFIG_OPTIONS=${CONFIG_OPTIONS:-""} 78MAKE_LOG="make.log" 79MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"} 80 81COLOR_GREEN="\033[0;32m" 82COLOR_RED="\033[0;31m" 83COLOR_BROWN="\033[0;33m" 84COLOR_RESET="\033[0m" 85 86usage() { 87cat << EOF 88USAGE: 89zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] 90 [-f path] [-o options] 91 92DESCRIPTION: 93 ZPOOL import verification tests 94 95OPTIONS: 96 -h Show this message 97 -v Verbose 98 -c No color 99 -k Keep temporary directory 100 -r <repo> Source repository ($REPO) 101 -s <src-tag>... Verify ZoL versions with the listed tags 102 -i <pool-dir> Pool image directory 103 -p <pool-tag>... Verify pools created with the listed tags 104 -f <path> Temporary directory to use 105 -o <options> Additional options to pass to 'zpool create' 106 107EOF 108} 109 110while getopts 'hvckr:s:i:p:f:o:?' OPTION; do 111 case $OPTION in 112 h) 113 usage 114 exit 1 115 ;; 116 v) 117 VERBOSE="yes" 118 ;; 119 c) 120 COLOR="no" 121 ;; 122 k) 123 KEEP="yes" 124 ;; 125 r) 126 REPO="$OPTARG" 127 ;; 128 s) 129 SRC_TAGS="$OPTARG" 130 ;; 131 i) 132 IMAGES_DIR="$OPTARG" 133 ;; 134 p) 135 POOL_TAGS="$OPTARG" 136 ;; 137 f) 138 TEST_DIR="$OPTARG" 139 ;; 140 o) 141 POOL_CREATE_OPTIONS="$OPTARG" 142 ;; 143 ?) 144 usage 145 exit 1 146 ;; 147 esac 148done 149 150# 151# Verify the module start is not loaded 152# 153if lsmod | grep zfs >/dev/null; then 154 echo "ZFS modules must be unloaded" 155 exit 1 156fi 157 158# 159# Create a random directory tree of files and sub-directories to 160# to act as a copy source for the various regression tests. 161# 162populate() { 163 local ROOT=$1 164 local MAX_DIR_SIZE=$2 165 local MAX_FILE_SIZE=$3 166 167 # shellcheck disable=SC2086 168 mkdir -p $ROOT/{a,b,c,d,e,f,g}/{h,i} 169 DIRS=$(find "$ROOT") 170 171 for DIR in $DIRS; do 172 COUNT=$((RANDOM % MAX_DIR_SIZE)) 173 174 # shellcheck disable=SC2034 175 for i in $(seq $COUNT); do 176 FILE=$(mktemp -p "$DIR") 177 SIZE=$((RANDOM % MAX_FILE_SIZE)) 178 dd if=/dev/urandom of="$FILE" bs=1k \ 179 count="$SIZE" &>/dev/null 180 done 181 done 182 183 return 0 184} 185 186SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX) 187trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT 188populate "$SRC_DIR" 10 100 189 190SRC_DIR="$TEST_DIR/src" 191SRC_DIR_ZFS="$SRC_DIR/zfs" 192 193if [ "$COLOR" = "no" ]; then 194 COLOR_GREEN="" 195 COLOR_BROWN="" 196 COLOR_RED="" 197 COLOR_RESET="" 198fi 199 200pass_nonewline() { 201 echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t" 202} 203 204skip_nonewline() { 205 echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t" 206} 207 208fail_nonewline() { 209 echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t" 210} 211 212# 213# Log a failure message, cleanup, and return an error. 214# 215fail() { 216 echo -e "$PROG: $1" >&2 217 $ZFS_SH -u >/dev/null 2>&1 218 exit 1 219} 220 221# 222# Set several helper variables which are derived from a source tag. 223# 224# ZFS_TAG - The passed zfs-x.y.z tag 225# ZFS_DIR - The zfs directory name 226# ZFS_URL - The zfs github URL to fetch the tarball 227# 228src_set_vars() { 229 local TAG=$1 230 231 ZFS_TAG="$TAG" 232 ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG" 233 ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG" 234 235 if [ "$TAG" = "installed" ]; then 236 ZPOOL_CMD=$(command -v zpool) 237 ZFS_CMD=$(command -v zfs) 238 ZFS_SH="/usr/share/zfs/zfs.sh" 239 else 240 ZPOOL_CMD="./cmd/zpool/zpool" 241 ZFS_CMD="./cmd/zfs/zfs" 242 ZFS_SH="./scripts/zfs.sh" 243 fi 244} 245 246# 247# Set several helper variables which are derived from a pool name such 248# as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various 249# ZFS implementations which are used to verify compatibility. 250# 251# POOL_TAG - The example pools name in scripts/zfs-images/. 252# POOL_BZIP - The full path to the example bzip2 compressed pool. 253# POOL_DIR - The top level test path for this pool. 254# POOL_DIR_PRISTINE - The directory containing a pristine version of the pool. 255# POOL_DIR_COPY - The directory containing a working copy of the pool. 256# POOL_DIR_SRC - Location of a source build if it exists for this pool. 257# 258pool_set_vars() { 259 local TAG=$1 260 261 POOL_TAG=$TAG 262 POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2 263 POOL_DIR=$TEST_DIR/pools/$POOL_TAG 264 POOL_DIR_PRISTINE=$POOL_DIR/pristine 265 POOL_DIR_COPY=$POOL_DIR/copy 266 POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}" 267} 268 269# 270# Construct a non-trivial pool given a specific version of the source. More 271# interesting pools provide better test coverage so this function should 272# extended as needed to create more realistic pools. 273# 274pool_create() { 275 pool_set_vars "$1" 276 src_set_vars "$1" 277 278 if [ "$POOL_TAG" != "installed" ]; then 279 cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'" 280 fi 281 282 $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \ 283 fail "Failed to load kmods" 284 285 # Create a file vdev RAIDZ pool. 286 truncate -s 1G \ 287 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 288 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 289 fail "Failed 'truncate -s 1G ...'" 290 # shellcheck disable=SC2086 291 $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \ 292 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 293 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 294 fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'" 295 296 # Create a pool/fs filesystem with some random contents. 297 $ZFS_CMD create "$POOL_TAG/fs" || \ 298 fail "Failed '$ZFS_CMD create $POOL_TAG/fs'" 299 populate "/$POOL_TAG/fs/" 10 100 300 301 # Snapshot that filesystem, clone it, remove the files/dirs, 302 # replace them with new files/dirs. 303 $ZFS_CMD snap "$POOL_TAG/fs@snap" || \ 304 fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'" 305 $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \ 306 fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'" 307 # shellcheck disable=SC2086 308 rm -Rf /$POOL_TAG/clone/* 309 populate "/$POOL_TAG/clone/" 10 100 310 311 # Scrub the pool, delay slightly, then export it. It is now 312 # somewhat interesting for testing purposes. 313 $ZPOOL_CMD scrub "$POOL_TAG" || \ 314 fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'" 315 sleep 10 316 $ZPOOL_CMD export "$POOL_TAG" || \ 317 fail "Failed '$ZPOOL_CMD export $POOL_TAG'" 318 319 $ZFS_SH -u || fail "Failed to unload kmods" 320} 321 322# If the zfs-images directory doesn't exist fetch a copy from Github then 323# cache it in the $TEST_DIR and update $IMAGES_DIR. 324if [ ! -d "$IMAGES_DIR" ]; then 325 IMAGES_DIR="$TEST_DIR/zfs-images" 326 mkdir -p "$IMAGES_DIR" 327 curl -sL "$IMAGES_TAR" | \ 328 tar -xz -C "$IMAGES_DIR" --strip-components=1 || \ 329 fail "Failed to download pool images" 330fi 331 332# Given the available images in the zfs-images directory substitute the 333# list of available images for the reserved keyword 'all'. 334for TAG in $POOL_TAGS; do 335 336 if [ "$TAG" = "all" ]; then 337 # shellcheck disable=SC2010 338 ALL_TAGS=$(ls "$IMAGES_DIR" | grep "tar.bz2" | \ 339 sed 's/.tar.bz2//' | tr '\n' ' ') 340 NEW_TAGS="$NEW_TAGS $ALL_TAGS" 341 else 342 NEW_TAGS="$NEW_TAGS $TAG" 343 fi 344done 345POOL_TAGS="$NEW_TAGS" 346 347if [ "$VERBOSE" = "yes" ]; then 348 echo "---------------------------- Options ----------------------------" 349 echo "VERBOSE=$VERBOSE" 350 echo "KEEP=$KEEP" 351 echo "REPO=$REPO" 352 echo "SRC_TAGS=$SRC_TAGS" 353 echo "POOL_TAGS=$POOL_TAGS" 354 echo "PATH=$TEST_DIR" 355 echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS" 356 echo 357fi 358 359if [ ! -d "$TEST_DIR" ]; then 360 mkdir -p "$TEST_DIR" 361fi 362 363if [ ! -d "$SRC_DIR" ]; then 364 mkdir -p "$SRC_DIR" 365fi 366 367# Print a header for all tags which are being tested. 368echo "------------------------ OpenZFS Source Versions ----------------" 369printf "%-16s" " " 370for TAG in $SRC_TAGS; do 371 src_set_vars "$TAG" 372 373 if [ "$TAG" = "installed" ]; then 374 ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }') 375 if [ -n "$ZFS_VERSION" ]; then 376 printf "%-16s" "$ZFS_VERSION" 377 else 378 fail "ZFS is not installed" 379 fi 380 else 381 printf "%-16s" "$TAG" 382 fi 383done 384echo -e "\n-----------------------------------------------------------------" 385 386# 387# Attempt to generate the tarball from your local git repository, if that 388# fails then attempt to download the tarball from Github. 389# 390printf "%-16s" "Clone ZFS" 391for TAG in $SRC_TAGS; do 392 src_set_vars "$TAG" 393 394 if [ -d "$ZFS_DIR" ]; then 395 skip_nonewline 396 elif [ "$ZFS_TAG" = "installed" ]; then 397 skip_nonewline 398 else 399 cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" 400 401 if [ ! -d "$SRC_DIR_ZFS" ]; then 402 mkdir -p "$SRC_DIR_ZFS" 403 fi 404 405 git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \ 406 -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \ 407 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 408 if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then 409 tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS" 410 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 411 echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" 412 else 413 mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR" 414 curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \ 415 --strip-components=1 || \ 416 fail "Failed to download $ZFS_URL" 417 echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" 418 fi 419 fi 420done 421printf "\n" 422 423# Build the listed tags 424printf "%-16s" "Build ZFS" 425for TAG in $SRC_TAGS; do 426 src_set_vars "$TAG" 427 428 if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then 429 skip_nonewline 430 elif [ "$ZFS_TAG" = "installed" ]; then 431 skip_nonewline 432 else 433 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 434 make distclean &>/dev/null 435 ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ 436 fail "Failed ZFS 'autogen.sh'" 437 # shellcheck disable=SC2086 438 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \ 439 fail "Failed ZFS 'configure $CONFIG_OPTIONS'" 440 # shellcheck disable=SC2086 441 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ 442 fail "Failed ZFS 'make $MAKE_OPTIONS'" 443 pass_nonewline 444 fi 445done 446printf "\n" 447echo "-----------------------------------------------------------------" 448 449# Either create a new pool using 'zpool create', or alternately restore an 450# existing pool from another ZFS implementation for compatibility testing. 451for TAG in $POOL_TAGS; do 452 pool_set_vars "$TAG" 453 SKIP=0 454 455 printf "%-16s" "$POOL_TAG" 456 rm -Rf "$POOL_DIR" 457 mkdir -p "$POOL_DIR_PRISTINE" 458 459 # Use the existing compressed image if available. 460 if [ -f "$POOL_BZIP" ]; then 461 tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \ 462 --strip-components=1 || \ 463 fail "Failed 'tar -xjf $POOL_BZIP" 464 # Use the installed version to create the pool. 465 elif [ "$TAG" = "installed" ]; then 466 pool_create "$TAG" 467 # A source build is available to create the pool. 468 elif [ -d "$POOL_DIR_SRC" ]; then 469 pool_create "$TAG" 470 else 471 SKIP=1 472 fi 473 474 # Verify 'zpool import' works for all listed source versions. 475 for SRC_TAG in $SRC_TAGS; do 476 477 if [ $SKIP -eq 1 ]; then 478 skip_nonewline 479 continue 480 fi 481 482 src_set_vars "$SRC_TAG" 483 if [ "$SRC_TAG" != "installed" ]; then 484 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 485 fi 486 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY" 487 488 cp -a --sparse=always "$POOL_DIR_PRISTINE" \ 489 "$POOL_DIR_COPY" || \ 490 fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" 491 POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ 492 awk '/pool:/ { print $2; exit 0 }') 493 494 $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" \ 495 "$POOL_NAME" &>/dev/null 496 # shellcheck disable=SC2181 497 if [ $? -ne 0 ]; then 498 fail_nonewline 499 ERROR=1 500 else 501 $ZPOOL_CMD export "$POOL_NAME" || \ 502 fail "Failed to export pool" 503 pass_nonewline 504 fi 505 506 rm -Rf "$POOL_DIR_COPY" 507 508 $ZFS_SH -u || fail "Failed to unload kmods" 509 done 510 printf "\n" 511done 512 513if [ "$KEEP" = "no" ]; then 514 rm -Rf "$TEST_DIR" 515fi 516 517exit $ERROR 518