1#!/usr/bin/env bash 2# 3# Verify that an assortment of known good reference pools can be imported 4# using different versions of OpenZFS code. 5# 6# By default references pools for the major ZFS implementation will be 7# checked against the most recent OpenZFS tags and the master development branch. 8# Alternate tags or branches may be verified with the '-s <src-tag> option. 9# Passing the keyword "installed" will instruct the script to test whatever 10# version is installed. 11# 12# Preferentially a reference pool is used for all tests. However, if one 13# does not exist and the pool-tag matches one of the src-tags then a new 14# reference pool will be created using binaries from that source build. 15# This is particularly useful when you need to test your changes before 16# opening a pull request. The keyword 'all' can be used as short hand 17# refer to all available reference pools. 18# 19# New reference pools may be added by placing a bzip2 compressed tarball 20# of the pool in the scripts/zfs-images directory and then passing 21# the -p <pool-tag> option. To increase the test coverage reference pools 22# should be collected for all the major ZFS implementations. Having these 23# pools easily available is also helpful to the developers. 24# 25# Care should be taken to run these tests with a kernel supported by all 26# the listed tags. Otherwise build failure will cause false positives. 27# 28# 29# EXAMPLES: 30# 31# The following example will verify the zfs-0.6.2 tag, the master branch, 32# and the installed zfs version can correctly import the listed pools. 33# Note there is no reference pool available for master and installed but 34# because binaries are available one is automatically constructed. The 35# working directory is also preserved between runs (-k) preventing the 36# need to rebuild from source for multiple runs. 37# 38# zimport.sh -k -f /var/tmp/zimport \ 39# -s "zfs-0.6.2 master installed" \ 40# -p "zevo-1.1.1 zol-0.6.2 zol-0.6.2-173 master installed" 41# 42# ------------------------ OpenZFS Source Versions ---------------- 43# zfs-0.6.2 master 0.6.2-175_g36eb554 44# ----------------------------------------------------------------- 45# Clone ZFS Local Local Skip 46# Build ZFS Pass Pass Skip 47# ----------------------------------------------------------------- 48# zevo-1.1.1 Pass Pass Pass 49# zol-0.6.2 Pass Pass Pass 50# zol-0.6.2-173 Fail Pass Pass 51# master Pass Pass Pass 52# installed Pass Pass Pass 53# 54 55BASE_DIR=$(dirname "$0") 56SCRIPT_COMMON=common.sh 57if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then 58 . "${BASE_DIR}/${SCRIPT_COMMON}" 59else 60 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 61fi 62 63PROG=zimport.sh 64SRC_TAGS="zfs-0.6.5.11 master" 65POOL_TAGS="all master" 66POOL_CREATE_OPTIONS= 67TEST_DIR=$(mktemp -u -d -p /var/tmp zimport.XXXXXXXX) 68KEEP="no" 69VERBOSE="no" 70COLOR="yes" 71REPO="https://github.com/openzfs" 72IMAGES_DIR="${BASE_DIR}/zfs-images/" 73IMAGES_TAR="https://github.com/openzfs/zfs-images/tarball/master" 74ERROR=0 75 76CONFIG_LOG="configure.log" 77CONFIG_OPTIONS=${CONFIG_OPTIONS:-""} 78MAKE_LOG="make.log" 79MAKE_OPTIONS=${MAKE_OPTIONS:-"-s -j$(nproc)"} 80 81COLOR_GREEN="\033[0;32m" 82COLOR_RED="\033[0;31m" 83COLOR_BROWN="\033[0;33m" 84COLOR_RESET="\033[0m" 85 86usage() { 87cat << EOF 88USAGE: 89zimport.sh [hvl] [-r repo] [-s src-tag] [-i pool-dir] [-p pool-tag] 90 [-f path] [-o options] 91 92DESCRIPTION: 93 ZPOOL import verification tests 94 95OPTIONS: 96 -h Show this message 97 -v Verbose 98 -c No color 99 -k Keep temporary directory 100 -r <repo> Source repository ($REPO) 101 -s <src-tag>... Verify OpenZFS versions with the listed tags 102 -i <pool-dir> Pool image directory 103 -p <pool-tag>... Verify pools created with the listed tags 104 -f <path> Temporary directory to use 105 -o <options> Additional options to pass to 'zpool create' 106 107EOF 108} 109 110while getopts 'hvckr:s:i:p:f:o:?' OPTION; do 111 case $OPTION in 112 h) 113 usage 114 exit 1 115 ;; 116 v) 117 VERBOSE="yes" 118 ;; 119 c) 120 COLOR="no" 121 ;; 122 k) 123 KEEP="yes" 124 ;; 125 r) 126 REPO="$OPTARG" 127 ;; 128 s) 129 SRC_TAGS="$OPTARG" 130 ;; 131 i) 132 IMAGES_DIR="$OPTARG" 133 ;; 134 p) 135 POOL_TAGS="$OPTARG" 136 ;; 137 f) 138 TEST_DIR="$OPTARG" 139 ;; 140 o) 141 POOL_CREATE_OPTIONS="$OPTARG" 142 ;; 143 *) 144 usage 145 exit 1 146 ;; 147 esac 148done 149 150# 151# Verify the module start is not loaded 152# 153if lsmod | grep zfs >/dev/null; then 154 echo "ZFS modules must be unloaded" 155 exit 1 156fi 157 158# 159# Create a random directory tree of files and sub-directories to 160# to act as a copy source for the various regression tests. 161# 162populate() { 163 local ROOT=$1 164 local MAX_DIR_SIZE=$2 165 local MAX_FILE_SIZE=$3 166 167 mkdir -p "$ROOT"/{a,b,c,d,e,f,g}/{h,i} 168 DIRS=$(find "$ROOT") 169 170 for DIR in $DIRS; do 171 COUNT=$((RANDOM % MAX_DIR_SIZE)) 172 173 for _ in $(seq "$COUNT"); do 174 FILE=$(mktemp -p "$DIR") 175 SIZE=$((RANDOM % MAX_FILE_SIZE)) 176 dd if=/dev/urandom of="$FILE" bs=1k \ 177 count="$SIZE" &>/dev/null 178 done 179 done 180 181 return 0 182} 183 184SRC_DIR=$(mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX) 185trap 'rm -Rf "$SRC_DIR"' INT TERM EXIT 186populate "$SRC_DIR" 10 100 187 188SRC_DIR="$TEST_DIR/src" 189SRC_DIR_ZFS="$SRC_DIR/zfs" 190 191if [ "$COLOR" = "no" ]; then 192 COLOR_GREEN="" 193 COLOR_BROWN="" 194 COLOR_RED="" 195 COLOR_RESET="" 196fi 197 198pass_nonewline() { 199 echo -n -e "${COLOR_GREEN}Pass${COLOR_RESET}\t\t" 200} 201 202skip_nonewline() { 203 echo -n -e "${COLOR_BROWN}Skip${COLOR_RESET}\t\t" 204} 205 206fail_nonewline() { 207 echo -n -e "${COLOR_RED}Fail${COLOR_RESET}\t\t" 208} 209 210# 211# Log a failure message, cleanup, and return an error. 212# 213fail() { 214 echo -e "$PROG: $1" >&2 215 $ZFS_SH -u >/dev/null 2>&1 216 exit 1 217} 218 219# 220# Set several helper variables which are derived from a source tag. 221# 222# ZFS_TAG - The passed zfs-x.y.z tag 223# ZFS_DIR - The zfs directory name 224# ZFS_URL - The zfs github URL to fetch the tarball 225# 226src_set_vars() { 227 local TAG=$1 228 229 ZFS_TAG="$TAG" 230 ZFS_DIR="$SRC_DIR_ZFS/$ZFS_TAG" 231 ZFS_URL="$REPO/zfs/tarball/$ZFS_TAG" 232 233 if [ "$TAG" = "installed" ]; then 234 ZPOOL_CMD=$(command -v zpool) 235 ZFS_CMD=$(command -v zfs) 236 ZFS_SH="/usr/share/zfs/zfs.sh" 237 else 238 ZPOOL_CMD="./zpool" 239 ZFS_CMD="./zfs" 240 ZFS_SH="./scripts/zfs.sh" 241 fi 242} 243 244# 245# Set several helper variables which are derived from a pool name such 246# as zol-0.6.x, zevo-1.1.1, etc. These refer to example pools from various 247# ZFS implementations which are used to verify compatibility. 248# 249# POOL_TAG - The example pools name in scripts/zfs-images/. 250# POOL_BZIP - The full path to the example bzip2 compressed pool. 251# POOL_DIR - The top level test path for this pool. 252# POOL_DIR_PRISTINE - The directory containing a pristine version of the pool. 253# POOL_DIR_COPY - The directory containing a working copy of the pool. 254# POOL_DIR_SRC - Location of a source build if it exists for this pool. 255# 256pool_set_vars() { 257 local TAG=$1 258 259 POOL_TAG=$TAG 260 POOL_BZIP=$IMAGES_DIR/$POOL_TAG.tar.bz2 261 POOL_DIR=$TEST_DIR/pools/$POOL_TAG 262 POOL_DIR_PRISTINE=$POOL_DIR/pristine 263 POOL_DIR_COPY=$POOL_DIR/copy 264 POOL_DIR_SRC="$SRC_DIR_ZFS/${POOL_TAG//zol/zfs}" 265} 266 267# 268# Construct a non-trivial pool given a specific version of the source. More 269# interesting pools provide better test coverage so this function should 270# extended as needed to create more realistic pools. 271# 272pool_create() { 273 pool_set_vars "$1" 274 src_set_vars "$1" 275 276 if [ "$POOL_TAG" != "installed" ]; then 277 cd "$POOL_DIR_SRC" || fail "Failed 'cd $POOL_DIR_SRC'" 278 fi 279 280 $ZFS_SH zfs="spa_config_path=$POOL_DIR_PRISTINE" || \ 281 fail "Failed to load kmods" 282 283 # Create a file vdev RAIDZ pool. 284 truncate -s 1G \ 285 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 286 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 287 fail "Failed 'truncate -s 1G ...'" 288 # shellcheck disable=SC2086 289 $ZPOOL_CMD create $POOL_CREATE_OPTIONS "$POOL_TAG" raidz \ 290 "$POOL_DIR_PRISTINE/vdev1" "$POOL_DIR_PRISTINE/vdev2" \ 291 "$POOL_DIR_PRISTINE/vdev3" "$POOL_DIR_PRISTINE/vdev4" || \ 292 fail "Failed '$ZPOOL_CMD create $POOL_CREATE_OPTIONS $POOL_TAG ...'" 293 294 # Create a pool/fs filesystem with some random contents. 295 $ZFS_CMD create "$POOL_TAG/fs" || \ 296 fail "Failed '$ZFS_CMD create $POOL_TAG/fs'" 297 populate "/$POOL_TAG/fs/" 10 100 298 299 # Snapshot that filesystem, clone it, remove the files/dirs, 300 # replace them with new files/dirs. 301 $ZFS_CMD snap "$POOL_TAG/fs@snap" || \ 302 fail "Failed '$ZFS_CMD snap $POOL_TAG/fs@snap'" 303 $ZFS_CMD clone "$POOL_TAG/fs@snap" "$POOL_TAG/clone" || \ 304 fail "Failed '$ZFS_CMD clone $POOL_TAG/fs@snap $POOL_TAG/clone'" 305 # shellcheck disable=SC2086 306 rm -Rf /$POOL_TAG/clone/* 307 populate "/$POOL_TAG/clone/" 10 100 308 309 # Scrub the pool, delay slightly, then export it. It is now 310 # somewhat interesting for testing purposes. 311 $ZPOOL_CMD scrub "$POOL_TAG" || \ 312 fail "Failed '$ZPOOL_CMD scrub $POOL_TAG'" 313 sleep 10 314 $ZPOOL_CMD export "$POOL_TAG" || \ 315 fail "Failed '$ZPOOL_CMD export $POOL_TAG'" 316 317 $ZFS_SH -u || fail "Failed to unload kmods" 318} 319 320# If the zfs-images directory doesn't exist fetch a copy from Github then 321# cache it in the $TEST_DIR and update $IMAGES_DIR. 322if [ ! -d "$IMAGES_DIR" ]; then 323 IMAGES_DIR="$TEST_DIR/zfs-images" 324 mkdir -p "$IMAGES_DIR" 325 curl -sL "$IMAGES_TAR" | \ 326 tar -xz -C "$IMAGES_DIR" --strip-components=1 || \ 327 fail "Failed to download pool images" 328fi 329 330# Given the available images in the zfs-images directory substitute the 331# list of available images for the reserved keyword 'all'. 332for TAG in $POOL_TAGS; do 333 334 if [ "$TAG" = "all" ]; then 335 ALL_TAGS=$(echo "$IMAGES_DIR"/*.tar.bz2 | \ 336 sed "s|$IMAGES_DIR/||g;s|.tar.bz2||g") 337 NEW_TAGS="$NEW_TAGS $ALL_TAGS" 338 else 339 NEW_TAGS="$NEW_TAGS $TAG" 340 fi 341done 342POOL_TAGS="$NEW_TAGS" 343 344if [ "$VERBOSE" = "yes" ]; then 345 echo "---------------------------- Options ----------------------------" 346 echo "VERBOSE=$VERBOSE" 347 echo "KEEP=$KEEP" 348 echo "REPO=$REPO" 349 echo "SRC_TAGS=$SRC_TAGS" 350 echo "POOL_TAGS=$POOL_TAGS" 351 echo "PATH=$TEST_DIR" 352 echo "POOL_CREATE_OPTIONS=$POOL_CREATE_OPTIONS" 353 echo 354fi 355 356if [ ! -d "$TEST_DIR" ]; then 357 mkdir -p "$TEST_DIR" 358fi 359 360if [ ! -d "$SRC_DIR" ]; then 361 mkdir -p "$SRC_DIR" 362fi 363 364# Print a header for all tags which are being tested. 365echo "------------------------ OpenZFS Source Versions ----------------" 366printf "%-16s" " " 367for TAG in $SRC_TAGS; do 368 src_set_vars "$TAG" 369 370 if [ "$TAG" = "installed" ]; then 371 ZFS_VERSION=$(modinfo zfs | awk '/version:/ { print $2; exit }') 372 if [ -n "$ZFS_VERSION" ]; then 373 printf "%-16s" "$ZFS_VERSION" 374 else 375 fail "ZFS is not installed" 376 fi 377 else 378 printf "%-16s" "$TAG" 379 fi 380done 381echo -e "\n-----------------------------------------------------------------" 382 383# 384# Attempt to generate the tarball from your local git repository, if that 385# fails then attempt to download the tarball from Github. 386# 387printf "%-16s" "Clone ZFS" 388for TAG in $SRC_TAGS; do 389 src_set_vars "$TAG" 390 391 if [ -d "$ZFS_DIR" ]; then 392 skip_nonewline 393 elif [ "$ZFS_TAG" = "installed" ]; then 394 skip_nonewline 395 else 396 cd "$SRC_DIR" || fail "Failed 'cd $SRC_DIR'" 397 398 if [ ! -d "$SRC_DIR_ZFS" ]; then 399 mkdir -p "$SRC_DIR_ZFS" 400 fi 401 402 git archive --format=tar --prefix="$ZFS_TAG/ $ZFS_TAG" \ 403 -o "$SRC_DIR_ZFS/$ZFS_TAG.tar" &>/dev/null || \ 404 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 405 if [ -s "$SRC_DIR_ZFS/$ZFS_TAG.tar" ]; then 406 tar -xf "$SRC_DIR_ZFS/$ZFS_TAG.tar" -C "$SRC_DIR_ZFS" 407 rm "$SRC_DIR_ZFS/$ZFS_TAG.tar" 408 echo -n -e "${COLOR_GREEN}Local${COLOR_RESET}\t\t" 409 else 410 mkdir -p "$ZFS_DIR" || fail "Failed to create $ZFS_DIR" 411 curl -sL "$ZFS_URL" | tar -xz -C "$ZFS_DIR" \ 412 --strip-components=1 || \ 413 fail "Failed to download $ZFS_URL" 414 echo -n -e "${COLOR_GREEN}Remote${COLOR_RESET}\t\t" 415 fi 416 fi 417done 418printf "\n" 419 420# Build the listed tags 421printf "%-16s" "Build ZFS" 422for TAG in $SRC_TAGS; do 423 src_set_vars "$TAG" 424 425 if [ -f "$ZFS_DIR/module/zfs/zfs.ko" ]; then 426 skip_nonewline 427 elif [ "$ZFS_TAG" = "installed" ]; then 428 skip_nonewline 429 else 430 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 431 make distclean &>/dev/null 432 ./autogen.sh >>"$CONFIG_LOG" 2>&1 || \ 433 fail "Failed ZFS 'autogen.sh'" 434 # shellcheck disable=SC2086 435 ./configure $CONFIG_OPTIONS >>"$CONFIG_LOG" 2>&1 || \ 436 fail "Failed ZFS 'configure $CONFIG_OPTIONS'" 437 # shellcheck disable=SC2086 438 make $MAKE_OPTIONS >>"$MAKE_LOG" 2>&1 || \ 439 fail "Failed ZFS 'make $MAKE_OPTIONS'" 440 pass_nonewline 441 fi 442done 443printf "\n" 444echo "-----------------------------------------------------------------" 445 446# Either create a new pool using 'zpool create', or alternately restore an 447# existing pool from another ZFS implementation for compatibility testing. 448for TAG in $POOL_TAGS; do 449 pool_set_vars "$TAG" 450 SKIP=0 451 452 printf "%-16s" "$POOL_TAG" 453 rm -Rf "$POOL_DIR" 454 mkdir -p "$POOL_DIR_PRISTINE" 455 456 # Use the existing compressed image if available. 457 if [ -f "$POOL_BZIP" ]; then 458 tar -xjf "$POOL_BZIP" -C "$POOL_DIR_PRISTINE" \ 459 --strip-components=1 || \ 460 fail "Failed 'tar -xjf $POOL_BZIP" 461 # Use the installed version to create the pool. 462 elif [ "$TAG" = "installed" ]; then 463 pool_create "$TAG" 464 # A source build is available to create the pool. 465 elif [ -d "$POOL_DIR_SRC" ]; then 466 pool_create "$TAG" 467 else 468 SKIP=1 469 fi 470 471 # Verify 'zpool import' works for all listed source versions. 472 for SRC_TAG in $SRC_TAGS; do 473 474 if [ "$SKIP" -eq 1 ]; then 475 skip_nonewline 476 continue 477 fi 478 479 src_set_vars "$SRC_TAG" 480 if [ "$SRC_TAG" != "installed" ]; then 481 cd "$ZFS_DIR" || fail "Failed 'cd $ZFS_DIR'" 482 fi 483 $ZFS_SH zfs="spa_config_path=$POOL_DIR_COPY" 484 485 cp -a --sparse=always "$POOL_DIR_PRISTINE" \ 486 "$POOL_DIR_COPY" || \ 487 fail "Failed to copy $POOL_DIR_PRISTINE to $POOL_DIR_COPY" 488 POOL_NAME=$($ZPOOL_CMD import -d "$POOL_DIR_COPY" | \ 489 awk '/pool:/ { print $2; exit }') 490 491 if ! $ZPOOL_CMD import -N -d "$POOL_DIR_COPY" 492 "$POOL_NAME" &>/dev/null; then 493 fail_nonewline 494 ERROR=1 495 else 496 $ZPOOL_CMD export "$POOL_NAME" || \ 497 fail "Failed to export pool" 498 pass_nonewline 499 fi 500 501 rm -Rf "$POOL_DIR_COPY" 502 503 $ZFS_SH -u || fail "Failed to unload kmods" 504 done 505 printf "\n" 506done 507 508if [ "$KEEP" = "no" ]; then 509 rm -Rf "$TEST_DIR" 510fi 511 512exit "$ERROR" 513