1#!/bin/sh
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License, Version 1.0 only
7# (the "License").  You may not use this file except in compliance
8# with the License.
9#
10# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11# or http://www.opensolaris.org/os/licensing.
12# See the License for the specific language governing permissions
13# and limitations under the License.
14#
15# When distributing Covered Code, include this CDDL HEADER in each
16# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17# If applicable, add the following below this CDDL HEADER, with the
18# fields enclosed by brackets "[]" replaced with your own identifying
19# information: Portions Copyright [yyyy] [name of copyright owner]
20#
21# CDDL HEADER END
22#
23
24#
25# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
26#
27
28BASE_DIR=$(dirname "$0")
29SCRIPT_COMMON=common.sh
30if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
31. "${BASE_DIR}/${SCRIPT_COMMON}"
32else
33echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
34fi
35
36PROG=zfs-tests.sh
37VERBOSE="no"
38QUIET=""
39CLEANUP="yes"
40CLEANUPALL="no"
41LOOPBACK="yes"
42STACK_TRACER="no"
43FILESIZE="4G"
44DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
45RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
46FILEDIR=${FILEDIR:-/var/tmp}
47DISKS=${DISKS:-""}
48SINGLETEST=""
49SINGLETESTUSER="root"
50TAGS=""
51ITERATIONS=1
52ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
53ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
54UNAME=$(uname -s)
55RERUN=""
56
57# Override some defaults if on FreeBSD
58if [ "$UNAME" = "FreeBSD" ] ; then
59	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
60	LOSETUP=/sbin/mdconfig
61	DMSETUP=/sbin/gpart
62else
63	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
64	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
65	LOSETUP=${LOSETUP:-/sbin/losetup}
66	DMSETUP=${DMSETUP:-/sbin/dmsetup}
67fi
68
69#
70# Log an informational message when additional verbosity is enabled.
71#
72msg() {
73	if [ "$VERBOSE" = "yes" ]; then
74		echo "$@"
75	fi
76}
77
78#
79# Log a failure message, cleanup, and return an error.
80#
81fail() {
82	echo "$PROG: $1" >&2
83	cleanup
84	exit 1
85}
86
87cleanup_freebsd_loopback() {
88	for TEST_LOOPBACK in ${LOOPBACKS}; do
89		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
90			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
91			    echo "Failed to destroy: ${TEST_LOOPBACK}"
92		fi
93	done
94}
95
96cleanup_linux_loopback() {
97	for TEST_LOOPBACK in ${LOOPBACKS}; do
98		LOOP_DEV="${TEST_LOOPBACK##*/}"
99		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
100		    grep "${LOOP_DEV}" | cut -f1)
101
102		if [ -n "$DM_DEV" ]; then
103			sudo "${DMSETUP}" remove "${DM_DEV}" ||
104			    echo "Failed to remove: ${DM_DEV}"
105		fi
106
107		if [ -n "${TEST_LOOPBACK}" ]; then
108			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
109			    echo "Failed to remove: ${TEST_LOOPBACK}"
110		fi
111	done
112}
113
114#
115# Attempt to remove loopback devices and files which where created earlier
116# by this script to run the test framework.  The '-k' option may be passed
117# to the script to suppress cleanup for debugging purposes.
118#
119cleanup() {
120	if [ "$CLEANUP" = "no" ]; then
121		return 0
122	fi
123
124
125	if [ "$LOOPBACK" = "yes" ]; then
126		if [ "$UNAME" = "FreeBSD" ] ; then
127			cleanup_freebsd_loopback
128		else
129			cleanup_linux_loopback
130		fi
131	fi
132
133	for TEST_FILE in ${FILES}; do
134		rm -f "${TEST_FILE}" >/dev/null 2>&1
135	done
136
137	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
138		rm -Rf "$STF_PATH"
139	fi
140}
141trap cleanup EXIT
142
143#
144# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
145# loopback devices, and files.  This is a useful way to cleanup a previous
146# test run failure which has left the system in an unknown state.  This can
147# be dangerous and should only be used in a dedicated test environment.
148#
149cleanup_all() {
150	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
151	if [ "$UNAME" = "FreeBSD" ] ; then
152		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
153	else
154		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
155	fi
156	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
157
158	msg
159	msg "--- Cleanup ---"
160	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
161	for TEST_POOL in $TEST_POOLS; do
162		sudo "$ZPOOL" destroy "${TEST_POOL}"
163	done
164
165	if [ "$UNAME" != "FreeBSD" ] ; then
166		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
167		    grep loop | tr '\n' ' ')"
168		sudo "${DMSETUP}" remove_all
169	fi
170
171	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
172	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
173		if [ "$UNAME" = "FreeBSD" ] ; then
174			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
175		else
176			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
177		fi
178	done
179
180	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
181	for TEST_FILE in $TEST_FILES; do
182		sudo rm -f "${TEST_FILE}"
183	done
184}
185
186#
187# Takes a name as the only arguments and looks for the following variations
188# on that name.  If one is found it is returned.
189#
190# $RUNFILE_DIR/<name>
191# $RUNFILE_DIR/<name>.run
192# <name>
193# <name>.run
194#
195find_runfile() {
196	NAME=$1
197	RESULT=""
198
199	if [ -f "$RUNFILE_DIR/$NAME" ]; then
200		RESULT="$RUNFILE_DIR/$NAME"
201	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
202		RESULT="$RUNFILE_DIR/$NAME.run"
203	elif [ -f "$NAME" ]; then
204		RESULT="$NAME"
205	elif [ -f "$NAME.run" ]; then
206		RESULT="$NAME.run"
207	fi
208
209	echo "$RESULT"
210}
211
212#
213# Symlink file if it appears under any of the given paths.
214#
215create_links() {
216	dir_list="$1"
217	file_list="$2"
218
219	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
220
221	for i in $file_list; do
222		for j in $dir_list; do
223			[ ! -e "$STF_PATH/$i" ] || continue
224
225			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
226				ln -sf "$j/$i" "$STF_PATH/$i" || \
227				    fail "Couldn't link $i"
228				break
229			fi
230		done
231
232		[ ! -e "$STF_PATH/$i" ] && \
233		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
234	done
235	STF_MISSING_BIN=${STF_MISSING_BIN# }
236}
237
238#
239# Constrain the path to limit the available binaries to a known set.
240# When running in-tree a top level ./bin/ directory is created for
241# convenience, otherwise a temporary directory is used.
242#
243constrain_path() {
244	. "$STF_SUITE/include/commands.cfg"
245
246	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
247	# install to /usr/local/sbin. To avoid testing the wrong utils we
248	# need /usr/local to come before / in the path search order.
249	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
250	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
251
252	if [ "$INTREE" = "yes" ]; then
253		# Constrained path set to ./zfs/bin/
254		STF_PATH="$BIN_DIR"
255		STF_PATH_REMOVE="no"
256		STF_MISSING_BIN=""
257		if [ ! -d "$STF_PATH" ]; then
258			mkdir "$STF_PATH"
259			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
260		fi
261
262		# Special case links for standard zfs utilities
263		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
264		    ! -name .libs \) -print | tr '\n' ' ')"
265		create_links "$DIRS" "$ZFS_FILES"
266
267		# Special case links for zfs test suite utilities
268		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
269		    ! -name .libs \) -print | tr '\n' ' ')"
270		create_links "$DIRS" "$ZFSTEST_FILES"
271	else
272		# Constrained path set to /var/tmp/constrained_path.*
273		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
274		STF_PATH=$(mktemp -d "$SYSTEMDIR")
275		STF_PATH_REMOVE="yes"
276		STF_MISSING_BIN=""
277
278		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
279
280		# Special case links for standard zfs utilities
281		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
282
283		# Special case links for zfs test suite utilities
284		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
285	fi
286
287	# Standard system utilities
288	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
289	if [ "$UNAME" = "FreeBSD" ] ; then
290		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
291	else
292		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
293	fi
294	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
295
296	# Exceptions
297	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
298	if [ "$UNAME" = "Linux" ] ; then
299		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
300		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
301		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
302		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
303		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
304		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
305	elif [ "$UNAME" = "FreeBSD" ] ; then
306		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
307	fi
308}
309
310#
311# Output a useful usage message.
312#
313usage() {
314cat << EOF
315USAGE:
316$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
317
318DESCRIPTION:
319	ZFS Test Suite launch script
320
321OPTIONS:
322	-h          Show this message
323	-v          Verbose zfs-tests.sh output
324	-q          Quiet test-runner output
325	-x          Remove all testpools, dm, lo, and files (unsafe)
326	-k          Disable cleanup after test failure
327	-f          Use files only, disables block device tests
328	-S          Enable stack tracer (negative performance impact)
329	-c          Only create and populate constrained path
330	-R          Automatically rerun failing tests
331	-n NFSFILE  Use the nfsfile to determine the NFS configuration
332	-I NUM      Number of iterations
333	-d DIR      Use DIR for files and loopback devices
334	-s SIZE     Use vdevs of SIZE (default: 4G)
335	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
336	-t PATH     Run single test at PATH relative to test suite
337	-T TAGS     Comma separated list of tags (default: 'functional')
338	-u USER     Run single test as USER (default: root)
339
340EXAMPLES:
341# Run the default (linux) suite of tests and output the configuration used.
342$0 -v
343
344# Run a smaller suite of tests designed to run more quickly.
345$0 -r linux-fast
346
347# Run a single test
348$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
349
350# Cleanup a previous run of the test suite prior to testing, run the
351# default (linux) suite of tests and perform no cleanup on exit.
352$0 -x
353
354EOF
355}
356
357while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
358	case $OPTION in
359	h)
360		usage
361		exit 1
362		;;
363	v)
364		VERBOSE="yes"
365		;;
366	q)
367		QUIET="yes"
368		;;
369	x)
370		CLEANUPALL="yes"
371		;;
372	k)
373		CLEANUP="no"
374		;;
375	f)
376		LOOPBACK="no"
377		;;
378	S)
379		STACK_TRACER="yes"
380		;;
381	c)
382		constrain_path
383		exit
384		;;
385	R)
386		RERUN="yes"
387		;;
388	n)
389		nfsfile=$OPTARG
390		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
391		export NFS=1
392		. "$nfsfile"
393		;;
394	d)
395		FILEDIR="$OPTARG"
396		;;
397	I)
398		ITERATIONS="$OPTARG"
399		if [ "$ITERATIONS" -le 0 ]; then
400			fail "Iterations must be greater than 0."
401		fi
402		;;
403	s)
404		FILESIZE="$OPTARG"
405		;;
406	r)
407		RUNFILES="$OPTARG"
408		;;
409	t)
410		if [ -n "$SINGLETEST" ]; then
411			fail "-t can only be provided once."
412		fi
413		SINGLETEST="$OPTARG"
414		;;
415	T)
416		TAGS="$OPTARG"
417		;;
418	u)
419		SINGLETESTUSER="$OPTARG"
420		;;
421	?)
422		usage
423		exit
424		;;
425	esac
426done
427
428shift $((OPTIND-1))
429
430FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
431LOOPBACKS=${LOOPBACKS:-""}
432
433if [ -n "$SINGLETEST" ]; then
434	if [ -n "$TAGS" ]; then
435		fail "-t and -T are mutually exclusive."
436	fi
437	RUNFILE_DIR="/var/tmp"
438	RUNFILES="zfs-tests.$$.run"
439	SINGLEQUIET="False"
440
441	if [ -n "$QUIET" ]; then
442		SINGLEQUIET="True"
443	fi
444
445	cat >$RUNFILE_DIR/$RUNFILES << EOF
446[DEFAULT]
447pre =
448quiet = $SINGLEQUIET
449pre_user = root
450user = $SINGLETESTUSER
451timeout = 600
452post_user = root
453post =
454outputdir = /var/tmp/test_results
455EOF
456	SINGLETESTDIR=$(dirname "$SINGLETEST")
457	SINGLETESTFILE=$(basename "$SINGLETEST")
458	SETUPSCRIPT=
459	CLEANUPSCRIPT=
460
461	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
462		SETUPSCRIPT="setup"
463	fi
464
465	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
466		CLEANUPSCRIPT="cleanup"
467	fi
468
469	cat >>$RUNFILE_DIR/$RUNFILES << EOF
470
471[$SINGLETESTDIR]
472tests = ['$SINGLETESTFILE']
473pre = $SETUPSCRIPT
474post = $CLEANUPSCRIPT
475tags = ['functional']
476EOF
477fi
478
479#
480# Use default tag if none was specified
481#
482TAGS=${TAGS:='functional'}
483
484#
485# Attempt to locate the runfiles describing the test workload.
486#
487R=""
488IFS=,
489for RUNFILE in $RUNFILES; do
490	if [ -n "$RUNFILE" ]; then
491		SAVED_RUNFILE="$RUNFILE"
492		RUNFILE=$(find_runfile "$RUNFILE")
493		[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
494		R="$R,$RUNFILE"
495	fi
496
497	if [ ! -r "$RUNFILE" ]; then
498		fail "Cannot read runfile: $RUNFILE"
499	fi
500done
501unset IFS
502RUNFILES=${R#,}
503
504#
505# This script should not be run as root.  Instead the test user, which may
506# be a normal user account, needs to be configured such that it can
507# run commands via sudo passwordlessly.
508#
509if [ "$(id -u)" = "0" ]; then
510	fail "This script must not be run as root."
511fi
512
513if [ "$(sudo whoami)" != "root" ]; then
514	fail "Passwordless sudo access required."
515fi
516
517#
518# Constrain the available binaries to a known set.
519#
520constrain_path
521
522#
523# Check if ksh exists
524#
525if [ "$UNAME" = "FreeBSD" ]; then
526	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
527fi
528[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
529[ -e "$STF_SUITE/include/default.cfg" ] || fail \
530    "Missing $STF_SUITE/include/default.cfg file."
531
532#
533# Verify the ZFS module stack is loaded.
534#
535if [ "$STACK_TRACER" = "yes" ]; then
536	sudo "${ZFS_SH}" -S >/dev/null 2>&1
537else
538	sudo "${ZFS_SH}" >/dev/null 2>&1
539fi
540
541#
542# Attempt to cleanup all previous state for a new test run.
543#
544if [ "$CLEANUPALL" = "yes" ]; then
545	cleanup_all
546fi
547
548#
549# By default preserve any existing pools
550# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
551# space-delimited to newline-delimited.
552#
553if [ -z "${KEEP}" ]; then
554	KEEP="$(sudo "$ZPOOL" list -H -o name)"
555	if [ -z "${KEEP}" ]; then
556		KEEP="rpool"
557	fi
558else
559	KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
560fi
561
562#
563# NOTE: The following environment variables are undocumented
564# and should be used for testing purposes only:
565#
566# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
567# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
568#
569# See libzfs/libzfs_config.c for more information.
570#
571if [ "$UNAME" = "FreeBSD" ] ; then
572	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
573else
574	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
575fi
576
577. "$STF_SUITE/include/default.cfg"
578
579#
580# No DISKS have been provided so a basic file or loopback based devices
581# must be created for the test suite to use.
582#
583if [ -z "${DISKS}" ]; then
584	#
585	# If this is a performance run, prevent accidental use of
586	# loopback devices.
587	#
588	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
589
590	#
591	# Create sparse files for the test suite.  These may be used
592	# directory or have loopback devices layered on them.
593	#
594	for TEST_FILE in ${FILES}; do
595		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
596		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
597		    fail "Failed creating: ${TEST_FILE} ($?)"
598	done
599
600	#
601	# If requested setup loopback devices backed by the sparse files.
602	#
603	if [ "$LOOPBACK" = "yes" ]; then
604		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
605
606		for TEST_FILE in ${FILES}; do
607			if [ "$UNAME" = "FreeBSD" ] ; then
608				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
609				if [ -z "$MDDEVICE" ] ; then
610					fail "Failed: ${TEST_FILE} -> loopback"
611				fi
612				DISKS="$DISKS $MDDEVICE"
613				LOOPBACKS="$LOOPBACKS $MDDEVICE"
614			else
615				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
616				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
617				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
618				BASELOOPBACK="${TEST_LOOPBACK##*/}"
619				DISKS="$DISKS $BASELOOPBACK"
620				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
621			fi
622		done
623		DISKS=${DISKS# }
624		LOOPBACKS=${LOOPBACKS# }
625	else
626		DISKS="$FILES"
627	fi
628fi
629
630#
631# It may be desirable to test with fewer disks than the default when running
632# the performance tests, but the functional tests require at least three.
633#
634NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
635if [ "$TAGS" != "perf" ]; then
636	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
637fi
638
639#
640# Disable SELinux until the ZFS Test Suite has been updated accordingly.
641#
642if [ -x "$STF_PATH/setenforce" ]; then
643	sudo setenforce permissive >/dev/null 2>&1
644fi
645
646#
647# Enable internal ZFS debug log and clear it.
648#
649if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
650	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
651	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
652fi
653
654msg
655msg "--- Configuration ---"
656msg "Runfiles:        $RUNFILES"
657msg "STF_TOOLS:       $STF_TOOLS"
658msg "STF_SUITE:       $STF_SUITE"
659msg "STF_PATH:        $STF_PATH"
660msg "FILEDIR:         $FILEDIR"
661msg "FILES:           $FILES"
662msg "LOOPBACKS:       $LOOPBACKS"
663msg "DISKS:           $DISKS"
664msg "NUM_DISKS:       $NUM_DISKS"
665msg "FILESIZE:        $FILESIZE"
666msg "ITERATIONS:      $ITERATIONS"
667msg "TAGS:            $TAGS"
668msg "STACK_TRACER:    $STACK_TRACER"
669msg "Keep pool(s):    $KEEP"
670msg "Missing util(s): $STF_MISSING_BIN"
671msg ""
672
673export STF_TOOLS
674export STF_SUITE
675export STF_PATH
676export DISKS
677export FILEDIR
678export KEEP
679export __ZFS_POOL_EXCLUDE
680export TESTFAIL_CALLBACKS
681export PATH=$STF_PATH
682
683if [ "$UNAME" = "FreeBSD" ] ; then
684	mkdir -p "$FILEDIR" || true
685	RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXXXX")
686	REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXXXX")
687else
688	RESULTS_FILE=$(mktemp -u -t zts-results.XXXXXX -p "$FILEDIR")
689	REPORT_FILE=$(mktemp -u -t zts-report.XXXXXX -p "$FILEDIR")
690fi
691
692#
693# Run all the tests as specified.
694#
695msg "${TEST_RUNNER} ${QUIET:+-q}" \
696    "-c \"${RUNFILES}\"" \
697    "-T \"${TAGS}\"" \
698    "-i \"${STF_SUITE}\"" \
699    "-I \"${ITERATIONS}\""
700${TEST_RUNNER} ${QUIET:+-q} \
701    -c "${RUNFILES}" \
702    -T "${TAGS}" \
703    -i "${STF_SUITE}" \
704    -I "${ITERATIONS}" \
705    2>&1 | tee "$RESULTS_FILE"
706#
707# Analyze the results.
708#
709${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
710RESULT=$?
711
712if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
713	MAYBES="$($ZTS_REPORT --list-maybes)"
714	TEMP_RESULTS_FILE=$(mktemp -u -t zts-results-tmp.XXXXX -p "$FILEDIR")
715	TEST_LIST=$(mktemp -u -t test-list.XXXXX -p "$FILEDIR")
716	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
717	for test_name in $MAYBES; do
718		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
719	done
720	${TEST_RUNNER} ${QUIET:+-q} \
721	    -c "${RUNFILES}" \
722	    -T "${TAGS}" \
723	    -i "${STF_SUITE}" \
724	    -I "${ITERATIONS}" \
725	    -l "${TEST_LIST}" \
726	    2>&1 | tee "$RESULTS_FILE"
727	#
728	# Analyze the results.
729	#
730	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
731	RESULT=$?
732fi
733
734
735cat "$REPORT_FILE"
736
737RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
738if [ -d "$RESULTS_DIR" ]; then
739	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
740fi
741
742rm -f "$RESULTS_FILE" "$REPORT_FILE"
743
744if [ -n "$SINGLETEST" ]; then
745	rm -f "$RUNFILES" >/dev/null 2>&1
746fi
747
748exit ${RESULT}
749