1#!/bin/sh
2# shellcheck disable=SC2154
3#
4# CDDL HEADER START
5#
6# The contents of this file are subject to the terms of the
7# Common Development and Distribution License, Version 1.0 only
8# (the "License").  You may not use this file except in compliance
9# with the License.
10#
11# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12# or https://opensource.org/licenses/CDDL-1.0.
13# See the License for the specific language governing permissions
14# and limitations under the License.
15#
16# When distributing Covered Code, include this CDDL HEADER in each
17# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18# If applicable, add the following below this CDDL HEADER, with the
19# fields enclosed by brackets "[]" replaced with your own identifying
20# information: Portions Copyright [yyyy] [name of copyright owner]
21#
22# CDDL HEADER END
23#
24
25#
26# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
27#
28
29SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
30. "${SCRIPT_COMMON}" || exit
31
32PROG=zfs-tests.sh
33VERBOSE="no"
34QUIET=""
35CLEANUP="yes"
36CLEANUPALL="no"
37KMSG=""
38LOOPBACK="yes"
39STACK_TRACER="no"
40FILESIZE="4G"
41DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
42RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
43FILEDIR=${FILEDIR:-/var/tmp}
44DISKS=${DISKS:-""}
45SINGLETEST=""
46SINGLETESTUSER="root"
47TAGS=""
48ITERATIONS=1
49ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
50ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
51UNAME=$(uname)
52RERUN=""
53KMEMLEAK=""
54
55# Override some defaults if on FreeBSD
56if [ "$UNAME" = "FreeBSD" ] ; then
57	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
58	LOSETUP=/sbin/mdconfig
59	DMSETUP=/sbin/gpart
60else
61	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
62	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
63	LOSETUP=${LOSETUP:-/sbin/losetup}
64	DMSETUP=${DMSETUP:-/sbin/dmsetup}
65fi
66
67#
68# Log an informational message when additional verbosity is enabled.
69#
70msg() {
71	if [ "$VERBOSE" = "yes" ]; then
72		echo "$@"
73	fi
74}
75
76#
77# Log a failure message, cleanup, and return an error.
78#
79fail() {
80	echo "$PROG: $1" >&2
81	cleanup
82	exit 1
83}
84
85cleanup_freebsd_loopback() {
86	for TEST_LOOPBACK in ${LOOPBACKS}; do
87		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
88			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
89			    echo "Failed to destroy: ${TEST_LOOPBACK}"
90		fi
91	done
92}
93
94cleanup_linux_loopback() {
95	for TEST_LOOPBACK in ${LOOPBACKS}; do
96		LOOP_DEV="${TEST_LOOPBACK##*/}"
97		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
98		    awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
99
100		if [ -n "$DM_DEV" ]; then
101			sudo "${DMSETUP}" remove "${DM_DEV}" ||
102			    echo "Failed to remove: ${DM_DEV}"
103		fi
104
105		if [ -n "${TEST_LOOPBACK}" ]; then
106			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
107			    echo "Failed to remove: ${TEST_LOOPBACK}"
108		fi
109	done
110}
111
112#
113# Attempt to remove loopback devices and files which where created earlier
114# by this script to run the test framework.  The '-k' option may be passed
115# to the script to suppress cleanup for debugging purposes.
116#
117cleanup() {
118	if [ "$CLEANUP" = "no" ]; then
119		return 0
120	fi
121
122
123	if [ "$LOOPBACK" = "yes" ]; then
124		if [ "$UNAME" = "FreeBSD" ] ; then
125			cleanup_freebsd_loopback
126		else
127			cleanup_linux_loopback
128		fi
129	fi
130
131	# shellcheck disable=SC2086
132	rm -f ${FILES} >/dev/null 2>&1
133
134	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
135		rm -Rf "$STF_PATH"
136	fi
137}
138trap cleanup EXIT
139
140#
141# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
142# loopback devices, and files.  This is a useful way to cleanup a previous
143# test run failure which has left the system in an unknown state.  This can
144# be dangerous and should only be used in a dedicated test environment.
145#
146cleanup_all() {
147	TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
148	if [ "$UNAME" = "FreeBSD" ] ; then
149		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
150	else
151		TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
152	fi
153	TEST_FILES=$(ls "${FILEDIR}"/file-vdev* /var/tmp/file-vdev* 2>/dev/null)
154
155	msg
156	msg "--- Cleanup ---"
157	# shellcheck disable=2116,2086
158	msg "Removing pool(s):     $(echo ${TEST_POOLS})"
159	for TEST_POOL in $TEST_POOLS; do
160		sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
161	done
162
163	if [ "$UNAME" != "FreeBSD" ] ; then
164		msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
165		    grep loop | tr '\n' ' ')"
166		sudo "${DMSETUP}" remove_all
167	fi
168
169	# shellcheck disable=2116,2086
170	msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
171	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
172		if [ "$UNAME" = "FreeBSD" ] ; then
173			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
174		else
175			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
176		fi
177	done
178
179	# shellcheck disable=2116,2086
180	msg "Removing files(s):    $(echo ${TEST_FILES})"
181	# shellcheck disable=2086
182	sudo rm -f ${TEST_FILES}
183}
184
185#
186# Takes a name as the only arguments and looks for the following variations
187# on that name.  If one is found it is returned.
188#
189# $RUNFILE_DIR/<name>
190# $RUNFILE_DIR/<name>.run
191# <name>
192# <name>.run
193#
194find_runfile() {
195	NAME=$1
196
197	if [ -f "$RUNFILE_DIR/$NAME" ]; then
198		echo "$RUNFILE_DIR/$NAME"
199	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
200		echo "$RUNFILE_DIR/$NAME.run"
201	elif [ -f "$NAME" ]; then
202		echo "$NAME"
203	elif [ -f "$NAME.run" ]; then
204		echo "$NAME.run"
205	else
206		return 1
207	fi
208}
209
210#
211# Symlink file if it appears under any of the given paths.
212#
213create_links() {
214	dir_list="$1"
215	file_list="$2"
216
217	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
218
219	for i in $file_list; do
220		for j in $dir_list; do
221			[ ! -e "$STF_PATH/$i" ] || continue
222
223			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
224				ln -sf "$j/$i" "$STF_PATH/$i" || \
225				    fail "Couldn't link $i"
226				break
227			fi
228		done
229
230		[ ! -e "$STF_PATH/$i" ] && \
231		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
232	done
233	STF_MISSING_BIN=${STF_MISSING_BIN# }
234}
235
236#
237# Constrain the path to limit the available binaries to a known set.
238# When running in-tree a top level ./bin/ directory is created for
239# convenience, otherwise a temporary directory is used.
240#
241constrain_path() {
242	. "$STF_SUITE/include/commands.cfg"
243
244	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
245	# install to /usr/local/sbin. To avoid testing the wrong utils we
246	# need /usr/local to come before / in the path search order.
247	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
248	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
249
250	if [ "$INTREE" = "yes" ]; then
251		# Constrained path set to $(top_builddir)/tests/zfs-tests/bin
252		STF_PATH="$BIN_DIR"
253		STF_PATH_REMOVE="no"
254		STF_MISSING_BIN=""
255		if [ ! -d "$STF_PATH" ]; then
256			mkdir "$STF_PATH"
257			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
258		fi
259
260		# Special case links for standard zfs utilities
261		create_links "$CMD_DIR" "$ZFS_FILES"
262
263		# Special case links for zfs test suite utilities
264		create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
265	else
266		# Constrained path set to /var/tmp/constrained_path.*
267		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
268		STF_PATH=$(mktemp -d "$SYSTEMDIR")
269		STF_PATH_REMOVE="yes"
270		STF_MISSING_BIN=""
271
272		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
273
274		# Special case links for standard zfs utilities
275		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
276
277		# Special case links for zfs test suite utilities
278		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
279	fi
280
281	# Standard system utilities
282	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
283	if [ "$UNAME" = "FreeBSD" ] ; then
284		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
285	else
286		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
287	fi
288	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
289
290	# Exceptions
291	if [ "$UNAME" = "Linux" ] ; then
292		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
293		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
294		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
295		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
296	elif [ "$UNAME" = "FreeBSD" ] ; then
297		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
298	fi
299}
300
301#
302# Output a useful usage message.
303#
304usage() {
305cat << EOF
306USAGE:
307$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
308
309DESCRIPTION:
310	ZFS Test Suite launch script
311
312OPTIONS:
313	-h          Show this message
314	-v          Verbose zfs-tests.sh output
315	-q          Quiet test-runner output
316	-x          Remove all testpools, dm, lo, and files (unsafe)
317	-k          Disable cleanup after test failure
318	-K          Log test names to /dev/kmsg
319	-f          Use files only, disables block device tests
320	-S          Enable stack tracer (negative performance impact)
321	-c          Only create and populate constrained path
322	-R          Automatically rerun failing tests
323	-m          Enable kmemleak reporting (Linux only)
324	-n NFSFILE  Use the nfsfile to determine the NFS configuration
325	-I NUM      Number of iterations
326	-d DIR      Use world-writable DIR for files and loopback devices
327	-s SIZE     Use vdevs of SIZE (default: 4G)
328	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
329	-t PATH|NAME  Run single test at PATH relative to test suite,
330	                or search for test by NAME
331	-T TAGS     Comma separated list of tags (default: 'functional')
332	-u USER     Run single test as USER (default: root)
333
334EXAMPLES:
335# Run the default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and output the configuration used.
336$0 -v
337
338# Run a smaller suite of tests designed to run more quickly.
339$0 -r linux-fast
340
341# Run a single test
342$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
343
344# Run a single test by name
345$0 -t zfs_bookmark_cliargs
346
347# Cleanup a previous run of the test suite prior to testing, run the
348# default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and perform no cleanup on exit.
349$0 -x
350
351EOF
352}
353
354while getopts 'hvqxkKfScRmn:d:s:r:?t:T:u:I:' OPTION; do
355	case $OPTION in
356	h)
357		usage
358		exit 1
359		;;
360	v)
361		VERBOSE="yes"
362		;;
363	q)
364		QUIET="yes"
365		;;
366	x)
367		CLEANUPALL="yes"
368		;;
369	k)
370		CLEANUP="no"
371		;;
372	K)
373		KMSG="yes"
374		;;
375	f)
376		LOOPBACK="no"
377		;;
378	S)
379		STACK_TRACER="yes"
380		;;
381	c)
382		constrain_path
383		exit
384		;;
385	R)
386		RERUN="yes"
387		;;
388	m)
389		KMEMLEAK="yes"
390		;;
391	n)
392		nfsfile=$OPTARG
393		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
394		export NFS=1
395		. "$nfsfile"
396		;;
397	d)
398		FILEDIR="$OPTARG"
399		;;
400	I)
401		ITERATIONS="$OPTARG"
402		if [ "$ITERATIONS" -le 0 ]; then
403			fail "Iterations must be greater than 0."
404		fi
405		;;
406	s)
407		FILESIZE="$OPTARG"
408		;;
409	r)
410		RUNFILES="$OPTARG"
411		;;
412	t)
413		if [ -n "$SINGLETEST" ]; then
414			fail "-t can only be provided once."
415		fi
416		SINGLETEST="$OPTARG"
417		;;
418	T)
419		TAGS="$OPTARG"
420		;;
421	u)
422		SINGLETESTUSER="$OPTARG"
423		;;
424	?)
425		usage
426		exit
427		;;
428	*)
429		;;
430	esac
431done
432
433shift $((OPTIND-1))
434
435FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
436LOOPBACKS=${LOOPBACKS:-""}
437
438if [ -n "$SINGLETEST" ]; then
439	if [ -n "$TAGS" ]; then
440		fail "-t and -T are mutually exclusive."
441	fi
442	RUNFILE_DIR="/var/tmp"
443	RUNFILES="zfs-tests.$$.run"
444	[ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
445
446	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
447[DEFAULT]
448pre =
449quiet = $SINGLEQUIET
450pre_user = root
451user = $SINGLETESTUSER
452timeout = 600
453post_user = root
454post =
455outputdir = /var/tmp/test_results
456EOF
457	if [ "$SINGLETEST" = "${SINGLETEST%/*}" ] ; then
458		NEWSINGLETEST=$(find "$STF_SUITE" -name "$SINGLETEST*" -print -quit)
459		if [ -z "$NEWSINGLETEST" ] ; then
460			fail "couldn't find test matching '$SINGLETEST'"
461		fi
462		SINGLETEST=$NEWSINGLETEST
463	fi
464
465	SINGLETESTDIR="${SINGLETEST%/*}"
466	SETUPDIR="$SINGLETESTDIR"
467	[ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
468	[ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
469	[ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
470
471	SINGLETESTFILE="${SINGLETEST##*/}"
472	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
473
474[$SINGLETESTDIR]
475tests = ['$SINGLETESTFILE']
476pre = $SETUPSCRIPT
477post = $CLEANUPSCRIPT
478tags = ['functional']
479EOF
480fi
481
482#
483# Use default tag if none was specified
484#
485TAGS=${TAGS:='functional'}
486
487#
488# Attempt to locate the runfiles describing the test workload.
489#
490R=""
491IFS=,
492for RUNFILE in $RUNFILES; do
493	if [ -n "$RUNFILE" ]; then
494		SAVED_RUNFILE="$RUNFILE"
495		RUNFILE=$(find_runfile "$RUNFILE") ||
496			fail "Cannot find runfile: $SAVED_RUNFILE"
497		R="$R,$RUNFILE"
498	fi
499
500	if [ ! -r "$RUNFILE" ]; then
501		fail "Cannot read runfile: $RUNFILE"
502	fi
503done
504unset IFS
505RUNFILES=${R#,}
506
507#
508# This script should not be run as root.  Instead the test user, which may
509# be a normal user account, needs to be configured such that it can
510# run commands via sudo passwordlessly.
511#
512if [ "$(id -u)" = "0" ]; then
513	fail "This script must not be run as root."
514fi
515
516if [ "$(sudo id -un)" != "root" ]; then
517	fail "Passwordless sudo access required."
518fi
519
520#
521# Constrain the available binaries to a known set.
522#
523constrain_path
524
525#
526# Check if ksh exists
527#
528if [ "$UNAME" = "FreeBSD" ]; then
529	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
530fi
531[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
532[ -e "$STF_SUITE/include/default.cfg" ] || fail \
533    "Missing $STF_SUITE/include/default.cfg file."
534
535#
536# Verify the ZFS module stack is loaded.
537#
538if [ "$STACK_TRACER" = "yes" ]; then
539	sudo "${ZFS_SH}" -S >/dev/null 2>&1
540else
541	sudo "${ZFS_SH}" >/dev/null 2>&1
542fi
543
544#
545# Attempt to cleanup all previous state for a new test run.
546#
547if [ "$CLEANUPALL" = "yes" ]; then
548	cleanup_all
549fi
550
551#
552# By default preserve any existing pools
553#
554if [ -z "${KEEP}" ]; then
555	KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
556	if [ -z "${KEEP}" ]; then
557		KEEP="rpool"
558	fi
559else
560	KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
561fi
562
563#
564# NOTE: The following environment variables are undocumented
565# and should be used for testing purposes only:
566#
567# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
568# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
569#
570# See libzfs/libzfs_config.c for more information.
571#
572__ZFS_POOL_EXCLUDE="$KEEP"
573
574. "$STF_SUITE/include/default.cfg"
575
576#
577# No DISKS have been provided so a basic file or loopback based devices
578# must be created for the test suite to use.
579#
580if [ -z "${DISKS}" ]; then
581	#
582	# If this is a performance run, prevent accidental use of
583	# loopback devices.
584	#
585	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
586
587	#
588	# Create sparse files for the test suite.  These may be used
589	# directory or have loopback devices layered on them.
590	#
591	for TEST_FILE in ${FILES}; do
592		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
593		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
594		    fail "Failed creating: ${TEST_FILE} ($?)"
595	done
596
597	#
598	# If requested setup loopback devices backed by the sparse files.
599	#
600	if [ "$LOOPBACK" = "yes" ]; then
601		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
602
603		for TEST_FILE in ${FILES}; do
604			if [ "$UNAME" = "FreeBSD" ] ; then
605				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
606				if [ -z "$MDDEVICE" ] ; then
607					fail "Failed: ${TEST_FILE} -> loopback"
608				fi
609				DISKS="$DISKS $MDDEVICE"
610				LOOPBACKS="$LOOPBACKS $MDDEVICE"
611			else
612				TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
613				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
614				BASELOOPBACK="${TEST_LOOPBACK##*/}"
615				DISKS="$DISKS $BASELOOPBACK"
616				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
617			fi
618		done
619		DISKS=${DISKS# }
620		LOOPBACKS=${LOOPBACKS# }
621	else
622		DISKS="$FILES"
623	fi
624fi
625
626#
627# It may be desirable to test with fewer disks than the default when running
628# the performance tests, but the functional tests require at least three.
629#
630NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
631if [ "$TAGS" != "perf" ]; then
632	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
633fi
634
635#
636# Disable SELinux until the ZFS Test Suite has been updated accordingly.
637#
638if command -v setenforce >/dev/null; then
639	sudo setenforce permissive >/dev/null 2>&1
640fi
641
642#
643# Enable internal ZFS debug log and clear it.
644#
645if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
646	sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
647	sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
648fi
649
650msg
651msg "--- Configuration ---"
652msg "Runfiles:        $RUNFILES"
653msg "STF_TOOLS:       $STF_TOOLS"
654msg "STF_SUITE:       $STF_SUITE"
655msg "STF_PATH:        $STF_PATH"
656msg "FILEDIR:         $FILEDIR"
657msg "FILES:           $FILES"
658msg "LOOPBACKS:       $LOOPBACKS"
659msg "DISKS:           $DISKS"
660msg "NUM_DISKS:       $NUM_DISKS"
661msg "FILESIZE:        $FILESIZE"
662msg "ITERATIONS:      $ITERATIONS"
663msg "TAGS:            $TAGS"
664msg "STACK_TRACER:    $STACK_TRACER"
665msg "Keep pool(s):    $KEEP"
666msg "Missing util(s): $STF_MISSING_BIN"
667msg ""
668
669export STF_TOOLS
670export STF_SUITE
671export STF_PATH
672export DISKS
673export FILEDIR
674export KEEP
675export __ZFS_POOL_EXCLUDE
676export TESTFAIL_CALLBACKS
677
678mktemp_file() {
679	if [ "$UNAME" = "FreeBSD" ]; then
680		mktemp -u "${FILEDIR}/$1.XXXXXX"
681	else
682		mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
683	fi
684}
685mkdir -p "$FILEDIR" || :
686RESULTS_FILE=$(mktemp_file zts-results)
687REPORT_FILE=$(mktemp_file zts-report)
688
689#
690# Run all the tests as specified.
691#
692msg "${TEST_RUNNER}" \
693    "${QUIET:+-q}" \
694    "${KMEMLEAK:+-m}" \
695    "${KMSG:+-K}" \
696    "-c \"${RUNFILES}\"" \
697    "-T \"${TAGS}\"" \
698    "-i \"${STF_SUITE}\"" \
699    "-I \"${ITERATIONS}\""
700{ PATH=$STF_PATH \
701    ${TEST_RUNNER} \
702    ${QUIET:+-q} \
703    ${KMEMLEAK:+-m} \
704    ${KMSG:+-K} \
705    -c "${RUNFILES}" \
706    -T "${TAGS}" \
707    -i "${STF_SUITE}" \
708    -I "${ITERATIONS}" \
709    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
710read -r RUNRESULT <"$REPORT_FILE"
711
712#
713# Analyze the results.
714#
715${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
716RESULT=$?
717
718if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
719	MAYBES="$($ZTS_REPORT --list-maybes)"
720	TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
721	TEST_LIST=$(mktemp_file test-list)
722	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
723	for test_name in $MAYBES; do
724		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
725	done
726	{ PATH=$STF_PATH \
727	    ${TEST_RUNNER} \
728	        ${QUIET:+-q} \
729	        ${KMEMLEAK:+-m} \
730	    -c "${RUNFILES}" \
731	    -T "${TAGS}" \
732	    -i "${STF_SUITE}" \
733	    -I "${ITERATIONS}" \
734	    -l "${TEST_LIST}" \
735	    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
736	read -r RUNRESULT <"$REPORT_FILE"
737	#
738	# Analyze the results.
739	#
740	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
741	RESULT=$?
742fi
743
744
745cat "$REPORT_FILE"
746
747RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
748if [ -d "$RESULTS_DIR" ]; then
749	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
750fi
751
752rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
753
754if [ -n "$SINGLETEST" ]; then
755	rm -f "$RUNFILES" >/dev/null 2>&1
756fi
757
758[ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"
759