1#! test/script/for/moderni/sh 2#! use safe -k 3#! use sys 4#! use var 5 6# Main execution script for the modernish regression test suite. 7# See README.md or type 'modernish --test -h' for more information. 8# 9# --- begin license --- 10# Copyright (c) 2019 Martijn Dekker <martijn@inlv.org>, Groningen, Netherlands 11# 12# Permission to use, copy, modify, and/or distribute this software for any 13# purpose with or without fee is hereby granted, provided that the above 14# copyright notice and this permission notice appear in all copies. 15# 16# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 17# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 18# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 19# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 20# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 21# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 22# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 23# --- end license --- 24 25showusage() { 26 echo "usage: modernish --test [ -ehqsx ] [ -t FILE[:NUM[,NUM,...]][/...] ] [ -F PATH ]" 27 echo " -e: disable or reduce expensive regression tests" 28 echo " -h: show this help" 29 echo " -q: quiet operation (use 2x for quieter, 3x for quietest)" 30 echo " -s: silent operation" 31 echo " -t: run specific tests by name and/or number, e.g.: -t match:3,4/stack" 32 echo " -x: produce xtrace, keep fails (use 2x to keep xfails, 3x to keep all)" 33 echo " -E: don't run tests, output commands to edit them instead (use with -t)" 34 echo " -F: specify 'find' utility to use for testing 'LOOP find'" 35} 36 37if ! test -n "${MSH_VERSION+s}"; then 38 echo "Run me with: modernish --test" >&2 39 showusage 40 exit 1 41fi 42 43chdir $MSH_PREFIX/$testsdir 44 45# parse options 46let "opt_e = opt_q = opt_s = opt_x = opt_E = 0" 47unset -v opt_t opt_F 48while getopts 'ehqst:xEF:' opt; do 49 case $opt in 50 ( \? ) exit -u 1 ;; 51 ( e ) inc opt_e ;; # disable/reduce expensive tests 52 ( h ) exit -u 0 ;; 53 ( q ) inc opt_q ;; # quiet operation 54 ( s ) inc opt_s ;; # silent operation 55 ( t ) opt_t=$OPTARG ;; # run specific tests 56 ( x ) inc opt_x ;; # produce xtrace 57 ( E ) inc opt_E ;; # output commands to edit tests 58 ( F ) opt_F=$OPTARG ;; 59 ( * ) thisshellhas BUG_GETOPTSMA && str eq $opt ':' && exit -u 1 60 exit 3 'internal error' ;; 61 esac 62done 63shift $(($OPTIND - 1)) 64case $# in 65( [!0]* ) exit -u 1 ;; 66esac 67if let opt_E; then 68 thisshellhas BUG_LNNOALIAS && exit 2 '-E requires a shell without BUG_LNNOALIAS' 69 thisshellhas BUG_LNNONEG && exit 2 '-E requires a shell without BUG_LNNONEG' 70 thisshellhas LINENO || exit 2 '-E requires a shell with LINENO' 71 let "opt_q = 3" "opt_e = opt_s = opt_x = 0" 72 editor_cmdline=${VISUAL:-${EDITOR:-vi}} 73fi 74 75# Before we change PATH, explicitly init var/loop/find so it has a chance to 76# find a standards-compliant 'find' utility in a nonstandard path if necessary. 77use var/loop/find -B ${opt_F+$opt_F} # '-B' allows compatibilty mode for obsolete/broken 'find' util 78 79# Make things awkward as an extra robustness test: 80# - Run the test suite with no PATH; modernish *must* cope with this, even 81# on 'yash -o posix' which does $PATH lookups on all regular builtins. 82PATH=/dev/null 83# - Run with 'umask 777' (zero default file permissions). This is to check 84# that library functions set safe umasks whenever files are created. It 85# also checks for BUG_HDOCMASK compatibility with here-documents. 86umask 777 87 88if let opt_s; then 89 opt_q=999 90 exec >/dev/null 91fi 92 93if let opt_x; then 94 # Create temporary directory for trace output (one file per test). 95 mktemp -ds /tmp/msh-xtrace.XXXXXXXXXX 96 xtracedir=$REPLY 97 shellquote xtracedir_q=$REPLY 98 if gt opt_x 2; then 99 shellquote xtracemsg_q="Leaving all xtraces in $xtracedir_q" 100 pushtrap "putln $xtracemsg_q >&4" INT PIPE TERM EXIT DIE 101 else 102 if gt opt_x 1; then 103 shellquote xtracemsg_q="Leaving failed and xfailed tests' xtraces in $xtracedir_q" 104 else 105 shellquote xtracemsg_q="Leaving failed tests' xtraces in $xtracedir_q" 106 fi 107 pushtrap "PATH=\$DEFPATH command rmdir $xtracedir_q 2>/dev/null || \ 108 putln $xtracemsg_q >&4" INT PIPE TERM EXIT DIE 109 fi 110fi 111 112# Parse -t option argument. 113# Format: one or more slash-separated entries, consisting of a test set name pattern (matching the basename of 114# one or more '*.t' scripts), optionally followed by a semicolon and a comma-separated list of test numbers. 115if not isset opt_t; then 116 opt_t='*' 117fi 118allsets= 119allnums= 120LOOP for --split=/ testspec in $opt_t; DO 121 LOCAL --split=: -- $testspec; BEGIN 122 if lt $# 1 || str empty $1; then 123 exit -u 1 "--test: -t: empty test set name" 124 fi 125 testsetpat=$1 # test set name pattern 126 testnums=${2-} # list of test numbers 127 END 128 if not str end $testsetpat '.t'; then 129 testsetpat=$testsetpat.t 130 fi 131 LOCAL IFS='' --glob -- $testsetpat; BEGIN 132 lt $# 1 && exit 1 "--test: -t: no such test set: ${testsetpat%.t}" 133 IFS=$CCn 134 testsets="$*" # $CCn-separated glob results 135 END 136 LOOP for --split=$CCn testset in $testsets; DO 137 testset=${testset%.t} 138 append --sep=: allsets $testset 139 if not str empty $testnums; then 140 validatednums= 141 LOOP for --split=,$WHITESPACE testnum in $testnums; DO 142 str match $testnum *[!0123456789]* && exit -u 1 "--test: -t: invalid test number: $testnum" 143 while str begin $testnum 0; do 144 testnum=${testnum#0} 145 done 146 if not str in ",$validatednums," ",$testnum,"; then 147 append --sep=, validatednums $testnum 148 fi 149 DONE 150 append --sep=/ allnums $testset:$validatednums 151 fi 152 DONE 153DONE 154 155# do this at the end of option parsing so error messages are not suppressed with -qq and -s 156exec 4>&2 # save stderr in 4 for msgs from traps 157if let "opt_q > 1 && opt_E == 0"; then 158 exec 2>/dev/null 159fi 160 161# determine terminal capabilities 162tReset= 163tRed= 164tBold= 165if is onterminal 1 && extern -pv tput >/dev/null; then 166 harden -p -e '>4' tput 167 if tReset=$(tput sgr0); then # tput uses terminfo codes 168 tBold=$(tput bold) 169 tRed=$tBold$(tput setaf 1) 170 elif tReset=$(tput me); then # tput uses termcap codes 171 tBold=$(tput md) 172 tRed=$tBold$(tput AF 1) 173 fi 2>/dev/null 174fi 175 176# Harden utilities used below and in tests, searching them in the system default PATH. 177harden -pP cat 178harden -p find 179harden -p ln 180harden -p mkdir -m 700 # u+rwx,go-rwx 181harden -p pr 182harden -p rm 183harden -p sed 184harden -p sort 185harden -p wc 186if thisshellhas BUG_PFRPAD; then 187 # use external 'printf' to circumvent right-hand blank padding bug in printf builtin 188 harden -pX printf 189else 190 harden -p printf 191fi 192 193# Run all the bug/quirk/feature tests and cache their results. 194thisshellhas --cache 195 196if lt opt_q 2; then 197 # intro 198 putln "$tReset$tBold--- modernish $MSH_VERSION regression test suite ---$tReset" 199 200 # Identify the version of this shell, if possible. 201 . $MSH_AUX/id.sh 202fi 203 204# A couple of helper functions for regression tests that verify bug/quirk/feature detection. 205# The exit status of these helper functions is to be passed down by the doTest* functions. 206mustNotHave() { 207 if not thisshellhas $1; then 208 case $1 in 209 ( BUG_* | QRK_* | WRN_* ) 210 ;; 211 ( * ) okmsg="no $1${okmsg:+ ($okmsg)}" 212 skipmsg="no $1${skipmsg:+ ($skipmsg)}" ;; 213 esac 214 else 215 failmsg="$1 wrongly detected${failmsg:+ ($failmsg)}" 216 return 1 217 fi 218} 219mustHave() { 220 if thisshellhas $1; then 221 case $1 in 222 ( BUG_* | WRN_* ) 223 xfailmsg="$1${xfailmsg:+ ($xfailmsg)}" 224 return 2 ;; 225 esac 226 okmsg=$1 227 else 228 failmsg="$1 not detected${failmsg:+ ($failmsg)}" 229 return 1 230 fi 231} 232 233# Helper function for tests that are only applicable in a UTF-8 locale. 234# Usage: utf8Locale || return 235utf8Locale() { 236 case ${LC_ALL:-${LC_CTYPE:-${LANG:-}}} in 237 ( *[Uu][Tt][Ff]8* | *[Uu][Tt][Ff]-8* ) 238 ;; 239 ( * ) skipmsg="non-UTF-8 locale${skipmsg:+ ($skipmsg)}" 240 return 3 ;; 241 esac 242} 243 244# Helper function to skip or reduce expensive tests. 245# Use: runExpensive || return 246# runExpensive || { reduce expense somehow; } 247runExpensive() { 248 if gt opt_e 0; then 249 skipmsg="expensive${skipmsg:+ ($skipmsg)}" 250 return 3 251 fi 252} 253 254# Create a temporary directory for the tests to use. 255# modernish mktemp: [s]ilent (no output); auto-[C]leanup; [d]irectory; store path in $REPLY 256mktemp -sCCCd /tmp/msh-test.XXXXXX 257tempdir=$REPLY 258 259# Tests in *.t are delimited by these aliases. 260let opt_E && alias TEST='{ _TESTLINE=$LINENO; testFn() {' || alias TEST='{ testFn() {' 261alias ENDT='}; doTest; }' 262 263# Function to run one test, called upon expanding the ENDT alias. 264doTest() { 265 inc num 266 if isset nums; then 267 not str in $nums ",$num," && return 268 replacein -a nums "$num," '' 269 fi 270 inc total 271 title='(untitled)' 272 unset -v okmsg failmsg xfailmsg skipmsg 273 if let opt_x; then 274 case $num in 275 ( ? ) xtracefile=00$num ;; 276 ( ?? ) xtracefile=0$num ;; 277 ( * ) xtracefile=$num ;; 278 esac 279 xtracefile=$xtracedir/${testscript##*/}.$xtracefile.out 280 umask 022 281 command : >$xtracefile || die "tst/run.sh: cannot create $xtracefile" 282 umask 777 283 { 284 set -x 285 testFn 286 result=$? 287 set +x 288 } 2>|$xtracefile 289 gt $? 0 && die "tst/run.sh: cannot write to $xtracefile" 290 elif let opt_E; then 291 editor_cmdline="${editor_cmdline} +${_TESTLINE} $testsdir/$testscript" 292 result=3 293 else 294 testFn 295 result=$? 296 fi 297 case $result in 298 ( 0 ) resultmsg=ok${okmsg+\: $okmsg} 299 let "opt_x > 0 && opt_x < 3" && { rm $xtracefile & } 300 inc oks ;; 301 ( 1 ) resultmsg=${tRed}FAIL${tReset}${failmsg+\: $failmsg} 302 inc fails ;; 303 ( 2 ) resultmsg=xfail${xfailmsg+\: $xfailmsg} 304 let "opt_x > 0 && opt_x < 2" && { rm $xtracefile & } 305 inc xfails ;; 306 ( 3 ) resultmsg=skipped${skipmsg+\: $skipmsg} 307 let "opt_x > 0 && opt_x < 3" && { rm $xtracefile & } 308 inc skips ;; 309 ( * ) die "$testset test $num: unexpected status $result" ;; 310 esac 311 if let "opt_q==0 || result==1 || (opt_q==1 && result==2)"; then 312 if isset -v header; then 313 putln $header 314 unset -v header 315 fi 316 printf ' %03d: %-40s - %s\n' $num $title $resultmsg 317 fi 318} 319 320# Run the tests. 321let "oks = fails = xfails = skips = total = 0" 322LOOP for --split=: testset in $allsets; DO 323 testscript=$testset.t 324 header="* ${tBold}$testsdir/$tRed$testset$tReset$tBold.t$tReset " 325 unset -v v 326 # ... determine which tests to execute 327 if str in "/$allnums/" "/$testset:"; then 328 # only execute numbers given with -t 329 nums=/$allnums 330 nums=${nums##*/$testset:} 331 nums=",${nums%%/*}," 332 else 333 unset -v nums 334 fi 335 # ... run the test script, automatically numbering the test functions 336 num=0 337 source $testscript # don't add '&& ...' or '|| ...' here; this kills tests involving ERR/ZERR traps 338 if not so; then 339 exit 128 "$testscript: failed to source" 340 fi 341 if isset nums && not str eq $nums ','; then 342 trim nums ',' 343 exit 128 "$testscript: not found: $nums" 344 fi 345DONE 346 347# report 348if let opt_E; then 349 putln $editor_cmdline 350elif lt opt_q 3; then 351 eq total 1 && v1=test || v1=tests 352 eq skips 1 && v2=was || v2=were 353 putln "Out of $total $v1:" "- $oks succeeded" "- $skips $v2 skipped" "- $xfails failed expectedly" 354fi 355if gt fails 0; then 356 putln "$tRed- $fails failed unexpectedly$tReset" 357 if lt opt_q 2 && gt opt_x 0; then 358 putln " Please report bug with xtrace at ${tBold}https://github.com/modernish/modernish/tree/0.16$tReset" 359 fi 360elif lt opt_q 3; then 361 putln "- 0 failed unexpectedly" 362fi 363wait 364 365# return/exit unsuccessfully if there were failures 366eq fails 0 367