1#!/usr/bin/env bash 2 3# 4# Copyright (c) 2017-2019, Intel Corporation. All rights reserved. 5# Copyright (c) 2016-2018, Cisco Systems, Inc. All rights reserved. 6# Copyright (c) 2016, Cray, Inc. All rights reserved. 7# Copyright (c) 2019 Amazon.com, Inc. or its affiliates. All rights reserved. 8# 9# This software is available to you under a choice of one of two 10# licenses. You may choose to be licensed under the terms of the GNU 11# General Public License (GPL) Version 2, available from the file 12# COPYING in the main directory of this source tree, or the 13# BSD license below: 14# 15# Redistribution and use in source and binary forms, with or 16# without modification, are permitted provided that the following 17# conditions are met: 18# 19# - Redistributions of source code must retain the above 20# copyright notice, this list of conditions and the following 21# disclaimer. 22# 23# - Redistributions in binary form must reproduce the above 24# copyright notice, this list of conditions and the following 25# disclaimer in the documentation and/or other materials 26# provided with the distribution. 27# 28# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 31# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 32# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 33# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 34# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 35# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 36# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 38# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 39# POSSIBILITY OF SUCH DAMAGE. 40# 41 42trap cleanup_and_exit SIGINT 43 44# 45# Default behavior with no args will use sockets provider with loopback 46# 47declare BIN_PATH 48declare PROV="" 49declare CORE="" 50declare UTIL="" 51declare TEST_TYPE="quick" 52declare SERVER="127.0.0.1" 53declare CLIENT="127.0.0.1" 54declare GOOD_ADDR="" 55declare -i VERBOSE=0 56declare -i SKIP_NEG=0 57declare COMPLEX_CFG 58declare TIMEOUT_VAL="120" 59declare STRICT_MODE=0 60declare FORK=0 61declare OOB=0 62declare C_ARGS="" 63declare S_ARGS="" 64 65declare cur_excludes="" 66declare file_excludes="" 67declare input_excludes="" 68 69declare -r c_outp=$(mktemp fabtests.c_outp.XXXXXX) 70declare -r s_outp=$(mktemp fabtests.s_outp.XXXXXX) 71 72declare -i skip_count=0 73declare -i pass_count=0 74declare -i fail_count=0 75declare -i total_failures=0 76 77if [[ "$(uname)" == "FreeBSD" ]]; then 78 declare -ri FI_ENODATA=$(python -c 'import errno; print(errno.ENOMSG)') 79else 80 declare -ri FI_ENODATA=$(python -c 'import errno; print(errno.ENODATA)') 81fi 82declare -ri FI_ENOSYS=$(python -c 'import errno; print(errno.ENOSYS)') 83 84neg_unit_tests=( 85 "fi_dgram g00n13s" 86 "fi_rdm g00n13s" 87 "fi_msg g00n13s" 88) 89 90functional_tests=( 91 "fi_av_xfer -e rdm" 92 "fi_av_xfer -e dgram" 93 "fi_cm_data" 94 "fi_cq_data -e msg" 95 "fi_cq_data -e rdm" 96 "fi_cq_data -e dgram" 97 "fi_dgram" 98 "fi_dgram_waitset" 99 "fi_msg" 100 "fi_msg_epoll" 101 "fi_msg_sockets" 102 "fi_poll -t queue" 103 "fi_poll -t counter" 104 "fi_rdm" 105 "fi_rdm_rma_simple" 106 "fi_rdm_rma_trigger" 107 "fi_shared_ctx" 108 "fi_shared_ctx --no-tx-shared-ctx" 109 "fi_shared_ctx --no-rx-shared-ctx" 110 "fi_shared_ctx -e msg" 111 "fi_shared_ctx -e msg --no-tx-shared-ctx" 112 "fi_shared_ctx -e msg --no-rx-shared-ctx" 113 "fi_shared_ctx -e dgram" 114 "fi_shared_ctx -e dgram --no-tx-shared-ctx" 115 "fi_shared_ctx -e dgram --no-rx-shared-ctx" 116 "fi_rdm_tagged_peek" 117 "fi_scalable_ep" 118 "fi_rdm_shared_av" 119 "fi_multi_mr -e msg -V" 120 "fi_multi_mr -e rdm -V" 121 "fi_recv_cancel -e rdm -V" 122 "fi_unexpected_msg -e msg -i 10" 123 "fi_unexpected_msg -e rdm -i 10" 124 "fi_unexpected_msg -e msg -S -i 10" 125 "fi_unexpected_msg -e rdm -S -i 10" 126 "fi_inj_complete -e msg" 127 "fi_inj_complete -e rdm" 128 "fi_inj_complete -e dgram" 129 "fi_inj_complete -e msg -SR" 130 "fi_inj_complete -e rdm -SR" 131 "fi_inj_complete -e dgram -SR" 132 "fi_bw -e rdm -v -T 1" 133 "fi_bw -e msg -v -T 1" 134) 135 136short_tests=( 137 "fi_msg_pingpong -I 5" 138 "fi_msg_pingpong -I 5 -v" 139 "fi_msg_bw -I 5" 140 "fi_msg_bw -I 5 -v" 141 "fi_rma_bw -e msg -o write -I 5" 142 "fi_rma_bw -e msg -o read -I 5" 143 "fi_rma_bw -e msg -o writedata -I 5" 144 "fi_rma_bw -e rdm -o write -I 5" 145 "fi_rma_bw -e rdm -o read -I 5" 146 "fi_rma_bw -e rdm -o writedata -I 5" 147 "fi_rdm_atomic -I 5 -o all" 148 "fi_rdm_cntr_pingpong -I 5" 149 "fi_multi_recv -e rdm -I 5" 150 "fi_multi_recv -e msg -I 5" 151 "fi_rdm_pingpong -I 5" 152 "fi_rdm_pingpong -I 5 -v" 153 "fi_rdm_tagged_pingpong -I 5" 154 "fi_rdm_tagged_pingpong -I 5 -v" 155 "fi_rdm_tagged_bw -I 5" 156 "fi_rdm_tagged_bw -I 5 -v" 157 "fi_dgram_pingpong -I 5" 158) 159 160standard_tests=( 161 "fi_msg_pingpong" 162 "fi_msg_pingpong -v" 163 "fi_msg_pingpong -k" 164 "fi_msg_pingpong -k -v" 165 "fi_msg_bw" 166 "fi_msg_bw -v" 167 "fi_rma_bw -e msg -o write" 168 "fi_rma_bw -e msg -o read" 169 "fi_rma_bw -e msg -o writedata" 170 "fi_rma_bw -e rdm -o write" 171 "fi_rma_bw -e rdm -o read" 172 "fi_rma_bw -e rdm -o writedata" 173 "fi_rdm_atomic -o all -I 1000" 174 "fi_rdm_cntr_pingpong" 175 "fi_multi_recv -e rdm" 176 "fi_multi_recv -e msg" 177 "fi_rdm_pingpong" 178 "fi_rdm_pingpong -v" 179 "fi_rdm_pingpong -k" 180 "fi_rdm_pingpong -k -v" 181 "fi_rdm_tagged_pingpong" 182 "fi_rdm_tagged_pingpong -v" 183 "fi_rdm_tagged_bw" 184 "fi_rdm_tagged_bw -v" 185 "fi_dgram_pingpong" 186 "fi_dgram_pingpong -k" 187) 188 189unit_tests=( 190 "fi_getinfo_test -s SERVER_ADDR GOOD_ADDR" 191 "fi_av_test -g GOOD_ADDR -n 1 -s SERVER_ADDR" 192 "fi_dom_test -n 2" 193 "fi_eq_test" 194 "fi_cq_test" 195 "fi_mr_test" 196 "fi_cntr_test" 197) 198 199complex_tests=( 200 "fi_ubertest" 201) 202 203multinode_tests=( 204 "fi_multinode -C msg" 205 "fi_multinode -C rma" 206 "fi_multinode_coll" 207) 208 209function errcho { 210 >&2 echo $* 211} 212 213function print_border { 214 printf "# " 215 printf "%.0s-" {1..78} 216 printf "\n" 217} 218 219function print_results { 220 local test_name=$1 221 local test_result=$2 222 local test_time=$3 223 local server_out_file=$4 224 local server_cmd=$5 225 local client_out_file=$6 226 local client_cmd=$7 227 228 if [ $VERBOSE -eq 0 ] ; then 229 # print a simple, single-line format that is still valid YAML 230 printf "%-70s%10s\n" "$test_exe:" "$test_result" 231 else 232 # Print a more detailed YAML format that is not a superset of 233 # the non-verbose output. See ofiwg/fabtests#259 for a 234 # rationale. 235 emit_stdout=0 236 case $test_result in 237 Pass*) 238 [ $VERBOSE -ge 3 ] && emit_stdout=1 239 ;; 240 Notrun|Excluded) 241 [ $VERBOSE -ge 2 ] && emit_stdout=1 242 ;; 243 Fail*) 244 [ $VERBOSE -ge 1 ] && emit_stdout=1 245 ;; 246 esac 247 248 printf -- "- name: %s\n" "$test_exe" 249 printf -- " result: %s\n" "$test_result" 250 printf -- " time: %s\n" "$test_time" 251 if [ $emit_stdout -eq 1 -a "$server_out_file" != "" ] ; then 252 if [ "$server_cmd" != "" ] ; then 253 printf -- " server_cmd: %s\n" "$server_cmd" 254 fi 255 printf -- " server_stdout: |\n" 256 sed -e 's/^/ /' < $server_out_file 257 fi 258 if [ $emit_stdout -eq 1 -a "$client_out_file" != "" ] ; then 259 if [ "$client_cmd" != "" ] ; then 260 printf -- " client_cmd: %s\n" "$client_cmd" 261 fi 262 printf -- " client_stdout: |\n" 263 sed -e 's/^/ /' < $client_out_file 264 fi 265 fi 266} 267 268function cleanup { 269 ${CLIENT_CMD} "ps -eo comm,pid | grep '^fi_' | awk '{print \$2}' | xargs kill -9" >& /dev/null 270 ${SERVER_CMD} "ps -eo comm,pid | grep '^fi_' | awk '{print \$2}' | xargs kill -9" >& /dev/null 271 rm -f $c_outp $s_outp 272} 273 274function cleanup_and_exit { 275 cleanup 276 exit 1 277} 278 279# compute the duration in seconds between two integer values 280# measured since the start of the UNIX epoch and print the result to stdout 281function compute_duration { 282 local -i s=$1 283 local -i e=$2 284 echo $(( $2 - $1)) 285} 286 287function read_exclude_file { 288 local excl_file=$1 289 290 if [ ! -f $excl_file ]; then 291 echo "Given exclusion file does not exist!" 292 exit 1 293 fi 294 295 while read -r pattern || [[ -n "$pattern" ]]; do 296 # Ignore patterns that are comments or just whitespaces 297 ignore_pattern="#.*|^[\t ]*$" 298 if [[ ! "$pattern" =~ $ignore_pattern ]]; then 299 if [ -z "$file_excludes" ]; then 300 file_excludes="$pattern" 301 else 302 file_excludes="${file_excludes},$pattern" 303 fi 304 fi 305 done < "$excl_file" 306} 307 308function auto_exclude { 309 local excl_file 310 local name=$UTIL 311 312 if [ -z $UTIL ]; then 313 name=$CORE 314 fi 315 316 excl_file="./fabtests/test_configs/${name}/${name}.exclude" 317 if [[ ! -f "$excl_file" ]]; then 318 excl_file="./test_configs/${name}/${name}.exclude" 319 if [[ ! -f "$excl_file" ]]; then 320 excl_file="../test_configs/${name}/${name}.exclude" 321 if [[ ! -f "$excl_file" ]]; then 322 return 323 fi 324 fi 325 fi 326 327 read_exclude_file ${excl_file} 328 cur_excludes=${file_excludes} 329 file_excludes="" 330} 331 332function set_excludes { 333 if [[ -n "$input_excludes" ]]; then 334 cur_excludes=${input_excludes} 335 fi 336 337 if [[ -n "$file_excludes" ]]; then 338 [[ -z "$cur_excludes" ]] && cur_excludes=${file_excludes} || \ 339 cur_excludes="${cur_excludes},${file_excludes}" 340 fi 341 342 if [[ -n "$cur_excludes" ]]; then 343 return 344 fi 345 346 auto_exclude 347} 348 349function is_excluded { 350 test_name=$1 351 352 [[ -z "$cur_excludes" ]] && return 1 353 354 IFS="," read -ra exclude_array <<< "$cur_excludes" 355 for pattern in "${exclude_array[@]}"; do 356 if [[ "$test_name" =~ $pattern ]]; then 357 print_results "$test_exe" "Excluded" "0" "" "" 358 skip_count+=1 359 return 0 360 fi 361 done 362 return 1 363} 364 365function unit_test { 366 local test=$1 367 local is_neg=$2 368 local ret1=0 369 local s_interface=$(eval "if [ $OOB -eq 1 ]; \ 370 then echo $GOOD_ADDR; \ 371 else echo $S_INTERFACE; \ 372 fi") 373 local test_exe=$(echo "${test} -p \"$PROV\"" | \ 374 sed -e "s/GOOD_ADDR/$GOOD_ADDR/g" -e "s/SERVER_ADDR/$s_interface/g") 375 local start_time 376 local end_time 377 local test_time 378 379 is_excluded "$test" && return 380 381 start_time=$(date '+%s') 382 383 cmd="${BIN_PATH}${test_exe}" 384 ${SERVER_CMD} "${EXPORT_ENV} $cmd" &> $s_outp & 385 p1=$! 386 387 wait $p1 388 ret=$? 389 390 end_time=$(date '+%s') 391 test_time=$(compute_duration "$start_time" "$end_time") 392 393 if [ $is_neg -eq 1 -a $ret -eq $FI_ENODATA ]; then 394 # negative test passed 395 ret=0 396 elif [ $is_neg -eq 1 ]; then 397 # negative test failed 398 ret=1 399 fi 400 if [[ $STRICT_MODE -eq 0 && $ret -eq $FI_ENODATA || $ret -eq $FI_ENOSYS ]]; then 401 print_results "$test_exe" "Notrun" "$test_time" "$s_outp" "$cmd" 402 skip_count+=1 403 elif [ $ret -ne 0 ]; then 404 print_results "$test_exe" "Fail" "$test_time" "$s_outp" "$cmd" 405 if [ $ret -eq 124 ]; then 406 cleanup 407 fi 408 fail_count+=1 409 else 410 print_results "$test_exe" "Pass" "$test_time" "$s_outp" "$cmd" 411 pass_count+=1 412 fi 413} 414 415function cs_test { 416 local test=$1 417 local s_ret=0 418 local c_ret=0 419 local test_exe="${test} -p \"${PROV}\"" 420 local start_time 421 local end_time 422 local test_time 423 424 is_excluded "$test" && return 425 426 start_time=$(date '+%s') 427 428 if [[ $OOB -eq 1 ]]; then 429 s_arg="-E" 430 else 431 s_arg="-s $S_INTERFACE" 432 fi 433 s_cmd="${BIN_PATH}${test_exe} ${S_ARGS} $s_arg" 434 ${SERVER_CMD} "${EXPORT_ENV} $s_cmd" &> $s_outp & 435 s_pid=$! 436 sleep 1 437 438 if [[ $OOB -eq 1 ]]; then 439 c_arg="-E $S_INTERFACE" 440 else 441 c_arg="-s $C_INTERFACE $S_INTERFACE" 442 fi 443 c_cmd="${BIN_PATH}${test_exe} ${C_ARGS} $c_arg" 444 ${CLIENT_CMD} "${EXPORT_ENV} $c_cmd" &> $c_outp & 445 c_pid=$! 446 447 wait $c_pid 448 c_ret=$? 449 450 [[ c_ret -ne 0 ]] && kill -9 $s_pid 2> /dev/null 451 452 wait $s_pid 453 s_ret=$? 454 455 end_time=$(date '+%s') 456 test_time=$(compute_duration "$start_time" "$end_time") 457 458 if [[ $STRICT_MODE -eq 0 && $s_ret -eq $FI_ENODATA && $c_ret -eq $FI_ENODATA ]] || 459 [[ $STRICT_MODE -eq 0 && $s_ret -eq $FI_ENOSYS && $c_ret -eq $FI_ENOSYS ]]; then 460 print_results "$test_exe" "Notrun" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 461 skip_count+=1 462 elif [ $s_ret -ne 0 -o $c_ret -ne 0 ]; then 463 print_results "$test_exe" "Fail" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 464 if [ $s_ret -eq 124 -o $c_ret -eq 124 ]; then 465 cleanup 466 fi 467 fail_count+=1 468 else 469 print_results "$test_exe" "Pass" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 470 pass_count+=1 471 fi 472} 473 474function set_cfg_file { 475 local cfg_file 476 local parent=$UTIL 477 local name=$CORE 478 479 if [ -z $UTIL ]; then 480 parent=$CORE 481 name=$1 482 fi 483 484 cfg_file="${PWD}/fabtests/test_configs/${parent}/${name}.test" 485 if [[ ! -f "$cfg_file" ]]; then 486 cfg_file="${PWD}/test_configs/${parent}/${name}.test" 487 if [[ ! -f "$cfg_file" ]]; then 488 return 489 fi 490 fi 491 492 COMPLEX_CFG=${cfg_file} 493} 494 495function complex_test { 496 local test=$1 497 local config=$2 498 local path=${PROV/;/\/} 499 local test_exe="${test}" 500 local s_ret=0 501 local c_ret=0 502 local start_time 503 local end_time 504 local test_time 505 506 is_excluded "$test" && return 507 if [[ -z "$COMPLEX_CFG" ]]; then 508 set_cfg_file $config 509 fi 510 511 start_time=$(date '+%s') 512 513 if [[ $FORK -eq 1 ]]; then 514 opts="-f" 515 else 516 opts="" 517 fi 518 519 if [[ $OOB -eq 1 ]]; then 520 opts+=" -E" 521 fi 522 523 s_cmd="${BIN_PATH}${test_exe} -x $opts" 524 FI_LOG_LEVEL=error ${SERVER_CMD} "${EXPORT_ENV} $s_cmd" &> $s_outp & 525 s_pid=$! 526 sleep 1 527 528 c_cmd="${BIN_PATH}${test_exe} -u "${COMPLEX_CFG}" $S_INTERFACE $opts" 529 FI_LOG_LEVEL=error ${CLIENT_CMD} "${EXPORT_ENV} $c_cmd" &> $c_outp & 530 c_pid=$! 531 532 wait $c_pid 533 c_ret=$? 534 535 [[ c_ret -ne 0 ]] && kill -9 $s_pid 536 537 wait $s_pid 538 s_ret=$? 539 540 end_time=$(date '+%s') 541 test_time=$(compute_duration "$start_time" "$end_time") 542 543 # case: config file doesn't exist or invalid option provided 544 if [ $s_ret -eq 1 -o $c_ret -eq 1 ]; then 545 print_results "$test_exe" "Notrun" "0" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 546 cleanup 547 skip_count+=1 548 return 549 # case: test didn't run becasue some error occured 550 elif [ $s_ret -ne 0 -o $c_ret -ne 0 ]; then 551 printf "%-50s%s\n" "$test_exe:" "Server returns $s_ret, client returns $c_ret" 552 print_results "$test_exe" "Fail [$f_cnt/$total]" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 553 cleanup 554 fail_count+=1 555 else 556 local f_cnt=$(cat $c_outp | awk -F': ' '/ENOSYS|ERROR/ {total += $2} END {print total}') 557 local s_cnt=$(cat $c_outp | awk -F': ' '/Success/ {total += $2} END {print total}') 558 local total=$(cat $c_outp | awk -F': ' '/Success|ENODATA|ENOSYS|ERROR/ {total += $2} END {print total}') 559 if [ $f_cnt -eq 0 ]; then 560 print_results "$test_exe" "Pass [$s_cnt/$total]" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 561 pass_count+=1 562 else 563 print_results "$test_exe" "Fail [$f_cnt/$total]" "$test_time" "$s_outp" "$s_cmd" "$c_outp" "$c_cmd" 564 cleanup 565 fail_count+=1 566 fi 567 fi 568} 569 570function multinode_test { 571 local test="$1" 572 local s_ret=0 573 local c_ret=0 574 local c_out_arr=() 575 local num_procs=$2 576 local test_exe="${test} -n $num_procs -p \"${PROV}\"" 577 local c_out 578 local start_time 579 local end_time 580 local test_time 581 582 is_excluded "$test" && return 583 584 start_time=$(date '+%s') 585 586 s_cmd="${BIN_PATH}${test_exe} ${S_ARGS} -s ${S_INTERFACE}" 587 ${SERVER_CMD} "${EXPORT_ENV} $s_cmd" &> $s_outp & 588 s_pid=$! 589 sleep 1 590 591 c_pid_arr=() 592 for ((i=1; i<num_procs; i++)) 593 do 594 local c_out=$(mktemp fabtests.c_outp${i}.XXXXXX) 595 c_cmd="${BIN_PATH}${test_exe} ${S_ARGS} -s ${S_INTERFACE}" 596 ${CLIENT_CMD} "${EXPORT_ENV} $c_cmd" &> $c_out & 597 c_pid_arr+=($!) 598 c_out_arr+=($c_out) 599 done 600 601 for pid in ${c_pid_arr[*]}; do 602 wait $pid 603 c_ret=($?)||$c_ret 604 done 605 606 [[ c_ret -ne 0 ]] && kill -9 $s_pid 2> /dev/null 607 608 wait $s_pid 609 s_ret=$? 610 echo "server finished" 611 612 end_time=$(date '+%s') 613 test_time=$(compute_duration "$start_time" "$end_time") 614 615 pe=1 616 if [[ $STRICT_MODE -eq 0 && $s_ret -eq $FI_ENODATA && $c_ret -eq $FI_ENODATA ]] || 617 [[ $STRICT_MODE -eq 0 && $s_ret -eq $FI_ENOSYS && $c_ret -eq $FI_ENOSYS ]]; then 618 print_results "$test_exe" "Notrun" "$test_time" "$s_outp" "$s_cmd" "" "$c_cmd" 619 for c_out in "${c_out_arr[@]}" 620 do 621 printf -- " client_stdout $pe: |\n" 622 sed -e 's/^/ /' < $c_out 623 pe=$((pe+1)) 624 done 625 skip_count+=1 626 elif [ $s_ret -ne 0 -o $c_ret -ne 0 ]; then 627 print_results "$test_exe" "Fail" "$test_time" "$s_outp" "$s_cmd" "" "$c_cmd" 628 for c_out in "${c_out_arr[@]}" 629 do 630 printf -- " client_stdout $pe: |\n" 631 sed -e 's/^/ /' < $c_out 632 pe=$((pe+1)) 633 done 634 if [ $s_ret -eq 124 -o $c_ret -eq 124 ]; then 635 cleanup 636 fi 637 fail_count+=1 638 else 639 print_results "$test_exe" "Pass" "$test_time" "$s_outp" "$s_cmd" "" "$c_cmd" 640 for c_out in "${c_out_arr[@]}" 641 do 642 printf -- " client_stdout $pe: |\n" 643 sed -e 's/^/ /' < $c_out 644 pe=$((pe+1)) 645 done 646 pass_count+=1 647 fi 648} 649 650function set_core_util { 651 prov_arr=$(echo $PROV | tr ";" " ") 652 CORE="" 653 UTIL="" 654 for p in $prov_arr; do 655 if [[ -z $CORE ]]; then 656 CORE=$p 657 else 658 UTIL=$p 659 fi 660 done 661} 662 663function main { 664 skip_count=0 665 pass_count=0 666 fail_count=0 667 local complex_type="quick" 668 669 set_core_util 670 set_excludes 671 672 673 if [[ $1 == "quick" ]]; then 674 local -r tests="unit functional short" 675 elif [[ $1 == "verify" ]]; then 676 local -r tests="complex" 677 complex_type=$1 678 else 679 local -r tests=$(echo $1 | sed 's/all/unit,functional,standard,complex,multinode/g' | tr ',' ' ') 680 if [[ $1 == "all" || $1 == "complex" ]]; then 681 complex_type="all" 682 fi 683 fi 684 685 if [ $VERBOSE -eq 0 ] ; then 686 printf "# %-68s%10s\n" "Test" "Result" 687 print_border 688 fi 689 690 for ts in ${tests}; do 691 case ${ts} in 692 unit) 693 for test in "${unit_tests[@]}"; do 694 unit_test "$test" "0" 695 done 696 697 if [ $SKIP_NEG -eq 0 ] ; then 698 for test in "${neg_unit_tests[@]}"; do 699 unit_test "$test" "1" 700 done 701 fi 702 ;; 703 functional) 704 for test in "${functional_tests[@]}"; do 705 cs_test "$test" 706 done 707 ;; 708 short) 709 for test in "${short_tests[@]}"; do 710 cs_test "$test" 711 done 712 ;; 713 standard) 714 for test in "${standard_tests[@]}"; do 715 cs_test "$test" 716 done 717 ;; 718 complex) 719 for test in "${complex_tests[@]}"; do 720 complex_test $test $complex_type 721 done 722 ;; 723 multinode) 724 for test in "${multinode_tests[@]}"; do 725 multinode_test "$test" 3 726 done 727 ;; 728 *) 729 errcho "Unknown test set: ${ts}" 730 exit 1 731 ;; 732 esac 733 done 734 735 total=$(( $pass_count + $fail_count )) 736 737 print_border 738 739 printf "# %-50s%10d\n" "Total Pass" $pass_count 740 printf "# %-50s%10d\n" "Total Notrun/Excluded" $skip_count 741 printf "# %-50s%10d\n" "Total Fail" $fail_count 742 743 if [[ "$total" > "0" ]]; then 744 printf "# %-50s%10d\n" "Percentage of Pass" $(( $pass_count * 100 / $total )) 745 fi 746 747 print_border 748 749 cleanup 750 total_failures+=$fail_count 751} 752 753function usage { 754 errcho "Usage:" 755 errcho " $0 [OPTIONS] [provider] [host] [client]" 756 errcho 757 errcho "Run fabtests using provider between host and client (default" 758 errcho "'sockets' provider in loopback-mode). Report pass/fail/notrun status." 759 errcho 760 errcho "Options:" 761 errcho -e " -g\tgood IP address from <host>'s perspective (default $GOOD_ADDR)" 762 errcho -e " -v\tprint output of failing" 763 errcho -e " -vv\tprint output of failing/notrun" 764 errcho -e " -vvv\tprint output of failing/notrun/passing" 765 errcho -e " -t\ttest set(s): all,quick,unit,functional,standard,short,complex (default quick)" 766 errcho -e " -e\texclude tests: comma delimited list of test names / 767 regex patterns e.g. \"dgram,rma.*write\"" 768 errcho -e " -E\texport provided variable name and value to ssh client and server processes. 769 options must of of the form '-E var=value'" 770 errcho -e " -f\texclude tests file: File containing list of test names / 771 regex patterns to exclude (one per line)" 772 errcho -e " -N\tskip negative unit tests" 773 errcho -e " -p\tpath to test bins (default PATH)" 774 errcho -e " -c\tclient interface" 775 errcho -e " -s\tserver/host interface" 776 errcho -e " -u\tconfigure option for complex tests" 777 errcho -e " -T\ttimeout value in seconds" 778 errcho -e " -S\tStrict mode: -FI_ENODATA, -FI_ENOSYS errors would be treated as failures instead of skipped/notrun" 779 errcho -e " -C\tAdditional client test arguments: Parameters to pass to client fabtests" 780 errcho -e " -L\tAdditional server test arguments: Parameters to pass to server fabtests" 781 errcho -e " -b\tenable out-of-band address exchange over the default port" 782 exit 1 783} 784 785while getopts ":vt:p:g:e:f:c:s:u:T:C:L:NRSbkE:" opt; do 786case ${opt} in 787 t) TEST_TYPE=$OPTARG 788 ;; 789 v) VERBOSE+=1 790 ;; 791 p) BIN_PATH="${OPTARG}/" 792 ;; 793 g) GOOD_ADDR=${OPTARG} 794 ;; 795 f) read_exclude_file ${OPTARG} 796 ;; 797 e) [[ -z "$input_excludes" ]] && input_excludes=${OPTARG} || \ 798 input_excludes="${input_excludes},${OPTARG}" 799 ;; 800 c) C_INTERFACE=${OPTARG} 801 ;; 802 s) S_INTERFACE=${OPTARG} 803 ;; 804 u) COMPLEX_CFG=${OPTARG} 805 ;; 806 T) TIMEOUT_VAL=${OPTARG} 807 ;; 808 N) SKIP_NEG+=1 809 ;; 810 R) 811 ;; 812 S) STRICT_MODE=1 813 ;; 814 b) OOB=1 815 ;; 816 k) FORK=1 817 ;; 818 C) C_ARGS="${OPTARG}" 819 ;; 820 L) S_ARGS="${OPTARG}" 821 ;; 822 E) 823 delimiter="=" 824 value=${OPTARG#*$delimiter} 825 var=${OPTARG:0:$(( ${#OPTARG} - ${#value} - ${#delimiter} ))} 826 EXPORT_STRING="export $var=\"$value\"" 827 if [[ -z $EXPORT_ENV ]] ; then 828 EXPORT_ENV="$EXPORT_STRING ;" 829 else 830 EXPORT_ENV="$EXPORT_ENV $EXPORT_STRING ;" 831 fi 832 ;; 833 :|\?) usage 834 ;; 835esac 836 837done 838 839# base ssh command 840declare bssh="ssh -n -o StrictHostKeyChecking=no -o ConnectTimeout=2 -o BatchMode=yes" 841if [ -z "$(which timeout 2> /dev/null)" ]; then 842 # forego timeout 843 declare SERVER_CMD="eval" 844 declare CLIENT_CMD="eval" 845else 846 declare SERVER_CMD="eval timeout ${TIMEOUT_VAL}" 847 declare CLIENT_CMD="eval timeout ${TIMEOUT_VAL}" 848 bssh="timeout ${TIMEOUT_VAL} ${bssh}" 849fi 850 851# shift past options 852shift $((OPTIND-1)) 853 854if [[ $# -ge 4 ]]; then 855 usage 856fi 857 858if [[ $# -ge 1 ]]; then 859 PROV=$1 860fi 861 862if [[ $# -ge 2 ]]; then 863 SERVER=$2 864 SERVER_CMD="${bssh} ${SERVER}" 865fi 866 867if [[ $# -ge 3 ]]; then 868 CLIENT=$3 869 CLIENT_CMD="${bssh} ${CLIENT}" 870fi 871 872[ -z $C_INTERFACE ] && C_INTERFACE=$CLIENT 873[ -z $S_INTERFACE ] && S_INTERFACE=$SERVER 874[ -z $GOOD_ADDR ] && GOOD_ADDR=$S_INTERFACE 875 876if [[ -z $PROV ]]; then 877 PROV="tcp" 878 main ${TEST_TYPE} 879 PROV="udp" 880 main ${TEST_TYPE} 881else 882 main ${TEST_TYPE} 883fi 884 885exit $total_failures 886