1#!/bin/bash 2# Bash functions to run an executable for testing. 3# 4# Version: 20201215 5# 6# When CHECK_WITH_ASAN is set to a non-empty value the test executable 7# is run with asan, otherwise it is run without. 8# 9# When CHECK_WITH_GDB is set to a non-empty value the test executable 10# is run with gdb, otherwise it is run without. 11# 12# When CHECK_WITH_STDERR is set to a non-empty value the test executable 13# is run with error output to stderr. 14# 15# When CHECK_WITH_VALGRIND is set to a non-empty value the test executable 16# is run with valgrind, otherwise it is run without. 17 18EXIT_SUCCESS=0; 19EXIT_FAILURE=1; 20EXIT_IGNORE=77; 21 22# Checks the availability of a binary and exits if not available. 23# 24# Arguments: 25# a string containing the name of the binary 26# 27assert_availability_binary() 28{ 29 local BINARY=$1; 30 31 which ${BINARY} > /dev/null 2>&1; 32 if test $? -ne ${EXIT_SUCCESS}; 33 then 34 echo "Missing binary: ${BINARY}"; 35 echo ""; 36 37 exit ${EXIT_FAILURE}; 38 fi 39} 40 41# Checks the availability of binaries and exits if not available. 42# 43# Globals: 44# CHECK_WITH_GDB 45# CHECK_WITH_VALGRIND 46# 47assert_availability_binaries() 48{ 49 assert_availability_binary cat; 50 assert_availability_binary cut; 51 assert_availability_binary diff; 52 assert_availability_binary file; 53 assert_availability_binary gzip; 54 assert_availability_binary ls; 55 assert_availability_binary readlink; 56 assert_availability_binary sed; 57 assert_availability_binary tr; 58 assert_availability_binary uname; 59 assert_availability_binary wc; 60 assert_availability_binary zcat; 61 62 if test -n "${CHECK_WITH_GDB}"; 63 then 64 assert_availability_binary gdb; 65 66 elif test -n "${CHECK_WITH_VALGRIND}"; 67 then 68 assert_availability_binary valgrind; 69 fi 70} 71 72# Checks if the test set is in the ignore list. 73# 74# Arguments: 75# a string containing the test set 76# a string containing space separated ignore list 77# 78# Returns: 79# an integer containing the exit status to indicate the input directory 80# was found in the ignore list. 81# 82check_for_test_set_in_ignore_list() 83{ 84 local TEST_SET=$1; 85 local IGNORE_LIST=$2; 86 87 local IFS=" "; 88 89 for LIST_ELEMENT in ${IGNORE_LIST}; 90 do 91 if test "${LIST_ELEMENT}" = "${TEST_SET}"; 92 then 93 return ${EXIT_SUCCESS}; 94 fi 95 done 96 return ${EXIT_FAILURE}; 97} 98 99# Checks if the input directory is in the ignore list. 100# 101# Arguments: 102# a string containing the path of the test input directory 103# a string containing space separated ignore list 104# 105# Returns: 106# an integer containing the exit status to indicate the input directory 107# was found in the ignore list. 108# 109check_for_directory_in_ignore_list() 110{ 111 local INPUT_DIRECTORY=$1; 112 local IGNORE_LIST=$2; 113 114 local INPUT_BASENAME=`basename ${INPUT_DIRECTORY}`; 115 116 local IFS=" "; 117 118 for LIST_ELEMENT in ${IGNORE_LIST}; 119 do 120 if test "${LIST_ELEMENT}" = "${INPUT_BASENAME}"; 121 then 122 return ${EXIT_SUCCESS}; 123 fi 124 done 125 return ${EXIT_FAILURE}; 126} 127 128# Searches for the binary variant of the executable in case the test executable 129# refers to a libtool shell script. 130# 131# Arguments: 132# a string containing the path of the test executable 133# 134# Returns: 135# a string containing the path of the binary variant of the test executable 136# 137find_binary_executable() 138{ 139 local TEST_EXECUTABLE=$1; 140 141 TEST_EXECUTABLE=$( readlink_f "${TEST_EXECUTABLE}" ); 142 143 # Note that the behavior of `file -bi` is not helpful on Mac OS X. 144 local EXECUTABLE_TYPE=`file -b ${TEST_EXECUTABLE}`; 145 146 # Check if the test executable is a libtool shell script. 147 # Linux: POSIX shell script, ASCII text executable, with very long lines 148 # Mac OS X: POSIX shell script text executable 149 echo "${EXECUTABLE_TYPE}" | grep "POSIX shell script" > /dev/null 2>&1; 150 RESULT=$?; 151 152 if test ${RESULT} -eq ${EXIT_SUCCESS}; 153 then 154 local TEST_EXECUTABLE_BASENAME=`basename ${TEST_EXECUTABLE}`; 155 local TEST_EXECUTABLE_DIRNAME=`dirname ${TEST_EXECUTABLE}`; 156 157 TEST_EXECUTABLE="${TEST_EXECUTABLE_DIRNAME}/.libs/${TEST_EXECUTABLE_BASENAME}"; 158 159 if test -x ${TEST_EXECUTABLE}; 160 then 161 # Note that the behavior of `file -bi` is not helpful on Mac OS X. 162 EXECUTABLE_TYPE=`file -b ${TEST_EXECUTABLE}`; 163 164 # Linux: ELF 64-bit LSB executable, x86-64, ... 165 # Mac OS X: Mach-O 64-bit executable x86_64 166 echo "${EXECUTABLE_TYPE}" | grep "executable" > /dev/null 2>&1; 167 RESULT=$?; 168 169 if test ${RESULT} -ne ${EXIT_SUCCESS}; 170 then 171 echo "Invalid test executable: ${TEST_EXECUTABLE}"; 172 173 exit ${EXIT_FAILURE}; 174 fi 175 fi 176 fi 177 echo ${TEST_EXECUTABLE}; 178} 179 180# Searches for the path to the binary variant of the library. 181# 182# Arguments: 183# a string containing the path of the test executable 184# 185# Returns: 186# a string containing the path of the binary variant of the library. 187# 188find_binary_library_path() 189{ 190 local TEST_EXECUTABLE=$1; 191 local LIBRARY_NAME="${TEST_EXECUTABLE}"; 192 193 echo ${LIBRARY_NAME} | grep 'tools' > /dev/null 2>&1; 194 195 if test $? -eq ${EXIT_SUCCESS}; 196 then 197 LIBRARY_NAME=`dirname ${LIBRARY_NAME}`; 198 LIBRARY_NAME=`dirname ${LIBRARY_NAME}`; 199 LIBRARY_NAME=`basename ${LIBRARY_NAME} | sed 's/\(.*\)tools$/lib\1/'`; 200 else 201 LIBRARY_NAME=`basename ${LIBRARY_NAME} | sed 's/^py//' | sed 's/^\([^_]*\)_test_.*$/lib\1/'`; 202 fi 203 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 204 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 205 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 206 207 local LIBRARY_PATH="${TEST_EXECUTABLE}/${LIBRARY_NAME}/.libs"; 208 209 if ! test -d "${LIBRARY_PATH}"; 210 then 211 LIBRARY_PATH="../${LIBRARY_NAME}/.libs"; 212 fi 213 echo "${LIBRARY_PATH}"; 214} 215 216# Searches for the path to the binary variant of the Python module 217# 218# Globals: 219# PYTHON_VERSION 220# 221# Arguments: 222# a string containing the path of the test executable 223# 224# Returns: 225# a string containing the path of the binary variant of the Python module 226# 227find_binary_python_module_path() 228{ 229 local TEST_EXECUTABLE=$1; 230 231 local PYTHON_MODULE_NAME=`basename ${TEST_EXECUTABLE} | sed 's/^py\(.*\)_test_.*$/py\1/'`; 232 233 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 234 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 235 TEST_EXECUTABLE=`dirname ${TEST_EXECUTABLE}`; 236 237 PYTHON_VERSION=`echo ${PYTHON_VERSION} | cut -c1`; 238 239 local PYTHON_MODULE_PATH="${TEST_EXECUTABLE}/${PYTHON_MODULE_NAME}-python${PYTHON_VERSION}/.libs"; 240 241 if ! test -d "${PYTHON_MODULE_PATH}"; 242 then 243 PYTHON_MODULE_PATH="../${PYTHON_MODULE_NAME}-python${PYTHON_VERSION}/.libs"; 244 fi 245 if ! test -d "${PYTHON_MODULE_PATH}"; 246 then 247 PYTHON_MODULE_PATH="${TEST_EXECUTABLE}/${PYTHON_MODULE_NAME}/.libs"; 248 fi 249 if ! test -d "${PYTHON_MODULE_PATH}"; 250 then 251 PYTHON_MODULE_PATH="../${PYTHON_MODULE_NAME}/.libs"; 252 fi 253 echo "${PYTHON_MODULE_PATH}"; 254} 255 256# Determines the test data option file. 257# 258# Arguments: 259# a string containing the path of the test set directory 260# a string containing the path of the test input file 261# a string containing the name of the test data option set 262# 263# Returns: 264# a string containing the test input files 265# 266get_test_data_option_file() 267{ 268 local TEST_SET_DIRECTORY=$1; 269 local INPUT_FILE=$2; 270 local OPTION_SET=$3; 271 272 local INPUT_NAME=`basename "${INPUT_FILE}"`; 273 local TEST_DATA_OPTION_FILE="${TEST_SET_DIRECTORY}/${INPUT_NAME}.${OPTION_SET}"; 274 275 echo "${TEST_DATA_OPTION_FILE}"; 276} 277 278# Determines the test profile directory. 279# The directory is created if it does not exist. 280# 281# Arguments: 282# a string containing the path of the test input directory 283# a string containing the name of the test profile 284# 285# Returns: 286# a string containing the path of the test profile directory 287# 288get_test_profile_directory() 289{ 290 local TEST_INPUT_DIRECTORY=$1; 291 local TEST_PROFILE=$2; 292 293 local TEST_PROFILE_DIRECTORY="${TEST_INPUT_DIRECTORY}/.${TEST_PROFILE}"; 294 295 if ! test -d "${TEST_PROFILE_DIRECTORY}"; 296 then 297 mkdir "${TEST_PROFILE_DIRECTORY}"; 298 fi 299 echo "${TEST_PROFILE_DIRECTORY}"; 300} 301 302# Determines the test set directory. 303# The directory is created if it does not exist. 304# 305# Arguments: 306# a string containing the path of the test profile directory 307# a string containing the path of the test set input directory 308# 309# Returns: 310# a string containing the path of the test set directory 311# 312get_test_set_directory() 313{ 314 local TEST_PROFILE_DIRECTORY=$1; 315 local TEST_SET_INPUT_DIRECTORY=$2; 316 317 local TEST_SET=`basename ${TEST_SET_INPUT_DIRECTORY}`; 318 local TEST_SET_DIRECTORY="${TEST_PROFILE_DIRECTORY}/${TEST_SET}"; 319 320 if ! test -d "${TEST_SET_DIRECTORY}"; 321 then 322 mkdir "${TEST_SET_DIRECTORY}"; 323 fi 324 echo "${TEST_SET_DIRECTORY}"; 325} 326 327# Provides a cross-platform variant of "readlink -f" 328# 329# Arguments: 330# a string containing a path 331# 332# Returns: 333# a string containing the path with all symbolic links resolved 334# 335readlink_f() { 336 local TARGET="$1" 337 338 if test -f "${TARGET}"; 339 then 340 while test -L "${TARGET}"; 341 do 342 TARGET=`readlink "${TARGET}"`; 343 done 344 fi 345 local BASENAME=`basename "${TARGET}"`; 346 local DIRNAME=`dirname "${TARGET}"`; 347 348 DIRNAME=`(cd "${DIRNAME}" && pwd -P)`; 349 350 echo "${DIRNAME}/${BASENAME}"; 351} 352 353# Reads the test profile ignore file if it exists. 354# 355# Arguments: 356# a string containing the path of the test profile directory 357# 358# Returns: 359# a string containing a space separated ignore list 360# 361read_ignore_list() 362{ 363 local TEST_PROFILE_DIRECTORY=$1; 364 local IGNORE_FILE="${TEST_PROFILE_DIRECTORY}/ignore"; 365 local IGNORE_LIST=""; 366 367 if test -f "${IGNORE_FILE}"; 368 then 369 IGNORE_LIST=`cat ${IGNORE_FILE} | sed '/^#/d' | tr '\n' ' '`; 370 fi 371 echo ${IGNORE_LIST}; 372} 373 374# Reads a test data option file. 375# 376# Arguments: 377# a string containing the path of the test set directory 378# a string containing the path of the test input file 379# a string containing the name of the test data option set 380# 381# Returns: 382# a string containing the test data specific options 383# 384read_test_data_option_file() 385{ 386 local TEST_SET_DIRECTORY=$1; 387 local INPUT_FILE=$2; 388 local OPTION_SET=$3; 389 390 local TEST_DATA_OPTION_FILE="${INPUT_FILE}.${OPTION_SET}"; 391 392 if ! test -f "${TEST_DATA_OPTION_FILE}"; 393 then 394 TEST_DATA_OPTION_FILE=$(get_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}"); 395 fi 396 397 local OPTIONS=() 398 local OPTIONS_STRING=`cat "${TEST_DATA_OPTION_FILE}" | head -n 1 | sed 's/[\r\n]*$//'`; 399 400 echo "${OPTIONS_STRING}"; 401} 402 403# Runs the test with optional arguments. 404# 405# Globals: 406# CHECK_WITH_ASAN 407# CHECK_WITH_GDB 408# CHECK_WITH_STDERR 409# CHECK_WITH_VALGRIND 410# PYTHON_VERSION 411# 412# Arguments: 413# a string containing the test description 414# a string containing the path of the test executable 415# an array containing the arguments for the test executable 416# 417# Returns: 418# an integer containg the exit status of the test executable 419# 420run_test_with_arguments() 421{ 422 local TEST_DESCRIPTION=$1; 423 local TEST_EXECUTABLE=$2; 424 shift 2; 425 local ARGUMENTS=("$@"); 426 427 if ! test -f "${TEST_EXECUTABLE}"; 428 then 429 echo "Missing test executable: ${TEST_EXECUTABLE}"; 430 echo ""; 431 432 return ${EXIT_FAILURE}; 433 fi 434 local PLATFORM=`uname -s | sed 's/-.*$//'`; 435 436 # Note that the behavior of `file -bi` is not helpful on Mac OS X. 437 local EXECUTABLE_TYPE=`file -b ${TEST_EXECUTABLE}`; 438 439 # Check if the test executable is a Python script. 440 # Linux: Python script, ASCII text executable 441 # Mac OS X: a python script text executable 442 echo "${EXECUTABLE_TYPE}" | grep -i "python script" > /dev/null 2>&1; 443 local IS_PYTHON_SCRIPT=$?; 444 445 if test ${IS_PYTHON_SCRIPT} -eq 0; 446 then 447 local PYTHON=`which python${PYTHON_VERSION} 2> /dev/null`; 448 449 if ! test -x ${PYTHON}; 450 then 451 echo "Missing executable: ${PYTHON}"; 452 453 exit ${EXIT_FAILURE}; 454 fi 455 fi 456 local RESULT=0; 457 458 if test -n "${CHECK_WITH_ASAN}"; 459 then 460 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 461 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 462 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 463 464 local LSAN_SUPPRESSIONS="lsan.suppressions"; 465 466 if ! test -f ${LSAN_SUPPRESSIONS}; 467 then 468 LSAN_SUPPRESSIONS="../lsan.suppressions"; 469 fi 470 if test "${PLATFORM}" = "Darwin"; 471 then 472 if test ${IS_PYTHON_SCRIPT} -eq 0; 473 then 474 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 475 RESULT=$?; 476 else 477 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" DYLD_LIBRARY_PATH="${LIBRARY_PATH}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 478 RESULT=$?; 479 fi 480 else 481 local CONFIG_LOG="../config.log"; 482 483 if ! test -f ${CONFIG_LOG}; 484 then 485 CONFIG_LOG="../../config.log"; 486 fi 487 local CC=`cat ${CONFIG_LOG} | grep -e "^CC=" | sed "s/CC='\\(.*\\)'/\1/"`; 488 local LIBASAN=""; 489 490 if test -z ${CC} || test ${CC} != "clang"; 491 then 492 local LDCONFIG=`which ldconfig 2> /dev/null`; 493 494 if test -z ${LDCONFIG} || ! test -x ${LDCONFIG}; 495 then 496 LDCONFIG="/sbin/ldconfig"; 497 fi 498 if test -z ${LDCONFIG} || ! test -x ${LDCONFIG}; 499 then 500 echo "Missing binary: ldconfig"; 501 echo ""; 502 503 exit ${EXIT_FAILURE}; 504 fi 505 LIBASAN=`${LDCONFIG} -p | grep libasan | sed 's/^.* => //' | sort | tail -n 1`; 506 507 if ! test -f ${LIBASAN}; 508 then 509 echo "Missing library: ${BINARY}"; 510 echo ""; 511 512 exit ${EXIT_FAILURE}; 513 fi 514 fi 515 if test ${IS_PYTHON_SCRIPT} -eq 0; 516 then 517 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" LD_PRELOAD="${LIBASAN}" LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 518 RESULT=$?; 519 else 520 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" LD_PRELOAD="${LIBASAN}" LD_LIBRARY_PATH="${LIBRARY_PATH}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 521 RESULT=$?; 522 fi 523 fi 524 525 elif test -n "${CHECK_WITH_GDB}"; 526 then 527 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 528 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 529 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 530 531 if test "${PLATFORM}" = "Darwin"; 532 then 533 if test ${IS_PYTHON_SCRIPT} -eq 0; 534 then 535 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 536 RESULT=$?; 537 else 538 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 539 RESULT=$?; 540 fi 541 542 elif test "${PLATFORM}" = "CYGWIN_NT"; 543 then 544 if test ${IS_PYTHON_SCRIPT} -eq 0; 545 then 546 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 547 RESULT=$?; 548 else 549 PATH="${LIBRARY_PATH}:${PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 550 RESULT=$?; 551 fi 552 553 else 554 if test ${IS_PYTHON_SCRIPT} -eq 0; 555 then 556 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 557 RESULT=$?; 558 else 559 LD_LIBRARY_PATH="${LIBRARY_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 560 RESULT=$?; 561 fi 562 fi 563 564 elif test -n "${CHECK_WITH_VALGRIND}"; 565 then 566 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 567 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 568 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 569 570 local VALGRIND_LOG="valgrind.log-$$"; 571 local VALGRIND_OPTIONS=("--tool=memcheck" "--leak-check=full" "--show-leak-kinds=definite,indirect,possible" "--track-origins=yes" "--log-file=${VALGRIND_LOG}"); 572 573 if test "${PLATFORM}" = "Darwin"; 574 then 575 if test ${IS_PYTHON_SCRIPT} -eq 0; 576 then 577 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 578 RESULT=$?; 579 else 580 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 581 RESULT=$?; 582 fi 583 584 elif test "${PLATFORM}" = "CYGWIN_NT"; 585 then 586 if test ${IS_PYTHON_SCRIPT} -eq 0; 587 then 588 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 589 RESULT=$?; 590 else 591 PATH="${LIBRARY_PATH}:${PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 592 RESULT=$?; 593 fi 594 595 else 596 if test ${IS_PYTHON_SCRIPT} -eq 0; 597 then 598 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 599 RESULT=$?; 600 else 601 LD_LIBRARY_PATH="${LIBRARY_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 602 RESULT=$?; 603 fi 604 fi 605 if test ${RESULT} -eq ${EXIT_SUCCESS}; 606 then 607 grep "All heap blocks were freed -- no leaks are possible" ${VALGRIND_LOG} > /dev/null 2>&1; 608 609 if test $? -ne ${EXIT_SUCCESS}; 610 then 611 # Ignore "still reachable" 612 # Also see: http://valgrind.org/docs/manual/faq.html#faq.deflost 613 614 grep "definitely lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 615 RESULT_DIRECTLY_LOST=$?; 616 617 grep "indirectly lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 618 RESULT_INDIRECTLY_LOST=$?; 619 620 grep "possibly lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 621 RESULT_POSSIBLY_LOST=$?; 622 623 grep "suppressed: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 624 RESULT_SUPPRESSED=$?; 625 626 if test ${RESULT_DIRECTLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_INDIRECTLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_POSSIBLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_SUPPRESSED} -ne ${EXIT_SUCCESS}; 627 then 628 echo "Memory leakage detected."; 629 cat ${VALGRIND_LOG}; 630 631 RESULT=${EXIT_FAILURE}; 632 fi 633 fi 634 # Detect valgrind warnings. 635 local NUMBER_OF_LINES=`wc -l ${VALGRIND_LOG} | awk '{ print $1 }'`; 636 637 if test ${NUMBER_OF_LINES} -ne 15 && test ${NUMBER_OF_LINES} -ne 22; 638 then 639 echo "Unsupported number of lines: ${NUMBER_OF_LINES}"; 640 cat ${VALGRIND_LOG}; 641 642 RESULT=${EXIT_FAILURE}; 643 fi 644 fi 645 rm -f ${VALGRIND_LOG}; 646 647 elif test ${IS_PYTHON_SCRIPT} -eq 0; 648 then 649 if ! test -f "${TEST_EXECUTABLE}"; 650 then 651 echo "Invalid test Python script: ${TEST_EXECUTABLE}"; 652 echo ""; 653 654 return ${EXIT_FAILURE}; 655 fi 656 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 657 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 658 659 if test "${PLATFORM}" = "Darwin"; 660 then 661 if test -n "${CHECK_WITH_STDERR}"; 662 then 663 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 664 RESULT=$?; 665 else 666 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} 2> /dev/null; 667 RESULT=$?; 668 fi 669 670 elif test "${PLATFORM}" = "CYGWIN_NT"; 671 then 672 if test -n "${CHECK_WITH_STDERR}"; 673 then 674 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 675 RESULT=$?; 676 else 677 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} 2> /dev/null; 678 RESULT=$?; 679 fi 680 681 else 682 if test -n "${CHECK_WITH_STDERR}"; 683 then 684 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]}; 685 RESULT=$?; 686 else 687 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} 2> /dev/null; 688 RESULT=$?; 689 fi 690 fi 691 else 692 if ! test -x "${TEST_EXECUTABLE}"; 693 then 694 echo "Invalid test executable: ${TEST_EXECUTABLE}"; 695 echo ""; 696 697 return ${EXIT_FAILURE}; 698 fi 699 700 if test -n "${CHECK_WITH_STDERR}"; 701 then 702 ${TEST_EXECUTABLE} ${ARGUMENTS[@]}; 703 RESULT=$?; 704 else 705 ${TEST_EXECUTABLE} ${ARGUMENTS[@]} 2> /dev/null; 706 RESULT=$?; 707 fi 708 fi 709 if test -n "${TEST_DESCRIPTION}"; 710 then 711 echo -n "${TEST_DESCRIPTION}"; 712 713 if test ${RESULT} -ne ${EXIT_SUCCESS}; 714 then 715 echo " (FAIL)"; 716 else 717 echo " (PASS)"; 718 fi 719 fi 720 return ${RESULT}; 721} 722 723# Runs the test with an input file and optional arguments. 724# 725# Globals: 726# CHECK_WITH_ASAN 727# CHECK_WITH_GDB 728# CHECK_WITH_STDERR 729# CHECK_WITH_VALGRIND 730# PYTHON_VERSION 731# 732# Arguments: 733# a string containing the path of the test executable 734# a string containing the path of the test input file 735# an array containing the arguments for the test executable 736# 737# Returns: 738# an integer containg the exit status of the test executable 739# 740run_test_with_input_and_arguments() 741{ 742 local TEST_EXECUTABLE=$1; 743 local INPUT_FILE=$2; 744 shift 2; 745 local ARGUMENTS=("$@"); 746 747 if ! test -f "${TEST_EXECUTABLE}"; 748 then 749 echo "Missing test executable: ${TEST_EXECUTABLE}"; 750 echo ""; 751 752 return ${EXIT_FAILURE}; 753 fi 754 local PLATFORM=`uname -s | sed 's/-.*$//'`; 755 756 # Note that the behavior of `file -bi` is not helpful on Mac OS X. 757 local EXECUTABLE_TYPE=`file -b ${TEST_EXECUTABLE}`; 758 759 # Check if the test executable is a Python script. 760 # Linux: Python script, ASCII text executable 761 # Mac OS X: a python script text executable 762 echo "${EXECUTABLE_TYPE}" | grep -i "python script" > /dev/null 2>&1; 763 local IS_PYTHON_SCRIPT=$?; 764 765 if test ${IS_PYTHON_SCRIPT} -eq 0; 766 then 767 local PYTHON=`which python${PYTHON_VERSION} 2> /dev/null`; 768 769 if ! test -x ${PYTHON}; 770 then 771 echo "Missing executable: ${PYTHON}"; 772 773 exit ${EXIT_FAILURE}; 774 fi 775 fi 776 local RESULT=0; 777 778 if test -n "${CHECK_WITH_ASAN}"; 779 then 780 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 781 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 782 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 783 784 local LSAN_SUPPRESSIONS="lsan.suppressions"; 785 786 if ! test -f ${LSAN_SUPPRESSIONS}; 787 then 788 LSAN_SUPPRESSIONS="../lsan.suppressions"; 789 fi 790 if test "${PLATFORM}" = "Darwin"; 791 then 792 # TODO DYLD_INSERT_LIBRARIES=/Library/Developer/CommandLineTools/usr/lib/clang/8.1.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib 793 if test ${IS_PYTHON_SCRIPT} -eq 0; 794 then 795 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 796 RESULT=$?; 797 else 798 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" DYLD_LIBRARY_PATH="${LIBRARY_PATH}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 799 RESULT=$?; 800 fi 801 else 802 local CONFIG_LOG="../config.log"; 803 804 if ! test -f ${CONFIG_LOG}; 805 then 806 CONFIG_LOG="../../config.log"; 807 fi 808 local CC=`cat ${CONFIG_LOG} | grep -e "^CC=" | sed "s/CC='\\(.*\\)'/\1/"`; 809 local LIBASAN=""; 810 811 if test -z ${CC} || test ${CC} != "clang"; 812 then 813 local LDCONFIG=`which ldconfig 2> /dev/null`; 814 815 if test -z ${LDCONFIG} || ! test -x ${LDCONFIG}; 816 then 817 LDCONFIG="/sbin/ldconfig"; 818 fi 819 if test -z ${LDCONFIG} || ! test -x ${LDCONFIG}; 820 then 821 echo "Missing binary: ldconfig"; 822 echo ""; 823 824 exit ${EXIT_FAILURE}; 825 fi 826 LIBASAN=`${LDCONFIG} -p | grep libasan | sed 's/^.* => //' | sort | tail -n 1`; 827 828 if ! test -f ${LIBASAN}; 829 then 830 echo "Missing library: ${BINARY}"; 831 echo ""; 832 833 exit ${EXIT_FAILURE}; 834 fi 835 fi 836 if test ${IS_PYTHON_SCRIPT} -eq 0; 837 then 838 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" LD_PRELOAD="${LIBASAN}" LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 839 RESULT=$?; 840 else 841 LSAN_OPTIONS=suppressions="${LSAN_SUPPRESSIONS}" LD_PRELOAD="${LIBASAN}" LD_LIBRARY_PATH="${LIBRARY_PATH}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 842 RESULT=$?; 843 fi 844 fi 845 846 elif test -n "${CHECK_WITH_GDB}"; 847 then 848 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 849 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 850 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 851 852 if test "${PLATFORM}" = "Darwin"; 853 then 854 if test ${IS_PYTHON_SCRIPT} -eq 0; 855 then 856 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 857 RESULT=$?; 858 else 859 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 860 RESULT=$?; 861 fi 862 863 elif test "${PLATFORM}" = "CYGWIN_NT"; 864 then 865 if test ${IS_PYTHON_SCRIPT} -eq 0; 866 then 867 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 868 RESULT=$?; 869 else 870 PATH="${LIBRARY_PATH}:${PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 871 RESULT=$?; 872 fi 873 874 else 875 if test ${IS_PYTHON_SCRIPT} -eq 0; 876 then 877 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 878 RESULT=$?; 879 else 880 LD_LIBRARY_PATH="${LIBRARY_PATH}" gdb -ex "set non-stop on" -ex "run" -ex "quit" --args "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 881 RESULT=$?; 882 fi 883 fi 884 885 elif test -n "${CHECK_WITH_VALGRIND}"; 886 then 887 local TEST_EXECUTABLE=$( find_binary_executable ${TEST_EXECUTABLE} ); 888 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 889 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 890 891 local VALGRIND_LOG="valgrind.log-$$"; 892 local VALGRIND_OPTIONS=("--tool=memcheck" "--leak-check=full" "--show-leak-kinds=definite,indirect,possible" "--track-origins=yes" "--log-file=${VALGRIND_LOG}"); 893 894 if test "${PLATFORM}" = "Darwin"; 895 then 896 if test ${IS_PYTHON_SCRIPT} -eq 0; 897 then 898 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 899 RESULT=$?; 900 else 901 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 902 RESULT=$?; 903 fi 904 905 elif test "${PLATFORM}" = "CYGWIN_NT"; 906 then 907 if test ${IS_PYTHON_SCRIPT} -eq 0; 908 then 909 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 910 RESULT=$?; 911 else 912 PATH="${LIBRARY_PATH}:${PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 913 RESULT=$?; 914 fi 915 916 else 917 if test ${IS_PYTHON_SCRIPT} -eq 0; 918 then 919 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 920 RESULT=$?; 921 else 922 LD_LIBRARY_PATH="${LIBRARY_PATH}" valgrind ${VALGRIND_OPTIONS[@]} "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 923 RESULT=$?; 924 fi 925 fi 926 if test ${RESULT} -eq ${EXIT_SUCCESS}; 927 then 928 grep "All heap blocks were freed -- no leaks are possible" ${VALGRIND_LOG} > /dev/null 2>&1; 929 930 if test $? -ne ${EXIT_SUCCESS}; 931 then 932 # Ignore "still reachable" 933 # Also see: http://valgrind.org/docs/manual/faq.html#faq.deflost 934 935 grep "definitely lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 936 RESULT_DIRECTLY_LOST=$?; 937 938 grep "indirectly lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 939 RESULT_INDIRECTLY_LOST=$?; 940 941 grep "possibly lost: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 942 RESULT_POSSIBLY_LOST=$?; 943 944 grep "suppressed: 0 bytes in 0 blocks" ${VALGRIND_LOG} > /dev/null 2>&1; 945 RESULT_SUPPRESSED=$?; 946 947 if test ${RESULT_DIRECTLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_INDIRECTLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_POSSIBLY_LOST} -ne ${EXIT_SUCCESS} || test ${RESULT_SUPPRESSED} -ne ${EXIT_SUCCESS}; 948 then 949 echo "Memory leakage detected."; 950 cat ${VALGRIND_LOG}; 951 952 RESULT=${EXIT_FAILURE}; 953 fi 954 fi 955 # Detect valgrind warnings. 956 local NUMBER_OF_LINES=`wc -l ${VALGRIND_LOG} | awk '{ print $1 }'`; 957 958 if test ${NUMBER_OF_LINES} -ne 15 && test ${NUMBER_OF_LINES} -ne 22; 959 then 960 echo "Unsupported number of lines: ${NUMBER_OF_LINES}"; 961 cat ${VALGRIND_LOG}; 962 963 RESULT=${EXIT_FAILURE}; 964 fi 965 fi 966 rm -f ${VALGRIND_LOG}; 967 968 elif test ${IS_PYTHON_SCRIPT} -eq 0; 969 then 970 if ! test -f "${TEST_EXECUTABLE}"; 971 then 972 echo "Invalid test Python script: ${TEST_EXECUTABLE}"; 973 echo ""; 974 975 return ${EXIT_FAILURE}; 976 fi 977 local LIBRARY_PATH=$( find_binary_library_path ${TEST_EXECUTABLE} ); 978 local PYTHON_MODULE_PATH=$( find_binary_python_module_path ${TEST_EXECUTABLE} ); 979 980 if test "${PLATFORM}" = "Darwin"; 981 then 982 if test -n "${CHECK_WITH_STDERR}"; 983 then 984 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 985 RESULT=$?; 986 else 987 DYLD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}" 2> /dev/null; 988 RESULT=$?; 989 fi 990 991 elif test "${PLATFORM}" = "CYGWIN_NT"; 992 then 993 if test -n "${CHECK_WITH_STDERR}"; 994 then 995 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 996 RESULT=$?; 997 else 998 PATH="${LIBRARY_PATH}:${PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}" 2> /dev/null; 999 RESULT=$?; 1000 fi 1001 1002 else 1003 if test -n "${CHECK_WITH_STDERR}"; 1004 then 1005 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}"; 1006 RESULT=$?; 1007 else 1008 LD_LIBRARY_PATH="${LIBRARY_PATH}" PYTHONPATH="${PYTHON_MODULE_PATH}" "${PYTHON}" "${TEST_EXECUTABLE}" ${ARGUMENTS[@]} "${INPUT_FILE}" 2> /dev/null; 1009 RESULT=$?; 1010 fi 1011 fi 1012 else 1013 if ! test -x "${TEST_EXECUTABLE}"; 1014 then 1015 echo "Invalid test executable: ${TEST_EXECUTABLE}"; 1016 echo ""; 1017 1018 return ${EXIT_FAILURE}; 1019 fi 1020 1021 if test -n "${CHECK_WITH_STDERR}"; 1022 then 1023 ${TEST_EXECUTABLE} ${ARGUMENTS[@]} "${INPUT_FILE}"; 1024 RESULT=$?; 1025 else 1026 ${TEST_EXECUTABLE} ${ARGUMENTS[@]} "${INPUT_FILE}" 2> /dev/null; 1027 RESULT=$?; 1028 fi 1029 fi 1030 return ${RESULT}; 1031} 1032 1033# Runs the test on the input file. 1034# 1035# Note that this function is not intended to be directly invoked 1036# from outside the test runner script. 1037# 1038# Arguments: 1039# a string containing the path of the test set directory 1040# a string containing the description of the test 1041# a string containing the test mode 1042# a string containing the name of the test data option set 1043# a string containing the path of the test executable 1044# a string containing the path of the test input file 1045# an array containing the arguments for the test executable 1046# 1047# Returns: 1048# an integer containg the exit status of the test executable 1049# 1050run_test_on_input_file() 1051{ 1052 local TEST_SET_DIRECTORY=$1; 1053 local TEST_DESCRIPTION=$2; 1054 local TEST_MODE=$3; 1055 local OPTION_SET=$4; 1056 local TEST_EXECUTABLE=$5; 1057 local INPUT_FILE=$6; 1058 shift 6; 1059 local ARGUMENTS=("$@"); 1060 1061 local INPUT_NAME=`basename "${INPUT_FILE}"`; 1062 local OPTIONS=(); 1063 local TEST_OUTPUT="${INPUT_NAME}"; 1064 1065 if test -n "${OPTION_SET}"; 1066 then 1067 OPTIONS_STRING=$(read_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}"); 1068 IFS=" " read -a OPTIONS <<< "${OPTIONS_STRING}"; 1069 1070 TEST_OUTPUT="${INPUT_NAME}-${OPTION_SET}"; 1071 fi 1072 1073 local TMPDIR="tmp$$"; 1074 local RESULT=0; 1075 1076 rm -rf ${TMPDIR}; 1077 mkdir ${TMPDIR}; 1078 1079 if test "${TEST_MODE}" = "with_callback"; 1080 then 1081 test_callback "${TMPDIR}" "${TEST_SET_DIRECTORY}" "${TEST_OUTPUT}" "${TEST_EXECUTABLE}" "${TEST_INPUT}" ${ARGUMENTS[@]} "${OPTIONS[@]}"; 1082 RESULT=$?; 1083 1084 elif test "${TEST_MODE}" = "with_stdout_reference"; 1085 then 1086 TEST_EXECUTABLE=$( readlink_f "${TEST_EXECUTABLE}" ); 1087 1088 if ! test -x ${TEST_EXECUTABLE}; 1089 then 1090 echo "Invalid test executable: ${TEST_EXECUTABLE}"; 1091 echo ""; 1092 1093 return ${EXIT_FAILURE}; 1094 fi 1095 local INPUT_FILE_FULL_PATH=$( readlink_f "${INPUT_FILE}" ); 1096 local TEST_LOG="${TEST_OUTPUT}.log"; 1097 1098 (cd ${TMPDIR} && run_test_with_input_and_arguments "${TEST_EXECUTABLE}" "${INPUT_FILE_FULL_PATH}" ${ARGUMENTS[@]} "${OPTIONS[@]}" > "${TEST_LOG}"); 1099 RESULT=$?; 1100 1101 # Compare output if test ran successfully. 1102 if test ${RESULT} -eq ${EXIT_SUCCESS}; 1103 then 1104 local TEST_RESULTS="${TMPDIR}/${TEST_LOG}"; 1105 local STORED_TEST_RESULTS="${TEST_SET_DIRECTORY}/${TEST_LOG}.gz"; 1106 1107 # Strip header with version. 1108 sed -i'~' '1,2d' "${TEST_RESULTS}"; 1109 1110 if test -f "${STORED_TEST_RESULTS}"; 1111 then 1112 # Using zcat here since zdiff has issues on Mac OS X. 1113 # Note that zcat on Mac OS X requires the input from stdin. 1114 zcat < "${STORED_TEST_RESULTS}" | diff "${TEST_RESULTS}" -; 1115 RESULT=$?; 1116 else 1117 gzip "${TEST_RESULTS}"; 1118 1119 mv "${TEST_RESULTS}.gz" ${TEST_SET_DIRECTORY}; 1120 fi 1121 fi 1122 1123 else 1124 run_test_with_input_and_arguments "${TEST_EXECUTABLE}" "${INPUT_FILE}" ${ARGUMENTS[@]} "${OPTIONS[@]}"; 1125 RESULT=$?; 1126 fi 1127 1128 rm -rf ${TMPDIR}; 1129 1130 if test -n "${TEST_DESCRIPTION}"; 1131 then 1132 ARGUMENTS=`echo "${ARGUMENTS[*]}" | tr '\n' ' ' | sed 's/[ ]\$//'`; 1133 OPTIONS=`echo "${OPTIONS[*]}" | tr '\n' ' ' | sed 's/[ ]\$//'`; 1134 1135 if test -z "${ARGUMENTS}" && test -z "${OPTIONS}"; 1136 then 1137 echo -n "${TEST_DESCRIPTION} with input: ${INPUT_FILE}"; 1138 1139 elif test -z "${ARGUMENTS}"; 1140 then 1141 echo -n "${TEST_DESCRIPTION} with options: '${OPTIONS}' and input: ${INPUT_FILE}"; 1142 1143 elif test -z "${OPTIONS}"; 1144 then 1145 echo -n "${TEST_DESCRIPTION} with options: '${ARGUMENTS}' and input: ${INPUT_FILE}"; 1146 1147 else 1148 echo -n "${TEST_DESCRIPTION} with options: '${ARGUMENTS} ${OPTIONS}' and input: ${INPUT_FILE}"; 1149 fi 1150 1151 if test ${RESULT} -ne ${EXIT_SUCCESS}; 1152 then 1153 echo " (FAIL)"; 1154 else 1155 echo " (PASS)"; 1156 fi 1157 fi 1158 return ${RESULT}; 1159} 1160 1161# Runs the test with options on the input file. 1162# 1163# Note that this function is not intended to be directly invoked 1164# from outside the test runner script. 1165# 1166# Arguments: 1167# a string containing the path of the test set directory 1168# a string containing the description of the test 1169# a string containing the test mode 1170# a string containing the name of the test data option sets 1171# a string containing the path of the test executable 1172# a string containing the path of the test input file 1173# an array containing the arguments for the test executable 1174# 1175# Returns: 1176# an integer containg the exit status of the test executable 1177# 1178run_test_on_input_file_with_options() 1179{ 1180 local TEST_SET_DIRECTORY=$1; 1181 local TEST_DESCRIPTION=$2; 1182 local TEST_MODE=$3; 1183 local OPTION_SETS=$4; 1184 local TEST_EXECUTABLE=$5; 1185 local INPUT_FILE=$6; 1186 shift 6; 1187 local ARGUMENTS=("$@"); 1188 1189 local RESULT=${EXIT_SUCCESS}; 1190 local TESTED_WITH_OPTIONS=0; 1191 1192 for OPTION_SET in `echo ${OPTION_SETS} | tr ' ' '\n'`; 1193 do 1194 local TEST_DATA_OPTION_FILE=$(get_test_data_option_file "${TEST_SET_DIRECTORY}" "${INPUT_FILE}" "${OPTION_SET}"); 1195 1196 if ! test -f ${TEST_DATA_OPTION_FILE}; 1197 then 1198 continue 1199 fi 1200 1201 run_test_on_input_file "${TEST_SET_DIRECTORY}" "${TEST_DESCRIPTION}" "${TEST_MODE}" "${OPTION_SET}" "${TEST_EXECUTABLE}" "${INPUT_FILE}" ${ARGUMENTS[@]}; 1202 RESULT=$?; 1203 1204 if test ${RESULT} -ne ${EXIT_SUCCESS}; 1205 then 1206 break; 1207 fi 1208 TESTED_WITH_OPTIONS=1; 1209 done 1210 1211 if test ${RESULT} -eq ${EXIT_SUCCESS} && test ${TESTED_WITH_OPTIONS} -eq 0; 1212 then 1213 run_test_on_input_file "${TEST_SET_DIRECTORY}" "${TEST_DESCRIPTION}" "${TEST_MODE}" "" "${TEST_EXECUTABLE}" "${INPUT_FILE}" ${ARGUMENTS[@]}; 1214 RESULT=$?; 1215 fi 1216 return ${RESULT}; 1217} 1218 1219# Runs the test with options on the file entries in the test set directory. 1220# 1221# Note that this function is not intended to be directly invoked 1222# from outside the test runner script. 1223# 1224# Arguments: 1225# a string containing the path of the test set directory 1226# a string containing the description of the test 1227# a string containing the test mode 1228# a string containing the name of the test data option sets 1229# a string containing the path of the test executable 1230# an array containing the arguments for the test executable 1231# 1232# Returns: 1233# an integer containg the exit status of the test executable 1234# 1235run_test_on_test_set_with_options() 1236{ 1237 local TEST_SET_DIRECTORY=$1; 1238 local TEST_DESCRIPTION=$2; 1239 local TEST_MODE=$3; 1240 local OPTION_SETS=$4; 1241 local TEST_EXECUTABLE=$5; 1242 shift 5; 1243 local ARGUMENTS=("$@"); 1244 1245 local RESULT=${EXIT_SUCCESS}; 1246 1247 # IFS="\n"; is not supported by all platforms. 1248 IFS=" 1249"; 1250 1251 if test -f "${TEST_SET_DIRECTORY}/files"; 1252 then 1253 for INPUT_FILE in `cat ${TEST_SET_DIRECTORY}/files | sed "s?^?${TEST_SET_INPUT_DIRECTORY}/?"`; 1254 do 1255 run_test_on_input_file_with_options "${TEST_SET_DIRECTORY}" "${TEST_DESCRIPTION}" "${TEST_MODE}" "${OPTION_SETS}" "${TEST_EXECUTABLE}" "${INPUT_FILE}" ${ARGUMENTS[@]}; 1256 RESULT=$?; 1257 1258 if test ${RESULT} -ne ${EXIT_SUCCESS}; 1259 then 1260 break; 1261 fi 1262 done 1263 else 1264 for INPUT_FILE in `ls -1d ${TEST_SET_INPUT_DIRECTORY}/${INPUT_GLOB}`; 1265 do 1266 run_test_on_input_file_with_options "${TEST_SET_DIRECTORY}" "${TEST_DESCRIPTION}" "${TEST_MODE}" "${OPTION_SETS}" "${TEST_EXECUTABLE}" "${INPUT_FILE}" ${ARGUMENTS[@]}; 1267 RESULT=$?; 1268 1269 if test ${RESULT} -ne ${EXIT_SUCCESS}; 1270 then 1271 break; 1272 fi 1273 done 1274 fi 1275 IFS=${OLDIFS}; 1276 1277 return ${RESULT}; 1278} 1279 1280