1# ------------------------------------------------------------------------------ 2# Programmer(s): Steven Smith and David J. Gardner @ LLNL 3# ------------------------------------------------------------------------------ 4# SUNDIALS Copyright Start 5# Copyright (c) 2002-2021, Lawrence Livermore National Security 6# and Southern Methodist University. 7# All rights reserved. 8# 9# See the top-level LICENSE and NOTICE files for details. 10# 11# SPDX-License-Identifier: BSD-3-Clause 12# SUNDIALS Copyright End 13# ------------------------------------------------------------------------------ 14# 15# SUNDIALS_ADD_TEST(<test name> <executable>) 16# 17# CMake macro to add a SUNDIALS regression test. Keyword input arguments can be 18# added after <executable> to set regression test options (see oneValueArgs and 19# multiValueArgs below). 20# 21# When SUNDIALS_TEST_DEVTESTS is OFF (default) the executable is run and success 22# or failure is determined by the executable return value (zero or non-zero 23# respectively). 24# 25# When SUNDIALS_TEST_DEVTESTS is ON the executable is run and its output is 26# compared with the corresponding .out file. If the output differs significantly 27# then the test fails. The default level of significance is 4 decimal places for 28# floating point values and 10% for integer values. 29# 30# The level of precision can be adjusted for an individual test with the 31# FLOAT_PRECISION AND INTEGER_PERCENTAGE keyword inputs to the macro or globally 32# for all tests with the cache variables SUNDIALS_TEST_FLOAT_PRECISION and 33# SUNDIALS_TEST_INTEGER_PRECISION. 34# 35# -D SUNDIALS_TEST_FLOAT_PRECISION=<number of digits> 36# -D SUNDIALS_TEST_INTEGER_PRECISION=<% difference> 37# 38# By default testing output is written to builddir/Testing/output and the .out 39# answer file directory is set using the ANSWER_DIR keyword input to 40# sourcedir/examples/package/testdir. These can be changed by setting the cache 41# variables SUNDIALS_TEST_OUTPUT_DIR and SUNDIALS_TEST_ANSWER_DIR. 42# 43# -D SUNDIALS_TEST_OUTPUT_DIR=<path to output directory> 44# -D SUNDIALS_TEST_ANSWER_DIR=<path to answer directory> 45# ------------------------------------------------------------------------------ 46 47macro(SUNDIALS_ADD_TEST NAME EXECUTABLE) 48 49 # macro options 50 # NODIFF = do not diff the test output against an answer file 51 set(options "NODIFF") 52 53 # macro keyword inputs followed by a single value 54 # MPI_NPROCS = number of mpi tasks to use in parallel tests 55 # FLOAT_PRECISION = precision for floating point failure comparision (num digits), 56 # to use the default, either don't provide the keyword, or 57 # provide the value "default" 58 # INTEGER_PRECENTAGE = integer percentage difference for failure comparison 59 # ANSWER_DIR = path to the directory containing the test answer file 60 # ANSWER_FILE = name of test answer file 61 # EXAMPLE_TYPE = release or develop examples 62 set(oneValueArgs "MPI_NPROCS" "FLOAT_PRECISION" "INTEGER_PERCENTAGE" 63 "ANSWER_DIR" "ANSWER_FILE" "EXAMPLE_TYPE") 64 65 # macro keyword inputs followed by multiple values 66 # TEST_ARGS = command line arguments to pass to the test executable 67 set(multiValueArgs "TEST_ARGS") 68 69 # parse inputs and create variables SUNDIALS_ADD_TEST_<keyword> 70 cmake_parse_arguments(SUNDIALS_ADD_TEST 71 "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) 72 73 # SGS add check to make sure parallel is integer 74 75 # check that the test is not excluded 76 if(NOT ("${SUNDIALS_ADD_TEST_EXAMPLE_TYPE}" STREQUAL "exclude")) 77 78 if(SUNDIALS_TEST_DEVTESTS) 79 80 # run all tests (standard and develop) with the test runner 81 82 # command line arguments for the test runner script 83 set(TEST_ARGS 84 "--verbose" 85 "--testname=${NAME}" 86 "--executablename=$<TARGET_FILE:${EXECUTABLE}>" 87 ) 88 89 # check for a non-default output directory 90 if(SUNDIALS_TEST_OUTPUT_DIR) 91 list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}") 92 else() 93 list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}") 94 endif() 95 96 # set a non-default answer directory (default is test/answers) 97 if(SUNDIALS_TEST_ANSWER_DIR) 98 list(APPEND TEST_ARGS "--answerdir=${SUNDIALS_TEST_ANSWER_DIR}") 99 elseif(SUNDIALS_ADD_TEST_ANSWER_DIR) 100 list(APPEND TEST_ARGS "--answerdir=${SUNDIALS_ADD_TEST_ANSWER_DIR}") 101 endif() 102 103 # set the test answer file name (default is test_name_test_agrs) 104 if(SUNDIALS_ADD_TEST_ANSWER_FILE) 105 list(APPEND TEST_ARGS "--answerfile=${SUNDIALS_ADD_TEST_ANSWER_FILE}") 106 endif() 107 108 # check if a diff is needed and if non-default precisions were provided 109 if(SUNDIALS_ADD_TEST_NODIFF) 110 # do not diff the output and answer files 111 list(APPEND TEST_ARGS "--nodiff") 112 else() 113 # set a non-default floating point precision (number of digits, default 4) 114 if(SUNDIALS_TEST_FLOAT_PRECISION) 115 list(APPEND TEST_ARGS "--floatprecision=${SUNDIALS_TEST_FLOAT_PRECISION}") 116 elseif(SUNDIALS_ADD_TEST_FLOAT_PRECISION AND 117 (NOT SUNDIALS_ADD_TEST_FLOAT_PRECISION MATCHES "DEFAULT|default")) 118 list(APPEND TEST_ARGS "--floatprecision=${SUNDIALS_ADD_TEST_FLOAT_PRECISION}") 119 endif() 120 # set a non-default integer precision (percent difference, default 10%) 121 if(SUNDIALS_TEST_INTEGER_PRECISION) 122 list(APPEND TEST_ARGS "--integerpercentage=${SUNDIALS_TEST_INTEGER_PRECISION}") 123 elseif(SUNDIALS_ADD_TEST_INTEGER_PERCENTAGE) 124 list(APPEND TEST_ARGS "--integerpercentage=${SUNDIALS_ADD_TEST_INTEGER_PERCENTAGE}") 125 endif() 126 endif() 127 128 # check if this test is run with MPI and set the MPI run command 129 if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE)) 130 if(MPIEXEC_EXECUTABLE MATCHES "srun") 131 set(RUN_COMMAND "srun -N1 -n${SUNDIALS_ADD_TEST_MPI_NPROCS} -ppdebug") 132 else() 133 set(RUN_COMMAND "${MPIEXEC_EXECUTABLE} -n ${SUNDIALS_ADD_TEST_MPI_NPROCS}") 134 endif() 135 list(APPEND TEST_ARGS "--runcommand=\"${RUN_COMMAND}\"") 136 endif() 137 138 # set the test input args 139 if(SUNDIALS_ADD_TEST_TEST_ARGS) 140 string(REPLACE ";" " " USER_ARGS "${SUNDIALS_ADD_TEST_TEST_ARGS}") 141 list(APPEND TEST_ARGS "--runargs=\"${USER_ARGS}\"") 142 endif() 143 144 # create test case with the corresponding test runner command and arguments 145 # all tests are added during development and only unlabeled tests when released 146 add_test(NAME ${NAME} COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_ARGS}) 147 148 elseif(NOT SUNDIALS_ADD_TEST_EXAMPLE_TYPE) 149 150 # if a test type was not set then it is a standard test that returns pass/fail 151 152 # convert string to list 153 if(SUNDIALS_ADD_TEST_TEST_ARGS) 154 string(REPLACE " " ";" TEST_ARGS "${SUNDIALS_ADD_TEST_TEST_ARGS}") 155 endif() 156 157 # check if this test is run with MPI and add the test run command 158 if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE)) 159 add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} -n ${SUNDIALS_ADD_TEST_MPI_NPROCS} $<TARGET_FILE:${EXECUTABLE}> ${TEST_ARGS}) 160 else() 161 add_test(NAME ${NAME} COMMAND $<TARGET_FILE:${EXECUTABLE}> ${TEST_ARGS}) 162 endif() 163 164 endif() 165 166 endif() 167 168endmacro() 169