1# ------------------------------------------------------------------------------
2# Programmer(s): Steven Smith and David J. Gardner @ LLNL
3# ------------------------------------------------------------------------------
4# SUNDIALS Copyright Start
5# Copyright (c) 2002-2021, Lawrence Livermore National Security
6# and Southern Methodist University.
7# All rights reserved.
8#
9# See the top-level LICENSE and NOTICE files for details.
10#
11# SPDX-License-Identifier: BSD-3-Clause
12# SUNDIALS Copyright End
13# ------------------------------------------------------------------------------
14#
15# SUNDIALS_ADD_TEST(<test name> <executable>)
16#
17# CMake macro to add a SUNDIALS regression test. Keyword input arguments can be
18# added after <executable> to set regression test options (see oneValueArgs and
19# multiValueArgs below).
20#
21# When SUNDIALS_TEST_DEVTESTS is OFF (default) the executable is run and success
22# or failure is determined by the executable return value (zero or non-zero
23# respectively).
24#
25# When SUNDIALS_TEST_DEVTESTS is ON the executable is run and its output is
26# compared with the corresponding .out file. If the output differs significantly
27# then the test fails. The default level of significance is 4 decimal places for
28# floating point values and 10% for integer values.
29#
30# The level of precision can be adjusted for an individual test with the
31# FLOAT_PRECISION AND INTEGER_PERCENTAGE keyword inputs to the macro or globally
32# for all tests with the cache variables SUNDIALS_TEST_FLOAT_PRECISION and
33# SUNDIALS_TEST_INTEGER_PRECISION.
34#
35#  -D SUNDIALS_TEST_FLOAT_PRECISION=<number of digits>
36#  -D SUNDIALS_TEST_INTEGER_PRECISION=<% difference>
37#
38# By default testing output is written to builddir/Testing/output and the .out
39# answer file directory is set using the ANSWER_DIR keyword input to
40# sourcedir/examples/package/testdir. These can be changed by setting the cache
41# variables SUNDIALS_TEST_OUTPUT_DIR and SUNDIALS_TEST_ANSWER_DIR.
42#
43#  -D SUNDIALS_TEST_OUTPUT_DIR=<path to output directory>
44#  -D SUNDIALS_TEST_ANSWER_DIR=<path to answer directory>
45# ------------------------------------------------------------------------------
46
47macro(SUNDIALS_ADD_TEST NAME EXECUTABLE)
48
49  # macro options
50  # NODIFF = do not diff the test output against an answer file
51  set(options "NODIFF")
52
53  # macro keyword inputs followed by a single value
54  # MPI_NPROCS         = number of mpi tasks to use in parallel tests
55  # FLOAT_PRECISION    = precision for floating point failure comparision (num digits),
56  #                      to use the default, either don't provide the keyword, or
57  #                      provide the value "default"
58  # INTEGER_PRECENTAGE = integer percentage difference for failure comparison
59  # ANSWER_DIR         = path to the directory containing the test answer file
60  # ANSWER_FILE        = name of test answer file
61  # EXAMPLE_TYPE       = release or develop examples
62  set(oneValueArgs "MPI_NPROCS" "FLOAT_PRECISION" "INTEGER_PERCENTAGE"
63    "ANSWER_DIR" "ANSWER_FILE" "EXAMPLE_TYPE")
64
65  # macro keyword inputs followed by multiple values
66  # TEST_ARGS = command line arguments to pass to the test executable
67  set(multiValueArgs "TEST_ARGS")
68
69  # parse inputs and create variables SUNDIALS_ADD_TEST_<keyword>
70  cmake_parse_arguments(SUNDIALS_ADD_TEST
71    "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
72
73  # SGS add check to make sure parallel is integer
74
75  # check that the test is not excluded
76  if(NOT ("${SUNDIALS_ADD_TEST_EXAMPLE_TYPE}" STREQUAL "exclude"))
77
78    if(SUNDIALS_TEST_DEVTESTS)
79
80      # run all tests (standard and develop) with the test runner
81
82      # command line arguments for the test runner script
83      set(TEST_ARGS
84        "--verbose"
85        "--testname=${NAME}"
86        "--executablename=$<TARGET_FILE:${EXECUTABLE}>"
87        )
88
89      # check for a non-default output directory
90      if(SUNDIALS_TEST_OUTPUT_DIR)
91        list(APPEND TEST_ARGS "--outputdir=${SUNDIALS_TEST_OUTPUT_DIR}")
92      else()
93        list(APPEND TEST_ARGS "--outputdir=${TEST_OUTPUT_DIR}")
94      endif()
95
96      # set a non-default answer directory (default is test/answers)
97      if(SUNDIALS_TEST_ANSWER_DIR)
98        list(APPEND TEST_ARGS "--answerdir=${SUNDIALS_TEST_ANSWER_DIR}")
99      elseif(SUNDIALS_ADD_TEST_ANSWER_DIR)
100        list(APPEND TEST_ARGS "--answerdir=${SUNDIALS_ADD_TEST_ANSWER_DIR}")
101      endif()
102
103      # set the test answer file name (default is test_name_test_agrs)
104      if(SUNDIALS_ADD_TEST_ANSWER_FILE)
105        list(APPEND TEST_ARGS "--answerfile=${SUNDIALS_ADD_TEST_ANSWER_FILE}")
106      endif()
107
108      # check if a diff is needed and if non-default precisions were provided
109      if(SUNDIALS_ADD_TEST_NODIFF)
110        # do not diff the output and answer files
111        list(APPEND TEST_ARGS "--nodiff")
112      else()
113        # set a non-default floating point precision (number of digits, default 4)
114        if(SUNDIALS_TEST_FLOAT_PRECISION)
115          list(APPEND TEST_ARGS "--floatprecision=${SUNDIALS_TEST_FLOAT_PRECISION}")
116        elseif(SUNDIALS_ADD_TEST_FLOAT_PRECISION AND
117               (NOT SUNDIALS_ADD_TEST_FLOAT_PRECISION MATCHES "DEFAULT|default"))
118          list(APPEND TEST_ARGS "--floatprecision=${SUNDIALS_ADD_TEST_FLOAT_PRECISION}")
119        endif()
120        # set a non-default integer precision (percent difference, default 10%)
121        if(SUNDIALS_TEST_INTEGER_PRECISION)
122          list(APPEND TEST_ARGS "--integerpercentage=${SUNDIALS_TEST_INTEGER_PRECISION}")
123        elseif(SUNDIALS_ADD_TEST_INTEGER_PERCENTAGE)
124          list(APPEND TEST_ARGS "--integerpercentage=${SUNDIALS_ADD_TEST_INTEGER_PERCENTAGE}")
125        endif()
126      endif()
127
128      # check if this test is run with MPI and set the MPI run command
129      if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE))
130        if(MPIEXEC_EXECUTABLE MATCHES "srun")
131          set(RUN_COMMAND "srun -N1 -n${SUNDIALS_ADD_TEST_MPI_NPROCS} -ppdebug")
132        else()
133          set(RUN_COMMAND "${MPIEXEC_EXECUTABLE} -n ${SUNDIALS_ADD_TEST_MPI_NPROCS}")
134        endif()
135        list(APPEND TEST_ARGS "--runcommand=\"${RUN_COMMAND}\"")
136      endif()
137
138      # set the test input args
139      if(SUNDIALS_ADD_TEST_TEST_ARGS)
140        string(REPLACE ";" " " USER_ARGS "${SUNDIALS_ADD_TEST_TEST_ARGS}")
141        list(APPEND TEST_ARGS "--runargs=\"${USER_ARGS}\"")
142      endif()
143
144      # create test case with the corresponding test runner command and arguments
145      # all tests are added during development and only unlabeled tests when released
146      add_test(NAME ${NAME} COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_ARGS})
147
148    elseif(NOT SUNDIALS_ADD_TEST_EXAMPLE_TYPE)
149
150      # if a test type was not set then it is a standard test that returns pass/fail
151
152      # convert string to list
153      if(SUNDIALS_ADD_TEST_TEST_ARGS)
154        string(REPLACE " " ";" TEST_ARGS "${SUNDIALS_ADD_TEST_TEST_ARGS}")
155      endif()
156
157      # check if this test is run with MPI and add the test run command
158      if((SUNDIALS_ADD_TEST_MPI_NPROCS) AND (MPIEXEC_EXECUTABLE))
159        add_test(NAME ${NAME} COMMAND ${MPIEXEC_EXECUTABLE} -n ${SUNDIALS_ADD_TEST_MPI_NPROCS} $<TARGET_FILE:${EXECUTABLE}> ${TEST_ARGS})
160      else()
161        add_test(NAME ${NAME} COMMAND $<TARGET_FILE:${EXECUTABLE}> ${TEST_ARGS})
162      endif()
163
164    endif()
165
166  endif()
167
168endmacro()
169