1#!/bin/sh
2
3_help()
4{
5  cat << EOF
6### run-regtests.sh ###
7  Running run-regtests.sh --help shows this information about how to use it
8
9  Run without parameters to run all tests in the current folder (including subfolders)
10    run-regtests.sh
11
12  Provide paths to run tests from (including subfolders):
13    run-regtests.sh ./tests1 ./tests2
14
15  Parameters:
16    --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
17      run-regtests.sh ./fasttest --j 16
18
19    --v, to run verbose
20      run-regtests.sh --v, disables the default vtest 'quiet' parameter
21
22    --debug to show test logs on standard ouput (implies --v)
23      run-regtests.sh --debug
24
25    --keep-logs to keep all log directories (by default kept if test fails)
26      run-regtests.sh --keep-logs
27
28    --vtestparams <ARGS>, passes custom ARGS to vtest
29      run-regtests.sh --vtestparams "-n 10"
30
31    --clean to cleanup previous reg-tests log directories and exit
32      run-regtests.sh --clean
33
34    --use-htx to use the HTX in tests
35      run-regtests.sh --use-htx, unsets the macro \${no-htx}
36      In .vtc files, in HAProxy configuration, you should use the following line
37      to "templatize" your tests:
38
39          \${no-htx} option http-use-htx
40
41  Including text below into a .vtc file will check for its requirements
42  related to haproxy's target and compilation options
43    # Below targets are not capable of completing this test succesfully
44    #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
45
46    #EXCLUDE_TARGETS=dos,freebsd,windows
47
48    # Below option is required to complete this test succesfully
49    #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
50
51    #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
52
53    # To define a range of versions that a test can run with:
54    #REQUIRE_VERSION=0.0
55    #REQUIRE_VERSION_BELOW=99.9
56
57  Configure environment variables to set the haproxy and vtest binaries to use
58    setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
59    setenv VTEST_PROGRAM /usr/local/bin/vtest
60  or
61    export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
62    export VTEST_PROGRAM=/usr/local/bin/vtest
63EOF
64  exit 0
65}
66
67add_range_to_test_list()
68{
69    level0="*.vtc"
70    level1="h*.vtc"
71    level2="s*.vtc"
72    level3="l*.vtc"
73    level4="b*.vtc"
74    level5="k*.vtc"
75    level6="e*.vtc"
76
77    new_range=$(echo $1 | tr '-' ' ')
78    non_digit=$(echo $new_range | grep '[^0-9 ]')
79    if [ -n "$non_digit" ] ; then
80        return
81    fi
82    if [ "$new_range" = "$1" ] ; then
83        if [ $1 -gt 6 ] ; then
84            return
85        fi
86        eval echo '$'level$1
87        return
88    fi
89    if [ -z "$new_range" ] ; then
90        return
91    fi
92    list=
93    for l in $(seq $new_range) ; do
94        if [ -n "l" ] ; then
95            if [ -z "$list" ] ; then
96                list="$(eval echo '$'level${l})"
97            else
98                list="$list $(eval echo '$'level${l})"
99            fi
100        fi
101    done
102
103    echo $list
104}
105
106
107build_test_list()
108{
109    # Remove any spacing character
110    LEVEL="$(echo $LEVEL | tr -d ' ')"
111    # Replave any comma character by a space character
112    LEVEL="$(echo $LEVEL | tr ',' ' ')"
113    list=
114    for range in $LEVEL ; do
115        if [ -z "$list" ] ; then
116            list=$(add_range_to_test_list $range)
117        else
118            list="$list $(add_range_to_test_list $range)"
119        fi
120    done
121
122    echo $list
123}
124
125build_find_expr()
126{
127    expr=
128    for i in $@; do
129        if [ -z "$expr" ] ; then
130            expr="-name \"$i\""
131        else
132            expr="$expr -o -name \"$i\""
133        fi
134    done
135
136    echo $expr
137}
138
139_startswith() {
140  _str="$1"
141  _sub="$2"
142  echo "$_str" | grep "^$_sub" >/dev/null 2>&1
143}
144
145_findtests() {
146  set -f
147  LEVEL=${LEVEL:-0};
148  list=$(build_test_list "$LEVEL")
149  if [ -z "$list" ] ; then
150      echo "Invalid level specification '"$LEVEL"' or no file was found."
151      exit 1
152  fi
153  EXPR=$(build_find_expr $list)
154
155  for i in $( find "$1" $(eval echo $EXPR) ); do
156    skiptest=
157    require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
158    require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
159    require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed  -e 's/,/ /g')"
160    exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed  -e 's/,/ /g')"
161
162    requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed  -e 's/,.*//')"
163    if [ -n "$requiredoption" ]; then
164      require_options="$require_options $requiredoption"
165    fi
166
167    excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed  -e 's/,.*//')"
168    if [ -n "$excludedtarget" ]; then
169      exclude_targets="$exclude_targets $excludedtarget"
170    fi
171
172    if [ -n "$require_version" ]; then
173      if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
174        echo "  Skip $i because option haproxy is version: $HAPROXY_VERSION"
175        echo "    REASON: this test requires at least version: $require_version"
176        skiptest=1
177      fi
178    fi
179    if [ -n "$require_version_below" ]; then
180      if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
181        echo "  Skip $i because option haproxy is version: $HAPROXY_VERSION"
182        echo "    REASON: this test requires a version below: $require_version_below"
183        skiptest=1
184      fi
185    fi
186
187    for excludedtarget in $exclude_targets; do
188      if [ "$excludedtarget" = "$TARGET" ]; then
189        echo "  Skip $i because haproxy is compiled for the excluded target $TARGET"
190        skiptest=1
191      fi
192    done
193
194    for requiredoption in $require_options; do
195      alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
196      found=
197      for alt in $alternatives; do
198        if [ -n "$( echo "$OPTIONS" | grep "USE_$alt=1" )" ]; then
199          found=1;
200	fi
201      done
202      if [ -z $found ]; then
203        echo "  Skip $i because haproxy is not compiled with the required option $requiredoption"
204        skiptest=1
205      fi
206    done
207
208    if [ -z $skiptest ]; then
209      echo "  Add test: $i"
210      testlist="$testlist $i"
211    fi
212  done
213}
214
215_cleanup()
216{
217  DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
218  if [ -z "${DIRS}" ]; then
219    echo "No reg-tests log directory found"
220  else
221    echo "Cleanup following reg-tests log directories:"
222    for d in ${DIRS}; do
223      echo  "    o ${TESTDIR}/$d"
224    done
225    read -p "Continue (y/n)?" reply
226    case "$reply" in
227      y|Y)
228        for d in ${DIRS}; do
229          rm -r "${TESTDIR}/$d"
230        done
231        echo "done"
232        exit 0
233        ;;
234       *)
235        echo "aborted"
236        exit 1
237        ;;
238    esac
239  fi
240}
241
242
243_process() {
244  while [ ${#} -gt 0 ]; do
245    if _startswith "$1" "-"; then
246      case "${1}" in
247        --j)
248          jobcount="$2"
249          shift
250          ;;
251        --vtestparams)
252          vtestparams="$2"
253          shift
254          ;;
255        --v)
256          verbose=""
257          ;;
258        --debug)
259          verbose=""
260          debug="-v"
261          ;;
262        --keep-logs)
263          keep_logs="-L"
264          ;;
265        --LEVEL)
266          LEVEL="$2"
267          shift
268          ;;
269        --use-htx)
270          no_htx=""
271          ;;
272        --clean)
273          _cleanup
274          exit 0
275          ;;
276        --help)
277          _help
278          ;;
279        *)
280          echo "Unknown parameter : $1"
281          exit 1
282          ;;
283      esac
284    else
285      REGTESTS="${REGTESTS} $1"
286    fi
287    shift 1
288  done
289}
290
291_version() {
292  echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
293}
294
295
296HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
297VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
298TESTDIR="${TMPDIR:-/tmp}"
299REGTESTS=""
300
301jobcount=""
302verbose="-q"
303debug=""
304keep_logs="-l"
305no_htx="#"
306testlist=""
307
308_process "$@";
309
310echo ""
311echo "########################## Preparing to run tests ##########################"
312
313preparefailed=
314if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
315  echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
316  preparefailed=1
317fi
318if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
319  echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
320  preparefailed=1
321fi
322if [ $preparefailed ]; then
323  exit 1
324fi
325
326{ read HAPROXY_VERSION; read TARGET; read OPTIONS; } << EOF
327$($HAPROXY_PROGRAM -vv |grep 'HA-Proxy version\|TARGET\|OPTIONS' | sed 's/.* = //')
328EOF
329
330HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
331echo "Testing with haproxy version: $HAPROXY_VERSION"
332
333TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
334
335mkdir -p "$TESTDIR" || exit 1
336TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
337
338export TMPDIR="$TESTDIR"
339export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
340
341# Mimic implicit build options from haproxy MakeFile that are present for each target:
342
343if [ $TARGET = generic ] ; then
344  #generic system target has nothing specific
345  OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
346fi
347if [ $TARGET = haiku ] ; then
348  #For Haiku
349  OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
350fi
351if [ $TARGET = linux22 ] ; then
352  #This is for Linux 2.2
353  OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
354fi
355if [ $TARGET = linux24 ] ; then
356  #This is for standard Linux 2.4 with netfilter but without epoll()
357  OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
358fi
359if [ $TARGET = linux24e ] ; then
360  #This is for enhanced Linux 2.4 with netfilter and epoll() patch>0.21
361  OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_MY_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
362fi
363if [ $TARGET = linux26 ] ; then
364  #This is for standard Linux 2.6 with netfilter and standard epoll()
365  OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_FUTEX=1 USE_DL=1 USE_RT=1"
366fi
367if [ $TARGET = linux2628 ] ; then
368  #This is for standard Linux >= 2.6.28 with netfilter, epoll, tproxy and splice
369  OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_LINUX_SPLICE=1 USE_LINUX_TPROXY=1 USE_ACCEPT4=1 USE_FUTEX=1 USE_CPU_AFFINITY=1 ASSUME_SPLICE_WORKS=1 USE_DL=1 USE_RT=1 USE_THREAD=1"
370fi
371if [ $TARGET = solaris ] ; then
372  #This is for Solaris8
373  OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_CRYPT_H=1 USE_GETADDRINFO=1 USE_THREAD=1"
374fi
375if [ $TARGET = freebsd ] ; then
376  #This is for FreeBSD
377  OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_THREAD=1 USE_CPU_AFFINITY=1"
378fi
379if [ $TARGET = osx ] ; then
380  #This is for MacOS/X
381  OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1"
382fi
383if [ $TARGET = openbsd ] ; then
384  #This is for OpenBSD >= 5.7
385  OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1 USE_ACCEPT4=1 USE_THREAD=1"
386fi
387if [ $TARGET = netbsd ] ; then
388  #This is for NetBSD
389  OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1"
390fi
391if [ $TARGET = aix51 ] ; then
392  #This is for AIX 5.1
393  OPTIONS="$OPTIONS USE_POLL=1 USE_LIBCRYPT=1"
394fi
395if [ $TARGET = aix52 ] ; then
396  #This is for AIX 5.2 and later
397  OPTIONS="$OPTIONS USE_POLL=1 USE_LIBCRYPT=1"
398fi
399if [ $TARGET = cygwin ] ; then
400  #This is for Cygwin
401  OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
402fi
403
404echo "Target : $TARGET"
405echo "Options : $OPTIONS"
406
407echo "########################## Gathering tests to run ##########################"
408# if 'use-htx' option is set, but HAProxy version is lower to 1.9, disable it
409if [ -z "$no_htx" ]; then
410  if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
411    echo ""
412    echo "WARNING : Unset HTX for haproxy (version: $HAPROXY_VERSION)"
413    echo "    REASON: this test requires at least version: 1.9"
414    echo ""
415    no_htx="#"
416  fi
417fi
418
419if [ -z "$REGTESTS" ]; then
420  _findtests ./
421else
422  for t in $REGTESTS; do
423    _findtests $t
424  done
425fi
426
427echo "########################## Starting vtest ##########################"
428echo "Testing with haproxy version: $HAPROXY_VERSION"
429_vtresult=0
430if [ -n "$testlist" ]; then
431  if [ -n "$jobcount" ]; then
432    jobcount="-j $jobcount"
433  fi
434  cmd="$VTEST_PROGRAM -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
435  eval $cmd
436  _vtresult=$?
437else
438  echo "No tests found that meet the required criteria"
439fi
440
441
442if [ $_vtresult -eq 0 ]; then
443  # all tests were succesfull, removing tempdir (the last part.)
444  # ignore errors is the directory is not empty or if it does not exist
445   rmdir "$TESTDIR" 2>/dev/null
446fi
447
448if [ -d "${TESTDIR}" ]; then
449  echo "########################## Gathering results ##########################"
450  export TESTDIR
451  find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
452    if [ ! -e "$i/LOG" ] ; then continue; fi
453
454    cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
455$(echo "###### $(cat "$i/INFO") ######")
456$(echo "## test results in: \"$i\"")
457$(grep -E -- "^(----|\*    diag)" "$i/LOG")
458EOF
459  done' sh {} +
460fi
461
462exit $_vtresult
463