1#!/bin/sh
2#
3# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
4#
5# This Source Code Form is subject to the terms of the Mozilla Public
6# License, v. 2.0. If a copy of the MPL was not distributed with this
7# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8#
9# See the COPYRIGHT file distributed with this work for additional
10# information regarding copyright ownership.
11
12# Creates the system tests output file from the various test.output.* files.  It
13# then searches that file and prints the number of tests passed, failed, not
14# run.  It also checks whether the IP addresses 10.53.0.[1-8] were set up and,
15# if not, prints a warning.
16#
17# Usage:
18#    testsummary.sh [-n]
19#
20# -n	Do NOT delete the individual test.output.* files after concatenating
21#	them into systests.output.
22#
23# Status return:
24# 0 - no tests failed
25# 1 - one or more tests failed
26
27SYSTEMTESTTOP=.
28. $SYSTEMTESTTOP/conf.sh
29
30keepfile=0
31
32while getopts "n" flag; do
33    case $flag in
34	n) keepfile=1 ;;
35    esac
36done
37
38if [ `ls test.output.* 2> /dev/null | wc -l` -eq 0 ]; then
39    echowarn "I:No 'test.output.*' files were found."
40    echowarn "I:Printing summary from pre-existing 'systests.output'."
41else
42    cat test.output.* > systests.output
43    if [ $keepfile -eq 0 ]; then
44        rm -f test.output.*
45    fi
46fi
47
48status=0
49echoinfo "I:System test result summary:"
50echoinfo "`grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'`"
51
52FAILED_TESTS=`grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I:      /'`
53if [ -n "${FAILED_TESTS}" ]; then
54	echoinfo "I:The following system tests failed:"
55	echoinfo "${FAILED_TESTS}"
56	status=1
57fi
58
59CRASHED_TESTS=`find . -name 'core*' -or -name '*.core' | cut -d'/' -f2 | sort -u | sed -e 's/^/I:      /'`
60if [ -n "${CRASHED_TESTS}" ]; then
61	echoinfo "I:Core dumps were found for the following system tests:"
62	echoinfo "${CRASHED_TESTS}"
63fi
64
65ASSERTION_FAILED_TESTS=`find . -name named.run | xargs grep "assertion failure" | cut -d'/' -f2 | sort -u | sed -e 's/^/I:      /'`
66if [ -n "${ASSERTION_FAILED_TESTS}" ]; then
67	echoinfo "I:Assertion failures were detected for the following system tests:"
68	echoinfo "${ASSERTION_FAILED_TESTS}"
69fi
70
71TSAN_REPORT_TESTS=`find . -name 'tsan.*' | cut -d'/' -f2 | sort -u | sed -e 's/^/I:      /'`
72if [ -n "${TSAN_REPORT_TESTS}" ]; then
73	echoinfo "I:ThreadSanitizer reported issues for the following system tests:"
74	echoinfo "${TSAN_REPORT_TESTS}"
75fi
76
77RESULTS_FOUND=`grep -c 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output`
78TESTS_RUN=`echo "${SUBDIRS}" | wc -w`
79if [ "${RESULTS_FOUND}" -ne "${TESTS_RUN}" ]; then
80	echofail "I:Found ${RESULTS_FOUND} test results, but ${TESTS_RUN} tests were run"
81	status=1
82fi
83
84exit $status
85