1#! /bin/sh
2
3# For a specified tool and optional list of test variants, extract
4# test results from one or more test summary (.sum) files and combine
5# the results into a new test summary file, sent to the standard output.
6# The resulting file can be used with test result comparison scripts for
7# results from tests that were run in parallel.  See usage() below.
8
9# Copyright (C) 2008-2013 Free Software Foundation, Inc.
10# Contributed by Janis Johnson <janis187@us.ibm.com>
11#
12# This file is part of GCC.
13#
14# GCC is free software; you can redistribute it and/or modify
15# it under the terms of the GNU General Public License as published by
16# the Free Software Foundation; either version 3, or (at your option)
17# any later version.
18#
19# GCC is distributed in the hope that it will be useful,
20# but WITHOUT ANY WARRANTY; without even the implied warranty of
21# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22# GNU General Public License for more details.
23#
24# You should have received a copy of the GNU General Public License
25# along with this program.  If not, see <http://www.gnu.org/licenses/>.
26
27PROGNAME=dg-extract-results.sh
28
29usage() {
30  cat <<EOF >&2
31Usage: $PROGNAME [-t tool] [-l variant-list] [-L] sum-file ...
32
33    tool           The tool (e.g. g++, libffi) for which to create a
34                   new test summary file.  If not specified then all
35                   specified sum files must be for the same tool.
36    variant-list   One or more test variant names.  If the list is
37                   not specified then one is constructed from all
38                   variants in the files for <tool>.
39    sum-file       A test summary file with the format of those
40                   created by runtest from DejaGnu.
41    If -L is used, merge *.log files instead of *.sum.  In this
42    mode the exact order of lines may not be preserved, just different
43    Running *.exp chunks should be in correct order.
44EOF
45}
46
47# Write a message to the standard error.
48
49msg() {
50  echo "$@" >&2
51}
52
53# Parse the command-line options.
54
55VARIANTS=""
56TOOL=""
57MODE="sum"
58
59while getopts "l:t:L" ARG; do
60  case $ARG in
61  l)  VARIANTS="${VARIANTS} ${OPTARG}";;
62  t)  test -z "$TOOL" || (msg "${PROGNAME}: only one tool can be specified"; exit 1);
63      TOOL="${OPTARG}";;
64  L)  MODE="log";;
65  \?) usage; exit 0;;
66  esac
67done
68shift `expr ${OPTIND} - 1`
69
70if test $# -lt 1 ; then
71  usage
72  exit 1
73fi
74
75TMPDIR=${TMPDIR-/tmp}
76SUM_FILES="$@"
77FIRST_SUM=$1
78TMP=
79trap 'EXIT_STATUS=$?; rm -rf $TMP && exit $EXIT_STATUS' 0
80# Create a (secure) tmp directory for tmp files.
81{
82  TMP=`(umask 077 && mktemp -d -q "${TMPDIR}/dg-combine-results-$$-XXXXXX") 2>/dev/null` &&
83  test -n "$TMP" && test -d "$TMP"
84} ||
85{
86  TMP=${TMPDIR}/dg-combine-results-$$-$RANDOM
87  (umask 077 && mkdir $TMP)
88} ||
89{
90  msg "${PROGNAME}: cannot create a temporary directory"
91  { (exit 1); exit 1; }
92}
93
94# Find a good awk.
95
96if test -z "$AWK" ; then
97  for AWK in gawk nawk awk
98  do
99    if type $AWK 2>&1 | grep 'not found' > /dev/null 2>&1 ; then
100      :
101    else
102      break
103    fi
104  done
105fi
106
107# Verify that the specified summary files exist.
108
109ERROR=0
110for FILE in $SUM_FILES
111do
112  if ! test -f $FILE ; then
113    msg "${PROGNAME}: file $FILE does not exist."
114    ERROR=1
115  fi
116done
117test $ERROR -eq 0 || exit 1
118
119if [ -z "$TOOL" ]; then
120  # If no tool was specified, all specified summary files must be for
121  # the same tool.
122
123  CNT=`grep '=== .* tests ===' $SUM_FILES | $AWK '{ print $3 }' | sort -u | wc -l`
124  if [ $CNT -eq 1 ]; then
125    TOOL=`grep '=== .* tests ===' $FIRST_SUM | $AWK '{ print $2 }'`
126  else
127    msg "${PROGNAME}: sum files are for multiple tools, specify a tool"
128    msg ""
129    usage
130    exit 1
131  fi
132else
133  # Ignore the specified summary files that are not for this tool.  This
134  # should keep the relevant files in the same order.
135
136  SUM_FILES=`grep -l "=== $TOOL" $SUM_FILES`
137  if test -z "$SUM_FILES" ; then
138    msg "${PROGNAME}: none of the specified files are results for $TOOL"
139    exit 1
140  fi
141fi
142
143if [ "$TOOL" = acats ]; then
144  # Acats *.sum or *.log files aren't dejagnu generated, and they have
145  # somewhat different format.
146  ACATS_AWK=${TMP}/acats.awk
147  cat <<EOF > $ACATS_AWK
148BEGIN {
149  print_prologue=1; curfile=""; insummary=0
150  passcnt=0; failcnt=0; unsupcnt=0; failures=""
151}
152/^[ \t]*=== acats configuration ===/ {
153  insummary=0
154  if (print_prologue) print
155  next
156}
157/^[ \t]*=== acats tests ===/ {
158  if (print_prologue) print
159  print_prologue=0
160  next
161}
162/^Running chapter / {
163  if (curfile) close (curfile)
164  curfile="${TMP}/chapter-"\$3
165  print >> curfile
166  next
167}
168/^[ \t]*=== acats Summary ===/ {
169  if (curfile) close (curfile)
170  curfile=""
171  insummary=1
172  next
173}
174/^# of expected passes/		{ if (insummary == 1) passcnt += \$5; next; }
175/^# of unexpected failures/	{ if (insummary == 1) failcnt += \$5; next; }
176/^# of unsupported tests/	{ if (insummary == 1) unsupcnt += \$5; next; }
177/^\*\*\* FAILURES: / {
178  if (insummary == 1) {
179    if (failures) sub(/^\*\*\* FAILURES:/,"")
180    failures=failures""\$0
181  }
182}
183{
184  if (print_prologue) { print; next }
185  if (curfile) print >> curfile
186}
187END {
188  system ("cat ${TMP}/chapter-*")
189  print "		=== acats Summary ==="
190  print "# of expected passes		" passcnt
191  print "# of unexpected failures	" failcnt
192  if (unsupcnt) print "# of unsupported tests		" unsupcnt
193  if (failures) print failures
194}
195EOF
196
197  rm -f ${TMP}/chapter-*
198  $AWK -f $ACATS_AWK $SUM_FILES
199  exit 0
200fi
201
202# If no variants were specified, find all variants in the remaining
203# summary files.  Otherwise, ignore specified variants that aren't in
204# any of those summary files.
205
206if test -z "$VARIANTS" ; then
207  VAR_AWK=${TMP}/variants.awk
208  cat <<EOF > $VAR_AWK
209/^Schedule of variations:/      { in_vars=1; next }
210/^$/                            { in_vars=0 }
211/^Running target/               { exit }
212{ if (in_vars==1) print \$1; else next }
213EOF
214
215  touch ${TMP}/varlist
216  for FILE in $SUM_FILES; do
217    $AWK -f $VAR_AWK $FILE >> ${TMP}/varlist
218  done
219  VARIANTS="`sort -u ${TMP}/varlist`"
220else
221  VARS="$VARIANTS"
222  VARIANTS=""
223  for VAR in $VARS
224  do
225    grep "Running target $VAR" $SUM_FILES > /dev/null && VARIANTS="$VARIANTS $VAR"
226  done
227fi
228
229# Find out if we have more than one variant, or any at all.
230
231VARIANT_COUNT=0
232for VAR in $VARIANTS
233do
234  VARIANT_COUNT=`expr $VARIANT_COUNT + 1`
235done
236
237if test $VARIANT_COUNT -eq 0 ; then
238  msg "${PROGNAME}: no file for $TOOL has results for the specified variants"
239  exit 1
240fi
241
242cat $SUM_FILES \
243  | $AWK '/^Running/ { if ($2 != "target" && $3 == "...") print "EXPFILE: "$2 } ' \
244  | sort -u > ${TMP}/expfiles
245
246# Write the begining of the combined summary file.
247
248head -n 2 $FIRST_SUM
249echo
250echo "		=== $TOOL tests ==="
251echo
252echo "Schedule of variations:"
253for VAR in $VARIANTS
254do
255  echo "    $VAR"
256done
257echo
258
259# For each test variant for the tool, copy test reports from each of the
260# summary files.  Set up two awk scripts from within the loop to
261# initialize VAR and TOOL with the script, rather than assuming that the
262# available version of awk can pass variables from the command line.
263
264for VAR in $VARIANTS
265do
266  GUTS_AWK=${TMP}/guts.awk
267  cat << EOF > $GUTS_AWK
268BEGIN {
269  variant="$VAR"
270  firstvar=1
271  expfileno=1
272  cnt=0
273  print_using=0
274  need_close=0
275}
276/^EXPFILE: / {
277  expfiles[expfileno] = \$2
278  expfilesr[\$2] = expfileno
279  expfileno = expfileno + 1
280}
281/^Running target / {
282  curvar = \$3
283  if (variant == curvar && firstvar == 1) { print; print_using=1; firstvar = 0 }
284  next
285}
286/^Using / {
287  if (variant == curvar && print_using) { print; next }
288}
289/^Running .*\\.exp \\.\\.\\./ {
290  print_using=0
291  if (variant == curvar) {
292    if (need_close) close(curfile)
293    curfile="${TMP}/list"expfilesr[\$2]
294    expfileseen[\$2]=expfileseen[\$2] + 1
295    need_close=0
296    testname="00"
297    next
298  }
299}
300/^\t\t=== .* ===$/ { curvar = ""; next }
301/^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|WARNING|ERROR|UNSUPPORTED|UNTESTED|KFAIL):/ {
302  testname=\$2
303  # Ugly hack for gfortran.dg/dg.exp
304  if ("$TOOL" == "gfortran" && testname ~ /^gfortran.dg\/g77\//)
305    testname="h"testname
306}
307/^$/ { if ("$MODE" == "sum") next }
308{ if (variant == curvar && curfile) {
309    if ("$MODE" == "sum") {
310      printf "%s %08d|", testname, cnt >> curfile
311      cnt = cnt + 1
312    }
313    filewritten[curfile]=1
314    need_close=1
315    print >> curfile
316  } else
317    next
318}
319END {
320  n=1
321  while (n < expfileno) {
322    if (expfileseen[expfiles[n]]) {
323      print "Running "expfiles[n]" ..."
324      if (filewritten["${TMP}/list"n]) {
325	if (expfileseen[expfiles[n]] == 1)
326	  cmd="cat"
327	else
328	  cmd="LC_ALL=C sort"
329	if ("$MODE" == "sum")
330	  system (cmd" ${TMP}/list"n" | sed -n 's/^[^ ]* [^ |]*|//p'")
331	else
332	  system ("cat ${TMP}/list"n)
333      }
334    }
335    n = n + 1
336  }
337}
338EOF
339
340  SUMS_AWK=${TMP}/sums.awk
341  rm -f $SUMS_AWK
342  cat << EOF > $SUMS_AWK
343BEGIN {
344  variant="$VAR"
345  tool="$TOOL"
346  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kpasscnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0;
347  curvar=""; insummary=0
348}
349/^Running target /		{ curvar = \$3; next }
350/^# of /			{ if (variant == curvar) insummary = 1 }
351/^# of expected passes/		{ if (insummary == 1) passcnt += \$5; next; }
352/^# of unexpected successes/	{ if (insummary == 1) xpasscnt += \$5; next; }
353/^# of unexpected failures/	{ if (insummary == 1) failcnt += \$5; next; }
354/^# of expected failures/	{ if (insummary == 1) xfailcnt += \$5; next; }
355/^# of unknown successes/	{ if (insummary == 1) kpasscnt += \$5; next; }
356/^# of known failures/		{ if (insummary == 1) kfailcnt += \$5; next; }
357/^# of untested testcases/	{ if (insummary == 1) untstcnt += \$5; next; }
358/^# of unresolved testcases/	{ if (insummary == 1) unrescnt += \$5; next; }
359/^# of unsupported tests/	{ if (insummary == 1) unsupcnt += \$5; next; }
360/^$/				{ if (insummary == 1)
361				    { insummary = 0; curvar = "" }
362				  next
363				}
364{ next }
365END {
366  printf ("\t\t=== %s Summary for %s ===\n\n", tool, variant)
367  if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt)
368  if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt)
369  if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt)
370  if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt)
371  if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt)
372  if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt)
373  if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt)
374  if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt)
375  if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt)
376}
377EOF
378
379  PVAR=`echo $VAR | sed 's,/,.,g'`
380  TMPFILE=${TMP}/var-$PVAR
381  rm -f $TMPFILE
382  rm -f ${TMP}/list*
383  cat ${TMP}/expfiles $SUM_FILES | $AWK -f $GUTS_AWK
384  cat $SUM_FILES | $AWK -f $SUMS_AWK > $TMPFILE
385  # If there are multiple variants, output the counts for this one;
386  # otherwise there will just be the final counts at the end.
387  test $VARIANT_COUNT -eq 1 || cat $TMPFILE
388done
389
390# Set up an awk script to get the combined summary counts for the tool.
391
392TOTAL_AWK=${TMP}/total.awk
393cat << EOF > $TOTAL_AWK
394BEGIN {
395  tool="$TOOL"
396  passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0
397}
398/^# of expected passes/		{ passcnt += \$5 }
399/^# of unexpected failures/	{ failcnt += \$5 }
400/^# of unexpected successes/	{ xpasscnt += \$5 }
401/^# of expected failures/	{ xfailcnt += \$5 }
402/^# of unknown successes/	{ kpasscnt += \$5 }
403/^# of known failures/		{ kfailcnt += \$5 }
404/^# of untested testcases/	{ untstcnt += \$5 }
405/^# of unresolved testcases/	{ unrescnt += \$5 }
406/^# of unsupported tests/	{ unsupcnt += \$5 }
407END {
408  printf ("\n\t\t=== %s Summary ===\n\n", tool)
409  if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt)
410  if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt)
411  if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt)
412  if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt)
413  if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt)
414  if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt)
415  if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt)
416  if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt)
417  if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt)
418}
419EOF
420
421# Find the total summaries for the tool and add to the end of the output.
422cat ${TMP}/var-* | $AWK -f $TOTAL_AWK
423
424# This is ugly, but if there's version output from the compiler under test
425# at the end of the file, we want it.  The other thing that might be there
426# is the final summary counts.
427tail -2 $FIRST_SUM | grep '^#' > /dev/null || tail -2 $FIRST_SUM
428
429exit 0
430