1#! /bin/sh 2 3# For a specified tool and optional list of test variants, extract 4# test results from one or more test summary (.sum) files and combine 5# the results into a new test summary file, sent to the standard output. 6# The resulting file can be used with test result comparison scripts for 7# results from tests that were run in parallel. See usage() below. 8 9# Copyright (C) 2008, 2009, 2010, 2012 Free Software Foundation 10# Contributed by Janis Johnson <janis187@us.ibm.com> 11# 12# This file is part of GCC. 13# 14# GCC is free software; you can redistribute it and/or modify 15# it under the terms of the GNU General Public License as published by 16# the Free Software Foundation; either version 3, or (at your option) 17# any later version. 18# 19# GCC is distributed in the hope that it will be useful, 20# but WITHOUT ANY WARRANTY; without even the implied warranty of 21# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22# GNU General Public License for more details. 23# 24# You should have received a copy of the GNU General Public License 25# along with GCC; see the file COPYING. If not, write to 26# the Free Software Foundation, 51 Franklin Street, Fifth Floor, 27# Boston, MA 02110-1301, USA. 28 29PROGNAME=dg-extract-results.sh 30 31usage() { 32 cat <<EOF >&2 33Usage: $PROGNAME [-t tool] [-l variant-list] [-L] sum-file ... 34 35 tool The tool (e.g. g++, libffi) for which to create a 36 new test summary file. If not specified then all 37 specified sum files must be for the same tool. 38 variant-list One or more test variant names. If the list is 39 not specified then one is constructed from all 40 variants in the files for <tool>. 41 sum-file A test summary file with the format of those 42 created by runtest from DejaGnu. 43 If -L is used, merge *.log files instead of *.sum. In this 44 mode the exact order of lines may not be preserved, just different 45 Running *.exp chunks should be in correct order. 46EOF 47} 48 49# Write a message to the standard error. 50 51msg() { 52 echo "$@" >&2 53} 54 55# Parse the command-line options. 56 57VARIANTS="" 58TOOL="" 59MODE="sum" 60 61while getopts "l:t:L" ARG; do 62 case $ARG in 63 l) VARIANTS="${VARIANTS} ${OPTARG}";; 64 t) test -z "$TOOL" || (msg "${PROGNAME}: only one tool can be specified"; exit 1); 65 TOOL="${OPTARG}";; 66 L) MODE="log";; 67 \?) usage; exit 0;; 68 esac 69done 70shift `expr ${OPTIND} - 1` 71 72if test $# -lt 1 ; then 73 usage 74 exit 1 75fi 76 77TMPDIR=${TMPDIR-/tmp} 78SUM_FILES="$@" 79FIRST_SUM=$1 80TMP= 81trap 'EXIT_STATUS=$?; rm -rf $TMP && exit $EXIT_STATUS' 0 82# Create a (secure) tmp directory for tmp files. 83{ 84 TMP=`(umask 077 && mktemp -d -q "${TMPDIR}/dg-combine-results-$$-XXXXXX") 2>/dev/null` && 85 test -n "$TMP" && test -d "$TMP" 86} || 87{ 88 TMP=${TMPDIR}/dg-combine-results-$$-$RANDOM 89 (umask 077 && mkdir $TMP) 90} || 91{ 92 msg "${PROGNAME}: cannot create a temporary directory" 93 { (exit 1); exit 1; } 94} 95 96# Find a good awk. 97 98if test -z "$AWK" ; then 99 for AWK in gawk nawk awk 100 do 101 if type $AWK 2>&1 | grep 'not found' > /dev/null 2>&1 ; then 102 : 103 else 104 break 105 fi 106 done 107fi 108 109# Verify that the specified summary files exist. 110 111ERROR=0 112for FILE in $SUM_FILES 113do 114 if ! test -f $FILE ; then 115 msg "${PROGNAME}: file $FILE does not exist." 116 ERROR=1 117 fi 118done 119test $ERROR -eq 0 || exit 1 120 121if [ -z "$TOOL" ]; then 122 # If no tool was specified, all specified summary files must be for 123 # the same tool. 124 125 CNT=`grep '=== .* tests ===' $SUM_FILES | $AWK '{ print $3 }' | sort -u | wc -l` 126 if [ $CNT -eq 1 ]; then 127 TOOL=`grep '=== .* tests ===' $FIRST_SUM | $AWK '{ print $2 }'` 128 else 129 msg "${PROGNAME}: sum files are for multiple tools, specify a tool" 130 msg "" 131 usage 132 exit 1 133 fi 134else 135 # Ignore the specified summary files that are not for this tool. This 136 # should keep the relevant files in the same order. 137 138 SUM_FILES=`grep -l "=== $TOOL" $SUM_FILES` 139 if test -z "$SUM_FILES" ; then 140 msg "${PROGNAME}: none of the specified files are results for $TOOL" 141 exit 1 142 fi 143fi 144 145if [ "$TOOL" = acats ]; then 146 # Acats *.sum or *.log files aren't dejagnu generated, and they have 147 # somewhat different format. 148 ACATS_AWK=${TMP}/acats.awk 149 cat <<EOF > $ACATS_AWK 150BEGIN { 151 print_prologue=1; curfile=""; insummary=0 152 passcnt=0; failcnt=0; unsupcnt=0; failures="" 153} 154/^[ \t]*=== acats configuration ===/ { 155 insummary=0 156 if (print_prologue) print 157 next 158} 159/^[ \t]*=== acats tests ===/ { 160 if (print_prologue) print 161 print_prologue=0 162 next 163} 164/^Running chapter / { 165 if (curfile) close (curfile) 166 curfile="${TMP}/chapter-"\$3 167 print >> curfile 168 next 169} 170/^[ \t]*=== acats Summary ===/ { 171 if (curfile) close (curfile) 172 curfile="" 173 insummary=1 174 next 175} 176/^# of expected passes/ { if (insummary == 1) passcnt += \$5; next; } 177/^# of unexpected failures/ { if (insummary == 1) failcnt += \$5; next; } 178/^# of unsupported tests/ { if (insummary == 1) unsupcnt += \$5; next; } 179/^\*\*\* FAILURES: / { 180 if (insummary == 1) { 181 if (failures) sub(/^\*\*\* FAILURES:/,"") 182 failures=failures""\$0 183 } 184} 185{ 186 if (print_prologue) { print; next } 187 if (curfile) print >> curfile 188} 189END { 190 system ("cat ${TMP}/chapter-*") 191 print " === acats Summary ===" 192 print "# of expected passes " passcnt 193 print "# of unexpected failures " failcnt 194 if (unsupcnt) print "# of unsupported tests " unsupcnt 195 if (failures) print failures 196} 197EOF 198 199 rm -f ${TMP}/chapter-* 200 $AWK -f $ACATS_AWK $SUM_FILES 201 exit 0 202fi 203 204# If no variants were specified, find all variants in the remaining 205# summary files. Otherwise, ignore specified variants that aren't in 206# any of those summary files. 207 208if test -z "$VARIANTS" ; then 209 VAR_AWK=${TMP}/variants.awk 210 cat <<EOF > $VAR_AWK 211/^Schedule of variations:/ { in_vars=1; next } 212/^$/ { in_vars=0 } 213/^Running target/ { exit } 214{ if (in_vars==1) print \$1; else next } 215EOF 216 217 touch ${TMP}/varlist 218 for FILE in $SUM_FILES; do 219 $AWK -f $VAR_AWK $FILE >> ${TMP}/varlist 220 done 221 VARIANTS="`sort -u ${TMP}/varlist`" 222else 223 VARS="$VARIANTS" 224 VARIANTS="" 225 for VAR in $VARS 226 do 227 grep "Running target $VAR" $SUM_FILES > /dev/null && VARIANTS="$VARIANTS $VAR" 228 done 229fi 230 231# Find out if we have more than one variant, or any at all. 232 233VARIANT_COUNT=0 234for VAR in $VARIANTS 235do 236 VARIANT_COUNT=`expr $VARIANT_COUNT + 1` 237done 238 239if test $VARIANT_COUNT -eq 0 ; then 240 msg "${PROGNAME}: no file for $TOOL has results for the specified variants" 241 exit 1 242fi 243 244cat $SUM_FILES \ 245 | $AWK '/^Running/ { if ($2 != "target" && $3 == "...") print "EXPFILE: "$2 } ' \ 246 | sort -u > ${TMP}/expfiles 247 248# Write the begining of the combined summary file. 249 250head -n 2 $FIRST_SUM 251echo 252echo " === $TOOL tests ===" 253echo 254echo "Schedule of variations:" 255for VAR in $VARIANTS 256do 257 echo " $VAR" 258done 259echo 260 261# For each test variant for the tool, copy test reports from each of the 262# summary files. Set up two awk scripts from within the loop to 263# initialize VAR and TOOL with the script, rather than assuming that the 264# available version of awk can pass variables from the command line. 265 266for VAR in $VARIANTS 267do 268 GUTS_AWK=${TMP}/guts.awk 269 cat << EOF > $GUTS_AWK 270BEGIN { 271 variant="$VAR" 272 firstvar=1 273 expfileno=1 274 cnt=0 275 print_using=0 276 need_close=0 277} 278/^EXPFILE: / { 279 expfiles[expfileno] = \$2 280 expfilesr[\$2] = expfileno 281 expfileno = expfileno + 1 282} 283/^Running target / { 284 curvar = \$3 285 if (variant == curvar && firstvar == 1) { print; print_using=1; firstvar = 0 } 286 next 287} 288/^Using / { 289 if (variant == curvar && print_using) { print; next } 290} 291/^Running .*\\.exp \\.\\.\\./ { 292 print_using=0 293 if (variant == curvar) { 294 if (need_close) close(curfile) 295 curfile="${TMP}/list"expfilesr[\$2] 296 expfileseen[\$2]=expfileseen[\$2] + 1 297 need_close=0 298 testname="00" 299 next 300 } 301} 302/^\t\t=== .* ===$/ { curvar = ""; next } 303/^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|WARNING|ERROR|UNSUPPORTED|UNTESTED|KFAIL):/ { 304 testname=\$2 305 # Ugly hack for gfortran.dg/dg.exp 306 if ("$TOOL" == "gfortran" && testname ~ /^gfortran.dg\/g77\//) 307 testname="h"testname 308} 309/^$/ { if ("$MODE" == "sum") next } 310{ if (variant == curvar && curfile) { 311 if ("$MODE" == "sum") { 312 printf "%s %08d|", testname, cnt >> curfile 313 cnt = cnt + 1 314 } 315 filewritten[curfile]=1 316 need_close=1 317 print >> curfile 318 } else 319 next 320} 321END { 322 n=1 323 while (n < expfileno) { 324 if (expfileseen[expfiles[n]]) { 325 print "Running "expfiles[n]" ..." 326 if (filewritten["${TMP}/list"n]) { 327 if (expfileseen[expfiles[n]] == 1) 328 cmd="cat" 329 else 330 cmd="LC_ALL=C sort" 331 if ("$MODE" == "sum") 332 system (cmd" ${TMP}/list"n" | sed -n 's/^[^ ]* [^ |]*|//p'") 333 else 334 system ("cat ${TMP}/list"n) 335 } 336 } 337 n = n + 1 338 } 339} 340EOF 341 342 SUMS_AWK=${TMP}/sums.awk 343 rm -f $SUMS_AWK 344 cat << EOF > $SUMS_AWK 345BEGIN { 346 variant="$VAR" 347 tool="$TOOL" 348 passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kpasscnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0; 349 curvar=""; insummary=0 350} 351/^Running target / { curvar = \$3; next } 352/^# of / { if (variant == curvar) insummary = 1 } 353/^# of expected passes/ { if (insummary == 1) passcnt += \$5; next; } 354/^# of unexpected successes/ { if (insummary == 1) xpasscnt += \$5; next; } 355/^# of unexpected failures/ { if (insummary == 1) failcnt += \$5; next; } 356/^# of expected failures/ { if (insummary == 1) xfailcnt += \$5; next; } 357/^# of unknown successes/ { if (insummary == 1) kpasscnt += \$5; next; } 358/^# of known failures/ { if (insummary == 1) kfailcnt += \$5; next; } 359/^# of untested testcases/ { if (insummary == 1) untstcnt += \$5; next; } 360/^# of unresolved testcases/ { if (insummary == 1) unrescnt += \$5; next; } 361/^# of unsupported tests/ { if (insummary == 1) unsupcnt += \$5; next; } 362/^$/ { if (insummary == 1) 363 { insummary = 0; curvar = "" } 364 next 365 } 366{ next } 367END { 368 printf ("\t\t=== %s Summary for %s ===\n\n", tool, variant) 369 if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt) 370 if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt) 371 if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt) 372 if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt) 373 if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt) 374 if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt) 375 if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt) 376 if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt) 377 if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt) 378} 379EOF 380 381 PVAR=`echo $VAR | sed 's,/,.,g'` 382 TMPFILE=${TMP}/var-$PVAR 383 rm -f $TMPFILE 384 rm -f ${TMP}/list* 385 cat ${TMP}/expfiles $SUM_FILES | $AWK -f $GUTS_AWK 386 cat $SUM_FILES | $AWK -f $SUMS_AWK > $TMPFILE 387 # If there are multiple variants, output the counts for this one; 388 # otherwise there will just be the final counts at the end. 389 test $VARIANT_COUNT -eq 1 || cat $TMPFILE 390done 391 392# Set up an awk script to get the combined summary counts for the tool. 393 394TOTAL_AWK=${TMP}/total.awk 395cat << EOF > $TOTAL_AWK 396BEGIN { 397 tool="$TOOL" 398 passcnt=0; failcnt=0; untstcnt=0; xpasscnt=0; xfailcnt=0; kfailcnt=0; unsupcnt=0; unrescnt=0 399} 400/^# of expected passes/ { passcnt += \$5 } 401/^# of unexpected failures/ { failcnt += \$5 } 402/^# of unexpected successes/ { xpasscnt += \$5 } 403/^# of expected failures/ { xfailcnt += \$5 } 404/^# of unknown successes/ { kpasscnt += \$5 } 405/^# of known failures/ { kfailcnt += \$5 } 406/^# of untested testcases/ { untstcnt += \$5 } 407/^# of unresolved testcases/ { unrescnt += \$5 } 408/^# of unsupported tests/ { unsupcnt += \$5 } 409END { 410 printf ("\n\t\t=== %s Summary ===\n\n", tool) 411 if (passcnt != 0) printf ("# of expected passes\t\t%d\n", passcnt) 412 if (failcnt != 0) printf ("# of unexpected failures\t%d\n", failcnt) 413 if (xpasscnt != 0) printf ("# of unexpected successes\t%d\n", xpasscnt) 414 if (xfailcnt != 0) printf ("# of expected failures\t\t%d\n", xfailcnt) 415 if (kpasscnt != 0) printf ("# of unknown successes\t\t%d\n", kpasscnt) 416 if (kfailcnt != 0) printf ("# of known failures\t\t%d\n", kfailcnt) 417 if (untstcnt != 0) printf ("# of untested testcases\t\t%d\n", untstcnt) 418 if (unrescnt != 0) printf ("# of unresolved testcases\t%d\n", unrescnt) 419 if (unsupcnt != 0) printf ("# of unsupported tests\t\t%d\n", unsupcnt) 420} 421EOF 422 423# Find the total summaries for the tool and add to the end of the output. 424cat ${TMP}/var-* | $AWK -f $TOTAL_AWK 425 426# This is ugly, but if there's version output from the compiler under test 427# at the end of the file, we want it. The other thing that might be there 428# is the final summary counts. 429tail -2 $FIRST_SUM | grep '^#' > /dev/null || tail -2 $FIRST_SUM 430 431exit 0 432