1#!/bin/bash
2# Copyright (C) 2006, 2008 Free Software Foundation
3#
4# Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
5# Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
6#
7# See usage() below.
8
9usage () {
10    cat <<EOF >&2
11Usage:
12    dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
13    <variant-name> names the desired variant, "/" must be written as "\/".
14    Use the empty string ("") for the first variant in each file.
15    Output is to stdout.
16    Non-verbose output is degradation info like PASS->FAIL.
17    -v adds improvement info like FAIL->PASS.
18    -v -v adds info like tests that are no longer run.
19    -v -v -v adds info for tests that have not changed status.
20    -v -v -v -v is used for debugging.
21EOF
22}
23
24verbose=0
25while test "$1" = "-v"; do
26    verbose=`expr $verbose + 1`
27    shift
28done
29
30if test $# -ne 3 ; then
31    usage
32    exit 1
33fi
34
35if test ! -f "$2"; then
36    echo "unable to open $2" >&2
37    exit 1
38fi
39
40if test ! -f "$3"; then
41    echo "unable to open $3" >&2
42    exit 1
43fi
44
45# Command differences for various platforms.
46case `uname -s` in
47Darwin|NetBSD)
48    E=-E	# sed
49    ;;
50*)
51    E=-r	# sed
52    ;;
53esac
54
55# sections are identified by separator lines beginning with '\t\t==='.
56# section 0 identifies run date, target, and host.
57# section 1 and subsequent contain test data for a target variant.
58# -skip to /^Running target/ and use that line to identify the variant.
59# -subsequent lines contain the result data.  They begin with:
60# '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
61VARIANT="$1"
62OFILE="$2"
63OBASE=`basename "$2"`
64NFILE="$3"
65NBASE=`basename "$3"`
66TMPDIR=${TMPDIR:-/tmp}
67
68echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
69echo
70
71header="^Running target $VARIANT"
72
73temp=`grep "$header" $OFILE`
74if test -z "$temp"; then
75    echo "Error: variant \"$VARIANT\" not found in $OFILE."
76    exit 1
77fi
78temp=`grep "$header" $NFILE`
79if test -z "$temp"; then
80    echo "Error: variant \"$VARIANT\" not found in $NFILE."
81    exit 1
82fi
83unset temp
84
85# Copy out the old file's section 0.
86echo "Older log file: $OFILE"
87sed $E -e '/^[[:space:]]+===/,$d' $OFILE
88
89# Copy out the new file's section 0.
90echo "Newer log file: $NFILE"
91sed $E -e '/^[[:space:]]+===/,$d' $NFILE
92
93# Create a temporary file from the old file's interesting section.
94sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
95  -e '/^[A-Z]+:/!d' \
96  -e '/^(WARNING|ERROR):/d' \
97  -e 's/\r$//' \
98  -e 's/^/O:/' \
99  $OFILE |
100  sort -s -t : -k 3b - \
101  >$TMPDIR/o$$-$OBASE
102
103# Create a temporary file from the new file's interesting section.
104sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
105  -e '/^[A-Z]+:/!d' \
106  -e '/^(WARNING|ERROR):/d' \
107  -e 's/\r$//' \
108  -e 's/^/N:/' \
109  $NFILE |
110  sort -s -t : -k 3b - \
111  >$TMPDIR/n$$-$NBASE
112
113# Merge the two files, then compare adjacent lines.
114# Comparison is complicated by tests that may be run multiple times.
115# If that case, we assume that the order is the same in both files.
116cat <<EOF >compare-$$.awk
117BEGIN {
118    FS = ":"
119    queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
120    verbose = verbose + 0	# Make sure it's defined.
121}
122
123# FIFO circular queue
124function push(st, nm) {
125    queueN += 1; status[queueN] = st; name[queueN] = nm
126}
127function peek() {
128    result = 0
129    if (queueN >= queue1) result = queue1
130    return result
131}
132function drop() {
133    queue1 += 1
134    if (queue1 > queueN) { queue1 = 1; queueN = 0; }
135}
136
137function compare(st, nm) {
138    old = peek()
139    if (old == 0) {
140	# This new test wasn't run last time.
141	if(st == "FAIL" || st == "UNRESOLVED" || verbose >= 2) {
142	    # New test fails or we want all changes
143	    printf("NA->%s:%s\n", st, nm)
144	}
145    }
146    else {
147	# Compare this new test to the first queued old one.
148	if (verbose >= 4) {
149	    printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
150	     status[old], name[old], st, nm)
151	}
152        if (name[old] != nm) {
153	    # The old test wasn't run this time and
154	    # the new test wasn't run last time.
155	    if (verbose >= 2) {
156		printf("%s->NA:%s\n", status[old], name[old])
157		if (nm != "") printf("NA->%s:%s\n", st, nm)
158	    }
159	    drop()
160        }
161	else {
162	    notable = 0
163	    if (status[old] == st) {
164	        # Status of this test has not changed.
165		if (verbose >= 3) printf("%s:%s\n", st, nm)
166	    }
167	    else if(status[old] == "PASS" && st == "XFAIL") {
168	        if (verbose >= 1) notable = 1
169	    }
170	    else if(status[old] == "PASS" || st == "FAIL") {
171	        # Test did pass but doesn't now
172		# or didn't fail but does now.
173		notable = 1
174	    }
175	    else if(st == "PASS") {
176	        # Test didn't pass but does now.
177		if (verbose >= 1) notable = 1
178	    }
179	    else if(verbose >= 2) {
180	        # Miscellaneous status change.
181		notable = 1
182	    }
183	    if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
184	    drop()
185	}
186    }
187}
188
189/^O:/ {
190    while (old = peek()) {
191	if (name[old] == \$3) break;
192	# The queued test is no longer run.
193	compare("", "");
194    }
195    # Save this test for later comparison.
196    push(\$2, \$3)
197}
198
199/^N:/ {
200    compare(\$2, \$3)
201}
202
203END {
204    while (old = peek()) compare("", "")
205}
206EOF
207sort -m -s -t : -k 3b $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE |
208 awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
209
210# Delete the temporary files.
211rm -f compare-$$.awk $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE
212
213exit 0
214