1#!/bin/sh
2eval 'exec perl -S -x -w $0 ${1+"$@"}'
3#!perl
4
5#
6# compareSimulationResults.pl: program to do a toleranced comparison of compact model simulation results
7#
8#  Rel  Date            Who             Comments
9# ====  ==========      =============   ========
10#  1.0  04/13/06        Colin McAndrew  Initial version
11#
12
13sub usage() {
14    print "
15$prog: compare simulation results between two files
16
17Usage: $prog [options] refFile simFile
18
19Files:
20    refFile                reference results file
21    simFile                simulated results file
22
23Options:
24    -c CLIP                match numbers n1 and n2 with abs(n1)<CLIP and abs(n2)<CLIP
25    -n NDIGIT              match numbers n1 and n2 if they are within 1 of the NDIGITth digit
26    -r REL                 match any numbers n1 and n2 with abs(n1-n2)/(0.5*(abs(n1)+abs(n2)+abs(n1-n2)))<REL
27    -d                     debug mode
28    -h                     print this help message
29    -i                     print info on file formats and structure
30    -v                     verbose mode
31";
32} # End of usage
33
34sub info() {
35    print "
36This program numerically compares simulation results and returns a string
37that indicates the result of the comparision. Possible return values are:
38ERROR: cannot open file nameOfFile
39FAIL        (probably from some simulation failure)
40FAIL        (simulation output quantities differ)
41FAIL        (number of results is different)
42FAIL        (non-numeric results)
43DIFFER      (max rel error is relErr)
44MATCH       (within specified tolerances)
45MATCH       (exact)
46
47It is expected that each file is a columnar list of simulation results,
48with the first line being a title line that indicates column contents,
49and every other line being numerical simulation results.
50
51The comparisons are done in the order
52exact
53clip
54nDigits
55relErr
56and passing one test means the results are considered to be the same.
57
58Different tolerancing should be used for different types of simulations,
59and because of this mixing different types of simulation results in
60the one file is not recommended. Reasonable tolerances are:
61Quantity       Clip        nDigtis     relTol
62DC current     1.0e-13     6           1.0e-06
63AC conductance 1.0e-20     6           1.0e-06
64   capacitance 1.0e-20     6           1.0e-06
65noise          1.0e-30     5           1.0e-05
66Note that these numbers should be adjusted based on the precision
67to which the numbers to be compared are printed.
68
69If no tolerances are specified, then the refFile name must contain
70one of the strings \"Dc\", \"Ac\", or \"Noise\" and the above values
71are used as default tolerances.
72
73The UNIX utilities spiff and ndiff so toleranced numerical comparisons,
74but they seem not generally available now. Hence this simplified
75numerical comparison program is provided for verifying simulation
76results against reference test results. Also, clipping and comparison
77to a certain number of digits are more relevant for comparison of
78numbers printed in output (as compared to results held in memory),
79and these are not considered in other toleranced numerical comparisons.
80";
81} # End of info
82
83#
84# Set program names and variables
85#
86
87$\="\n";
88$,=" ";
89$debug=0;
90$verbose=0;
91@prog=split("/",$0);
92$prog=$prog[$#prog];
93$number='[+-]?\d+[\.]?\d*[eE][+-]?\d+|[+-]?[\.]\d+[eE][+-]?\d+|[+-]?\d+[\.]?\d*|[+-]?[\.]\d+';
94
95for (;;) {
96    if (!defined($ARGV[0])) {
97        last;
98    } elsif ($ARGV[0]  =~ /^-c/i) {
99        shift(@ARGV);
100        die("ERROR: no clip value specified for -c option, stopped") if ($#ARGV < 0);
101        $clip=$ARGV[0];
102        die("ERROR: clip must be a positive number, stopped") if ($clip !~ /^$number$/ || $clip <= 0);
103    } elsif ($ARGV[0]  =~ /^-n/i) {
104        shift(@ARGV);
105        die("ERROR: no number of digits value specified for -n option, stopped") if ($#ARGV < 0);
106        $nDigits=$ARGV[0];
107        die("ERROR: nDigits must be a positive integer, stopped") if ($nDigits !~ /^[1-9][0-9]*$/);
108    } elsif ($ARGV[0]  =~ /^-r/i) {
109        shift(@ARGV);
110        die("ERROR: no relTol value specified for -r option, stopped") if ($#ARGV < 0);
111        $relTol=$ARGV[0];
112        die("ERROR: relTol must be a positive number, stopped") if ($relTol !~ /^$number$/ || $relTol <= 0);
113    } elsif ($ARGV[0]  =~ /^-d/i) {
114        $debug=1;$verbose=1;
115    } elsif ($ARGV[0] =~ /^-h/i) {
116        &usage();exit(0);
117    } elsif ($ARGV[0] =~ /^-i/i) {
118        &usage();&info();exit(0);
119    } elsif ($ARGV[0] =~ /^-v/i) {
120        $verbose=1;
121    } elsif ($ARGV[0] =~ /^-/) {
122        &usage();
123        die("ERROR: unknown flag $ARGV[0], stopped");
124    } else {
125        last;
126    }
127    shift(@ARGV);
128}
129if ($#ARGV<1) {
130    &usage();exit(0);
131}
132
133if (!defined($clip)) {
134    if ($ARGV[0] =~ /Dc/i) {
135        $clip=1.0e-13;
136    } elsif ($ARGV[0] =~ /Ac/i) {
137        $clip=1.0e-20;
138    } elsif ($ARGV[0] =~ /Noise/i) {
139        $clip=1.0e-30;
140    } else {
141        die("ERROR: must specify -c CLIP value if file is not Dc, Ac, or noise, stopped");
142    }
143}
144if (!defined($relTol)) {
145    if ($ARGV[0] =~ /Dc/i) {
146        $relTol=1.0e-6;
147    } elsif ($ARGV[0] =~ /Ac/i) {
148        $relTol=1.0e-6;
149    } elsif ($ARGV[0] =~ /Noise/i) {
150        $relTol=1.0e-5;
151    } else {
152        die("ERROR: must specify -r RELTOL value if file is not Dc, Ac, or noise, stopped");
153    }
154}
155if (!defined($nDigits)) {
156    if ($ARGV[0] =~ /Dc/i) {
157        $nDigits=6;
158    } elsif ($ARGV[0] =~ /Ac/i) {
159        $nDigits=6;
160    } elsif ($ARGV[0] =~ /Noise/i) {
161        $nDigits=5;
162    } else {
163        die("ERROR: must specify -n NDIGITS value if file is not Dc, Ac, or noise, stopped");
164    }
165}
166
167if ($ARGV[0] =~ /reference/i) {
168    $reference="reference";
169} else {
170    ($reference=$ARGV[0])=~s/^.*\.//;
171}
172($variant=$ARGV[1])=~s/^.*\.//;
173$result=&compareResults($ARGV[0],$ARGV[1],$clip,$nDigits,$relTol);
174printf("     variant: %-20s(compared to: %-9s) %s\n",$variant,$reference,$result);
175
176sub compareResults {
177    use strict;
178    my($refFile,$simFile,$clip,$nDigits,$relTol)=@_;
179    my(@Ref,@Sim,$i,$j,$relErr,$maxRelErr,$absErr,$maxAbsErr);
180    my(@RefRes,@SimRes,@ColNames,$matchType,$mag,$lo,$hi);
181
182    return("ERROR: cannot open file $refFile") if (!open(IF,"$refFile"));
183    while (<IF>) {chomp;push(@Ref,$_)}
184    close(IF);
185    return("ERROR: cannot open file $simFile") if (!open(IF,"$simFile"));
186    while (<IF>) {chomp;push(@Sim,$_)}
187    close(IF);
188    if ($main::verbose && $#Ref != $#Sim) {print STDERR "Reference points: $#Ref\nSimulated points: $#Sim"}
189    return("FAIL        (probably from some simulation failure)") if ($#Ref != $#Sim || $#Sim<1);
190    if ($main::verbose && $Ref[0] ne $Sim[0]) {print STDERR "Reference quantities: $Ref[0]\nSimulated quantities: $Sim[0]"}
191    return("FAIL        (simulation output quantities differ)")   if ($Ref[0] ne $Sim[0]);
192    $maxAbsErr=0;$maxRelErr=0;$matchType=0;
193    @ColNames=split(/\s+/,$Ref[0]);
194    for ($j=1;$j<=$#Ref;++$j) {
195        @RefRes=split(/\s+/,$Ref[$j]);
196        @SimRes=split(/\s+/,$Sim[$j]);
197        if ($main::verbose && $#RefRes != $#SimRes) {print STDERR "Line $j: Ref data: $#RefRes\tSim data: $#SimRes"}
198        return("FAIL        (number of quantities simulated are different)") if ($#RefRes != $#SimRes);
199        for ($i=1;$i<=$#RefRes;++$i) { # ignore first column, this is the sweep variable
200            if ($RefRes[$i] !~ /^$main::number$/ || $SimRes[$i] !~ /^$main::number$/) {
201                return("FAIL        (non-numeric results");
202            }
203            next if ($RefRes[$i] == $SimRes[$i]);
204            $matchType=1 if ($matchType<1);
205            next if (abs($RefRes[$i]) < $clip && abs($SimRes[$i]) < $clip);
206            #next if (abs($RefRes[$i]) < $clip || abs($SimRes[$i]) < $clip);
207            if ($RefRes[$i]*$SimRes[$i] <= 0.0) {
208                $matchType=2 if ($matchType<2);
209                $absErr=abs($RefRes[$i]-$SimRes[$i]);
210                $relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr));
211                $maxRelErr=$relErr if ($relErr > $maxRelErr);
212                if ($main::verbose) {print STDERR $ColNames[$i],$RefRes[$i],$SimRes[$i],100*$relErr."\%"}
213                next;
214            }
215            $lo=abs($RefRes[$i]);
216            if (abs($SimRes[$i]) < $lo) {
217                $hi=$lo;
218                $lo=abs($SimRes[$i]);
219            } else {
220                $hi=abs($SimRes[$i]);
221            }
222            $mag=int(log($lo)/log(10))+1;
223            if ($lo < 1) {$mag-=1}
224            $lo=int(0.5+$lo*10**($nDigits+1-$mag));
225            $hi=int(0.5+$hi*10**($nDigits+1-$mag));
226            next if (abs($lo-$hi)<=10);
227            $absErr=abs($RefRes[$i]-$SimRes[$i]);
228            $relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr));
229            next if ($relErr<$relTol);
230            if ($main::verbose) {print STDERR $ColNames[$i],$RefRes[$i],$SimRes[$i],100*$relErr."\%"}
231            $matchType=2 if ($matchType<2);
232            $maxRelErr=$relErr if ($relErr > $maxRelErr);
233        }
234    }
235    if ($matchType==0) {
236        return("MATCH       (exact)");
237    } elsif ($matchType==1) {
238        return("MATCH       (within specified tolerances)");
239    } elsif ($matchType==2) {
240        $mag=int(log($maxRelErr)/log(10));
241        $i=$maxRelErr/10**$mag;
242        $maxRelErr=100*10**$mag*int(0.5+1e3*$i)/1e3;
243        return("DIFFER      (max rel error is $maxRelErr\%)");
244    }
245}
246