1#!/bin/sh 2eval 'exec perl -S -x -w $0 ${1+"$@"}' 3#!perl 4 5# 6# compareSimulationResults.pl: program to do a toleranced comparison of compact model simulation results 7# 8# Rel Date Who Comments 9# ==== ========== ============= ======== 10# 1.0 04/13/06 Colin McAndrew Initial version 11# 12 13sub usage() { 14 print " 15$prog: compare simulation results between two files 16 17Usage: $prog [options] refFile simFile 18 19Files: 20 refFile reference results file 21 simFile simulated results file 22 23Options: 24 -c CLIP match numbers n1 and n2 with abs(n1)<CLIP and abs(n2)<CLIP 25 -n NDIGIT match numbers n1 and n2 if they are within 1 of the NDIGITth digit 26 -r REL match any numbers n1 and n2 with abs(n1-n2)/(0.5*(abs(n1)+abs(n2)+abs(n1-n2)))<REL 27 -d debug mode 28 -h print this help message 29 -i print info on file formats and structure 30 -v verbose mode 31"; 32} # End of usage 33 34sub info() { 35 print " 36This program numerically compares simulation results and returns a string 37that indicates the result of the comparision. Possible return values are: 38ERROR: cannot open file nameOfFile 39FAIL (probably from some simulation failure) 40FAIL (simulation output quantities differ) 41FAIL (number of results is different) 42FAIL (non-numeric results) 43DIFFER (max rel error is relErr) 44MATCH (within specified tolerances) 45MATCH (exact) 46 47It is expected that each file is a columnar list of simulation results, 48with the first line being a title line that indicates column contents, 49and every other line being numerical simulation results. 50 51The comparisons are done in the order 52exact 53clip 54nDigits 55relErr 56and passing one test means the results are considered to be the same. 57 58Different tolerancing should be used for different types of simulations, 59and because of this mixing different types of simulation results in 60the one file is not recommended. Reasonable tolerances are: 61Quantity Clip nDigtis relTol 62DC current 1.0e-13 6 1.0e-06 63AC conductance 1.0e-20 6 1.0e-06 64 capacitance 1.0e-20 6 1.0e-06 65noise 1.0e-30 5 1.0e-05 66Note that these numbers should be adjusted based on the precision 67to which the numbers to be compared are printed. 68 69If no tolerances are specified, then the refFile name must contain 70one of the strings \"Dc\", \"Ac\", or \"Noise\" and the above values 71are used as default tolerances. 72 73The UNIX utilities spiff and ndiff so toleranced numerical comparisons, 74but they seem not generally available now. Hence this simplified 75numerical comparison program is provided for verifying simulation 76results against reference test results. Also, clipping and comparison 77to a certain number of digits are more relevant for comparison of 78numbers printed in output (as compared to results held in memory), 79and these are not considered in other toleranced numerical comparisons. 80"; 81} # End of info 82 83# 84# Set program names and variables 85# 86 87$\="\n"; 88$,=" "; 89$debug=0; 90$verbose=0; 91@prog=split("/",$0); 92$prog=$prog[$#prog]; 93$number='[+-]?\d+[\.]?\d*[eE][+-]?\d+|[+-]?[\.]\d+[eE][+-]?\d+|[+-]?\d+[\.]?\d*|[+-]?[\.]\d+'; 94 95for (;;) { 96 if (!defined($ARGV[0])) { 97 last; 98 } elsif ($ARGV[0] =~ /^-c/i) { 99 shift(@ARGV); 100 die("ERROR: no clip value specified for -c option, stopped") if ($#ARGV < 0); 101 $clip=$ARGV[0]; 102 die("ERROR: clip must be a positive number, stopped") if ($clip !~ /^$number$/ || $clip <= 0); 103 } elsif ($ARGV[0] =~ /^-n/i) { 104 shift(@ARGV); 105 die("ERROR: no number of digits value specified for -n option, stopped") if ($#ARGV < 0); 106 $nDigits=$ARGV[0]; 107 die("ERROR: nDigits must be a positive integer, stopped") if ($nDigits !~ /^[1-9][0-9]*$/); 108 } elsif ($ARGV[0] =~ /^-r/i) { 109 shift(@ARGV); 110 die("ERROR: no relTol value specified for -r option, stopped") if ($#ARGV < 0); 111 $relTol=$ARGV[0]; 112 die("ERROR: relTol must be a positive number, stopped") if ($relTol !~ /^$number$/ || $relTol <= 0); 113 } elsif ($ARGV[0] =~ /^-d/i) { 114 $debug=1;$verbose=1; 115 } elsif ($ARGV[0] =~ /^-h/i) { 116 &usage();exit(0); 117 } elsif ($ARGV[0] =~ /^-i/i) { 118 &usage();&info();exit(0); 119 } elsif ($ARGV[0] =~ /^-v/i) { 120 $verbose=1; 121 } elsif ($ARGV[0] =~ /^-/) { 122 &usage(); 123 die("ERROR: unknown flag $ARGV[0], stopped"); 124 } else { 125 last; 126 } 127 shift(@ARGV); 128} 129if ($#ARGV<1) { 130 &usage();exit(0); 131} 132 133if (!defined($clip)) { 134 if ($ARGV[0] =~ /Dc/i) { 135 $clip=1.0e-13; 136 } elsif ($ARGV[0] =~ /Ac/i) { 137 $clip=1.0e-20; 138 } elsif ($ARGV[0] =~ /Noise/i) { 139 $clip=1.0e-30; 140 } else { 141 die("ERROR: must specify -c CLIP value if file is not Dc, Ac, or noise, stopped"); 142 } 143} 144if (!defined($relTol)) { 145 if ($ARGV[0] =~ /Dc/i) { 146 $relTol=1.0e-6; 147 } elsif ($ARGV[0] =~ /Ac/i) { 148 $relTol=1.0e-6; 149 } elsif ($ARGV[0] =~ /Noise/i) { 150 $relTol=1.0e-5; 151 } else { 152 die("ERROR: must specify -r RELTOL value if file is not Dc, Ac, or noise, stopped"); 153 } 154} 155if (!defined($nDigits)) { 156 if ($ARGV[0] =~ /Dc/i) { 157 $nDigits=6; 158 } elsif ($ARGV[0] =~ /Ac/i) { 159 $nDigits=6; 160 } elsif ($ARGV[0] =~ /Noise/i) { 161 $nDigits=5; 162 } else { 163 die("ERROR: must specify -n NDIGITS value if file is not Dc, Ac, or noise, stopped"); 164 } 165} 166 167if ($ARGV[0] =~ /reference/i) { 168 $reference="reference"; 169} else { 170 ($reference=$ARGV[0])=~s/^.*\.//; 171} 172($variant=$ARGV[1])=~s/^.*\.//; 173$result=&compareResults($ARGV[0],$ARGV[1],$clip,$nDigits,$relTol); 174printf(" variant: %-20s(compared to: %-9s) %s\n",$variant,$reference,$result); 175 176sub compareResults { 177 use strict; 178 my($refFile,$simFile,$clip,$nDigits,$relTol)=@_; 179 my(@Ref,@Sim,$i,$j,$relErr,$maxRelErr,$absErr,$maxAbsErr); 180 my(@RefRes,@SimRes,@ColNames,$matchType,$mag,$lo,$hi); 181 182 return("ERROR: cannot open file $refFile") if (!open(IF,"$refFile")); 183 while (<IF>) {chomp;push(@Ref,$_)} 184 close(IF); 185 return("ERROR: cannot open file $simFile") if (!open(IF,"$simFile")); 186 while (<IF>) {chomp;push(@Sim,$_)} 187 close(IF); 188 if ($main::verbose && $#Ref != $#Sim) {print STDERR "Reference points: $#Ref\nSimulated points: $#Sim"} 189 return("FAIL (probably from some simulation failure)") if ($#Ref != $#Sim || $#Sim<1); 190 if ($main::verbose && $Ref[0] ne $Sim[0]) {print STDERR "Reference quantities: $Ref[0]\nSimulated quantities: $Sim[0]"} 191 return("FAIL (simulation output quantities differ)") if ($Ref[0] ne $Sim[0]); 192 $maxAbsErr=0;$maxRelErr=0;$matchType=0; 193 @ColNames=split(/\s+/,$Ref[0]); 194 for ($j=1;$j<=$#Ref;++$j) { 195 @RefRes=split(/\s+/,$Ref[$j]); 196 @SimRes=split(/\s+/,$Sim[$j]); 197 if ($main::verbose && $#RefRes != $#SimRes) {print STDERR "Line $j: Ref data: $#RefRes\tSim data: $#SimRes"} 198 return("FAIL (number of quantities simulated are different)") if ($#RefRes != $#SimRes); 199 for ($i=1;$i<=$#RefRes;++$i) { # ignore first column, this is the sweep variable 200 if ($RefRes[$i] !~ /^$main::number$/ || $SimRes[$i] !~ /^$main::number$/) { 201 return("FAIL (non-numeric results"); 202 } 203 next if ($RefRes[$i] == $SimRes[$i]); 204 $matchType=1 if ($matchType<1); 205 next if (abs($RefRes[$i]) < $clip && abs($SimRes[$i]) < $clip); 206 #next if (abs($RefRes[$i]) < $clip || abs($SimRes[$i]) < $clip); 207 if ($RefRes[$i]*$SimRes[$i] <= 0.0) { 208 $matchType=2 if ($matchType<2); 209 $absErr=abs($RefRes[$i]-$SimRes[$i]); 210 $relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr)); 211 $maxRelErr=$relErr if ($relErr > $maxRelErr); 212 if ($main::verbose) {print STDERR $ColNames[$i],$RefRes[$i],$SimRes[$i],100*$relErr."\%"} 213 next; 214 } 215 $lo=abs($RefRes[$i]); 216 if (abs($SimRes[$i]) < $lo) { 217 $hi=$lo; 218 $lo=abs($SimRes[$i]); 219 } else { 220 $hi=abs($SimRes[$i]); 221 } 222 $mag=int(log($lo)/log(10))+1; 223 if ($lo < 1) {$mag-=1} 224 $lo=int(0.5+$lo*10**($nDigits+1-$mag)); 225 $hi=int(0.5+$hi*10**($nDigits+1-$mag)); 226 next if (abs($lo-$hi)<=10); 227 $absErr=abs($RefRes[$i]-$SimRes[$i]); 228 $relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr)); 229 next if ($relErr<$relTol); 230 if ($main::verbose) {print STDERR $ColNames[$i],$RefRes[$i],$SimRes[$i],100*$relErr."\%"} 231 $matchType=2 if ($matchType<2); 232 $maxRelErr=$relErr if ($relErr > $maxRelErr); 233 } 234 } 235 if ($matchType==0) { 236 return("MATCH (exact)"); 237 } elsif ($matchType==1) { 238 return("MATCH (within specified tolerances)"); 239 } elsif ($matchType==2) { 240 $mag=int(log($maxRelErr)/log(10)); 241 $i=$maxRelErr/10**$mag; 242 $maxRelErr=100*10**$mag*int(0.5+1e3*$i)/1e3; 243 return("DIFFER (max rel error is $maxRelErr\%)"); 244 } 245} 246