1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
#!/bin/sh
eval 'exec perl -S -x -w $0 ${1+"$@"}'
#!perl
#
# compareSimulationResults.pl: program to do a toleranced comparison of compact model simulation results
#
# Rel Date Who Comments
# ==== ========== ============= ========
# 1.0 04/13/06 Colin McAndrew Initial version
#
sub usage() {
print "
$prog: compare simulation results between two files
Usage: $prog [options] refFile simFile
Files:
refFile reference results file
simFile simulated results file
Options:
-c CLIP match numbers n1 and n2 with abs(n1)<CLIP and abs(n2)<CLIP
-n NDIGIT match numbers n1 and n2 if they are within 1 of the NDIGITth digit
-r REL match any numbers n1 and n2 with abs(n1-n2)/(0.5*(abs(n1)+abs(n2)+abs(n1-n2)))<REL
-d debug mode
-h print this help message
-i print info on file formats and structure
-v verbose mode
";
} # End of usage
sub info() {
print "
This program numerically compares simulation results and returns a string
that indicates the result of the comparision. Possible return values are:
ERROR: cannot open file nameOfFile
FAIL (probably from some simulation failure)
FAIL (simulation output quantities differ)
FAIL (number of results is different)
FAIL (non-numeric results)
DIFFER (max rel error is relErr)
MATCH (within specified tolerances)
MATCH (exact)
It is expected that each file is a columnar list of simulation results,
with the first line being a title line that indicates column contents,
and every other line being numerical simulation results.
The comparisons are done in the order
exact
clip
nDigits
relErr
and passing one test means the results are considered to be the same.
Different tolerancing should be used for different types of simulations,
and because of this mixing different types of simulation results in
the one file is not recommended. Reasonable tolerances are:
Quantity Clip nDigtis relTol
DC current 1.0e-13 6 1.0e-06
AC conductance 1.0e-20 6 1.0e-06
capacitance 1.0e-20 6 1.0e-06
noise 1.0e-30 5 1.0e-05
Note that these numbers should be adjusted based on the precision
to which the numbers to be compared are printed.
If no tolerances are specified, then the refFile name must contain
one of the strings \"Dc\", \"Ac\", or \"Noise\" and the above values
are used as default tolerances.
The UNIX utilities spiff and ndiff so toleranced numerical comparisons,
but they seem not generally available now. Hence this simplified
numerical comparison program is provided for verifying simulation
results against reference test results. Also, clipping and comparison
to a certain number of digits are more relevant for comparison of
numbers printed in output (as compared to results held in memory),
and these are not considered in other toleranced numerical comparisons.
";
} # End of info
#
# Set program names and variables
#
$\="\n";
$,=" ";
$debug=0;
$verbose=0;
@prog=split("/",$0);
$prog=$prog[$#prog];
$number='[+-]?\d+[\.]?\d*[eE][+-]?\d+|[+-]?[\.]\d+[eE][+-]?\d+|[+-]?\d+[\.]?\d*|[+-]?[\.]\d+';
for (;;) {
if (!defined($ARGV[0])) {
last;
} elsif ($ARGV[0] =~ /^-c/i) {
shift(@ARGV);
die("ERROR: no clip value specified for -c option, stopped") if ($#ARGV < 0);
$clip=$ARGV[0];
die("ERROR: clip must be a positive number, stopped") if ($clip !~ /^$number$/ || $clip <= 0);
} elsif ($ARGV[0] =~ /^-n/i) {
shift(@ARGV);
die("ERROR: no number of digits value specified for -n option, stopped") if ($#ARGV < 0);
$nDigits=$ARGV[0];
die("ERROR: nDigits must be a positive integer, stopped") if ($nDigits !~ /^[1-9][0-9]*$/);
} elsif ($ARGV[0] =~ /^-r/i) {
shift(@ARGV);
die("ERROR: no relTol value specified for -r option, stopped") if ($#ARGV < 0);
$relTol=$ARGV[0];
die("ERROR: relTol must be a positive number, stopped") if ($relTol !~ /^$number$/ || $relTol <= 0);
} elsif ($ARGV[0] =~ /^-d/i) {
$debug=1;$verbose=1;
} elsif ($ARGV[0] =~ /^-h/i) {
&usage();exit(0);
} elsif ($ARGV[0] =~ /^-i/i) {
&usage();&info();exit(0);
} elsif ($ARGV[0] =~ /^-v/i) {
$verbose=1;
} elsif ($ARGV[0] =~ /^-/) {
&usage();
die("ERROR: unknown flag $ARGV[0], stopped");
} else {
last;
}
shift(@ARGV);
}
if ($#ARGV<1) {
&usage();exit(0);
}
if (!defined($clip)) {
if ($ARGV[0] =~ /Dc/i) {
$clip=1.0e-13;
} elsif ($ARGV[0] =~ /Ac/i) {
$clip=1.0e-20;
} elsif ($ARGV[0] =~ /Noise/i) {
$clip=1.0e-30;
} else {
die("ERROR: must specify -c CLIP value if file is not Dc, Ac, or noise, stopped");
}
}
if (!defined($relTol)) {
if ($ARGV[0] =~ /Dc/i) {
$relTol=1.0e-6;
} elsif ($ARGV[0] =~ /Ac/i) {
$relTol=1.0e-6;
} elsif ($ARGV[0] =~ /Noise/i) {
$relTol=1.0e-5;
} else {
die("ERROR: must specify -r RELTOL value if file is not Dc, Ac, or noise, stopped");
}
}
if (!defined($nDigits)) {
if ($ARGV[0] =~ /Dc/i) {
$nDigits=6;
} elsif ($ARGV[0] =~ /Ac/i) {
$nDigits=6;
} elsif ($ARGV[0] =~ /Noise/i) {
$nDigits=5;
} else {
die("ERROR: must specify -n NDIGITS value if file is not Dc, Ac, or noise, stopped");
}
}
if ($ARGV[0] =~ /reference/i) {
$reference="reference";
} else {
($reference=$ARGV[0])=~s/^.*\.//;
}
($variant=$ARGV[1])=~s/^.*\.//;
$result=&compareResults($ARGV[0],$ARGV[1],$clip,$nDigits,$relTol);
printf(" variant: %-20s(compared to: %-9s) %s\n",$variant,$reference,$result);
sub compareResults {
use strict;
my($refFile,$simFile,$clip,$nDigits,$relTol)=@_;
my(@Ref,@Sim,$i,$j,$relErr,$maxRelErr,$absErr,$maxAbsErr);
my(@RefRes,@SimRes,$matchType,$mag,$lo,$hi);
return("ERROR: cannot open file $refFile") if (!open(IF,"$refFile"));
while (<IF>) {
s/\s+$//;
push(@Ref,$_);
}
close(IF);
return("ERROR: cannot open file $simFile") if (!open(IF,"$simFile"));
while (<IF>) {
s/\s+$//;
push(@Sim,$_);
}
close(IF);
return("FAIL (probably from some simulation failure)") if ($#Ref != $#Sim || $#Sim<1);
return("FAIL (simulation output quantities differ)") if ($Ref[0] ne $Sim[0]);
$maxAbsErr=0;$maxRelErr=0;$matchType=0;
for ($j=1;$j<=$#Ref;++$j) {
@RefRes=split(/\s+/,$Ref[$j]);
@SimRes=split(/\s+/,$Sim[$j]);
return("FAIL (number of quantities simulated are different") if ($#RefRes != $#SimRes);
for ($i=1;$i<=$#RefRes;++$i) { # ignore first column, this is the sweep variable
if ($RefRes[$i] !~ /^$main::number$/ || $SimRes[$i] !~ /^$main::number$/) {
return("FAIL (non-numeric results");
}
next if ($RefRes[$i] == $SimRes[$i]);
$matchType=1 if ($matchType<1);
#next if (abs($RefRes[$i]) < $clip && abs($SimRes[$i]) < $clip);
next if (abs($RefRes[$i]) < $clip || abs($SimRes[$i]) < $clip);
if ($RefRes[$i]*$SimRes[$i] <= 0.0) {
$matchType=2 if ($matchType<2);
$absErr=abs($RefRes[$i]-$SimRes[$i]);
$relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr));
$maxRelErr=$relErr if ($relErr > $maxRelErr);
if ($main::verbose) {print STDERR $RefRes[$i],$SimRes[$i],100*$relErr."\%"}
next;
}
$lo=abs($RefRes[$i]);
if (abs($SimRes[$i]) < $lo) {
$hi=$lo;
$lo=abs($SimRes[$i]);
} else {
$hi=abs($SimRes[$i]);
}
$mag=int(log($lo)/log(10))+1;
if ($lo < 1) {$mag-=1}
$lo=int(0.5+$lo*10**($nDigits+1-$mag));
$hi=int(0.5+$hi*10**($nDigits+1-$mag));
next if (abs($lo-$hi)<=10);
$absErr=abs($RefRes[$i]-$SimRes[$i]);
$relErr=$absErr/(0.5*(abs($RefRes[$i])+abs($SimRes[$i])+$absErr));
next if ($relErr<$relTol);
if ($main::verbose) {print STDERR $RefRes[$i],$SimRes[$i],100*$relErr."\%"}
$matchType=2 if ($matchType<2);
$maxRelErr=$relErr if ($relErr > $maxRelErr);
}
}
if ($matchType==0) {
return("MATCH (exact)");
} elsif ($matchType==1) {
return("MATCH (within specified tolerances)");
} elsif ($matchType==2) {
$mag=int(log($maxRelErr)/log(10));
$i=$maxRelErr/10**$mag;
$maxRelErr=100*10**$mag*int(0.5+1e3*$i)/1e3;
return("DIFFER (max rel error is $maxRelErr\%)");
}
}
|
