1 /***************************************************************************
2 * blitz/benchext.cc Methods for Benchmarking class with external control.
3 *
4 * $Id$
5 *
6 * Copyright (C) 1997-2011 Todd Veldhuizen <tveldhui@acm.org>
7 *
8 * This file is a part of Blitz.
9 *
10 * Blitz is free software: you can redistribute it and/or modify
11 * it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation, either version 3
13 * of the License, or (at your option) any later version.
14 *
15 * Blitz is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Lesser General Public License for more details.
19 *
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with Blitz. If not, see <http://www.gnu.org/licenses/>.
22 *
23 * Suggestions: blitz-devel@lists.sourceforge.net
24 * Bugs: blitz-support@lists.sourceforge.net
25 *
26 * For more information, please see the Blitz++ Home Page:
27 * https://sourceforge.net/projects/blitz/
28 *
29 ****************************************************************************/
30 #ifndef BZ_BENCHEXT_CC
31 #define BZ_BENCHEXT_CC
32
33 #ifndef BZ_BENCHEXT_H
34 #error <blitz/benchext.cc> must be included via <blitz/benchext.h>
35 #endif
36
37 //#include <blitz/vector-et.h>
38
39 #ifdef BZ_HAVE_STD
40 #include <fstream>
41 #include <string>
42 #else
43 #include <fstream.h>
44 #include <string.h>
45 #endif
46
47 namespace blitz {
48
49 template<typename P_parameter>
BenchmarkExt(const char * name,int numImplementations)50 BenchmarkExt<P_parameter>::BenchmarkExt(const char* name,
51 int numImplementations)
52 {
53 BZPRECONDITION(numImplementations > 0);
54
55 description_ = name;
56 numImplementations_ = numImplementations;
57
58 implementationDescriptions_.resize(numImplementations);
59 parameterDescription_ = "Vector length";
60
61 // we don't really know this is the default, do we...
62 setDependentVariable("flops");
63
64 /*
65 // Set up default parameters and iterations
66 setNumParameters(19);
67
68 // NEEDS_WORK: once pow(X,Y) is supported, can just say
69 // parameters_ = pow(10.0, Range(1,20)/4.0);
70
71 for (unsigned i=0; i < numParameters_; ++i)
72 parameters_(i) = static_cast<P_parameter>(BZ_MATHFN_SCOPE(pow)(10.0, (i+1)/4.0));
73
74 iterations_ = 5.0e+5 / parameters_;
75 flopsPerIteration_ = parameters_;
76 */
77 // Set up initial state
78 state_ = initializing;
79 implementationNumber_ = 0;
80 }
81
82 template<typename P_parameter>
~BenchmarkExt()83 BenchmarkExt<P_parameter>::~BenchmarkExt()
84 {
85 }
86
87 template<typename P_parameter>
setNumParameters(int numParameters)88 void BenchmarkExt<P_parameter>::setNumParameters(int numParameters)
89 {
90 //BZPRECONDITION(state_ == initializing);
91
92 numParameters_ = numParameters;
93
94 parameters_.resize(numParameters_);
95 iterations_.resize(numParameters_);
96 flopsPerIteration_.resize(numParameters_);
97
98 // Set up timer and Mflops array
99 times_.resize(numImplementations_, numParameters_);
100 instr_.resize(numImplementations_, numParameters_);
101 flops_.resize(numImplementations_, numParameters_);
102 }
103
104 template<typename P_parameter>
setParameterVector(Array<P_parameter,1> parms)105 void BenchmarkExt<P_parameter>::setParameterVector(Array<P_parameter,1> parms)
106 {
107 BZPRECONDITION(state_ == initializing);
108 BZPRECONDITION(parms.size() == parameters_.size());
109
110 parameters_ = parms;
111 }
112
113 template<typename P_parameter>
setParameterDescription(const char * string)114 void BenchmarkExt<P_parameter>::setParameterDescription(const char* string)
115 {
116 parameterDescription_ = string;
117 }
118
119 template<typename P_parameter>
setIterations(Array<long,1> iters)120 void BenchmarkExt<P_parameter>::setIterations(Array<long,1> iters)
121 {
122 BZPRECONDITION(state_ == initializing);
123
124 iterations_ = iters;
125 }
126
127 template<typename P_parameter>
setOpsPerIteration(Array<double,1> flopsPerIteration)128 void BenchmarkExt<P_parameter>::setOpsPerIteration(Array<double,1>
129 flopsPerIteration)
130 {
131 BZPRECONDITION(flopsPerIteration_.size() == flopsPerIteration.size());
132
133 flopsPerIteration_ = flopsPerIteration;
134 }
135
136 /** Set the dependent variable of the measurements. If the independent
137 variable is seconds, we output "G<dvar>/s", if it is cycles, we
138 output "<dvar/c". */
139 template<typename P_parameter>
setDependentVariable(const char * dvar)140 void BenchmarkExt<P_parameter>::setDependentVariable(const char* dvar)
141 {
142 BZPRECONDITION(Timer::indep_var()!="");
143
144 if(timer_.indep_var()=="s") {
145 depvar_ = string("G")+dvar+"/s";
146 timerconversion_ = 1./1e9;
147 }
148 else if(timer_.indep_var()=="c") {
149 depvar_ = string(dvar)+"/c";
150 timerconversion_ = 1.;
151 }
152 else {
153 depvar_ = std::string(dvar)+"/"+timer_.indep_var();
154 timerconversion_ = 1;
155 }
156 }
157
158 template<typename P_parameter>
beginBenchmarking()159 void BenchmarkExt<P_parameter>::beginBenchmarking()
160 {
161 BZPRECONDITION(state_ == initializing);
162 state_ = benchmarking;
163 }
164
165 template<typename P_parameter>
beginImplementation(const char * description)166 void BenchmarkExt<P_parameter>::beginImplementation(const char* description)
167 {
168 // it would really be better if it worked for as many as we give it
169 BZPRECONDITION(implementationNumber_ < numImplementations_);
170 BZPRECONDITION(state_ == benchmarking);
171
172 implementationDescriptions_[implementationNumber_] = description;
173
174 state_ = benchmarkingImplementation;
175 parameterNumber_ = 0;
176 }
177
178 template<typename P_parameter>
doneImplementationBenchmark() const179 bool BenchmarkExt<P_parameter>::doneImplementationBenchmark() const
180 {
181 BZPRECONDITION(state_ == benchmarkingImplementation);
182 return parameterNumber_ == numParameters_;
183 }
184
185 template<typename P_parameter>
currentImplementation() const186 const string& BenchmarkExt<P_parameter>::currentImplementation() const
187 {
188 BZPRECONDITION(implementationNumber_ < numImplementations_);
189 return implementationDescriptions_[implementationNumber_];
190 }
191
192
193 template<typename P_parameter>
getParameter() const194 P_parameter BenchmarkExt<P_parameter>::getParameter() const
195 {
196 BZPRECONDITION(state_ == benchmarkingImplementation);
197 BZPRECONDITION(parameterNumber_ < numParameters_);
198
199 return parameters_(parameterNumber_);
200 }
201
202 template<typename P_parameter>
getIterations() const203 long BenchmarkExt<P_parameter>::getIterations() const
204 {
205 BZPRECONDITION(state_ == benchmarkingImplementation);
206 BZPRECONDITION(parameterNumber_ < numParameters_);
207
208 return iterations_(parameterNumber_);
209 }
210
211 template<typename P_parameter>
start()212 inline void BenchmarkExt<P_parameter>::start()
213 {
214 BZPRECONDITION(state_ == benchmarkingImplementation);
215 BZPRECONDITION(parameterNumber_ < numParameters_);
216 state_ = running;
217 timer_.start();
218 }
219
220 template<typename P_parameter>
stop()221 inline void BenchmarkExt<P_parameter>::stop()
222 {
223 timer_.stop();
224 BZPRECONDITION(state_ == running);
225 state_ = benchmarkingImplementation;
226
227 times_(int(implementationNumber_), int(parameterNumber_)) = timer_.elapsed();
228 instr_(int(implementationNumber_), int(parameterNumber_)) = timer_.instr();
229 flops_(int(implementationNumber_), int(parameterNumber_)) = timer_.flops();
230 ++parameterNumber_;
231 }
232
233 template<typename P_parameter>
startOverhead()234 inline void BenchmarkExt<P_parameter>::startOverhead()
235 {
236 BZPRECONDITION(state_ == benchmarkingImplementation);
237 BZPRECONDITION(parameterNumber_ > 0);
238 BZPRECONDITION(parameterNumber_ <= numParameters_);
239 state_ = runningOverhead;
240 timer_.start();
241 }
242
243 template<typename P_parameter>
stopOverhead()244 inline void BenchmarkExt<P_parameter>::stopOverhead()
245 {
246 BZPRECONDITION(state_ == runningOverhead);
247 timer_.stop();
248
249 cout << "\ttimer overhead: " <<
250 1.0*timer_.elapsed()/times_(int(implementationNumber_), int(parameterNumber_-1)) << endl;
251
252 times_(int(implementationNumber_), int(parameterNumber_-1)) -=
253 timer_.elapsed();
254 instr_(int(implementationNumber_), int(parameterNumber_-1)) -=
255 timer_.instr();
256 flops_(int(implementationNumber_), int(parameterNumber_-1)) -=
257 timer_.flops();
258
259 if(times_(int(implementationNumber_), int(parameterNumber_-1))<0) {
260 cerr << "\tError: Timer underflow in benchmark " << implementationDescriptions_[implementationNumber_] << " " << parameters_(parameterNumber_-1) << endl;
261 times_(int(implementationNumber_), int(parameterNumber_-1)) = blitz::huge(times_(0,0));
262 }
263 state_ = benchmarkingImplementation;
264 }
265
266 template<typename P_parameter>
skip()267 inline void BenchmarkExt<P_parameter>::skip()
268 {
269 BZPRECONDITION(state_ == benchmarkingImplementation);
270 BZPRECONDITION(parameterNumber_ < numParameters_);
271 times_(int(implementationNumber_), int(parameterNumber_)) = blitz::quiet_NaN(double());
272 ++parameterNumber_;
273 state_ = benchmarkingImplementation;
274 }
275
276 template<typename P_parameter>
endImplementation()277 void BenchmarkExt<P_parameter>::endImplementation()
278 {
279 BZPRECONDITION(state_ == benchmarkingImplementation);
280 BZPRECONDITION(parameterNumber_ == numParameters_);
281
282 ++implementationNumber_;
283
284 state_ = benchmarking;
285 }
286
287 template<typename P_parameter>
endBenchmarking()288 void BenchmarkExt<P_parameter>::endBenchmarking()
289 {
290 BZPRECONDITION(state_ == benchmarking);
291 BZPRECONDITION(implementationNumber_ == numImplementations_);
292
293 state_ = done;
294 }
295
296 template<typename P_parameter>
getMflops(unsigned implementation,unsigned parameterNum) const297 double BenchmarkExt<P_parameter>::getMflops(unsigned implementation,
298 unsigned parameterNum) const
299 {
300 BZPRECONDITION(state_ == done);
301 BZPRECONDITION(implementation < numImplementations_);
302 BZPRECONDITION(parameterNum < numParameters_);
303 return 1.0*iterations_(parameterNum) * flopsPerIteration_(parameterNum)
304 / times_(int(implementation), int(parameterNum)) * timerconversion_;
305 }
306
307 template<typename P_parameter>
getinstrperc(int implementation,int parameterNum) const308 double BenchmarkExt<P_parameter>::getinstrperc(int implementation,
309 int parameterNum) const
310 {
311 BZPRECONDITION(state_ == done);
312 BZPRECONDITION(implementation < numImplementations_);
313 BZPRECONDITION(parameterNum < numParameters_);
314 return 1.0*instr_(implementation,parameterNum)/
315 times_(int(implementation), int(parameterNum));
316 }
317 template<typename P_parameter>
getflopsperc(int implementation,int parameterNum) const318 double BenchmarkExt<P_parameter>::getflopsperc(int implementation,
319 int parameterNum) const
320 {
321 BZPRECONDITION(state_ == done);
322 BZPRECONDITION(implementation < numImplementations_);
323 BZPRECONDITION(parameterNum < numParameters_);
324 return 1.0*flops_(implementation,parameterNum)/
325 times_(int(implementation), int(parameterNum));
326 }
327
328 template<typename P_parameter>
saveMatlabGraph(const char * filename,const char * graphType) const329 void BenchmarkExt<P_parameter>::saveMatlabGraph(const char* filename, const char* graphType) const
330 {
331 BZPRECONDITION(state_ == done);
332
333 {
334 //ugly but saveMatlabGraph is coded into all benchmarks
335 std::string pyfn(filename);
336 pyfn=pyfn.replace(pyfn.find(".m"),2,std::string(".py"),0,3);
337 savePylabGraph(pyfn.c_str());
338 }
339
340 ofstream ofs(filename);
341
342 assert(ofs.good());
343
344 ofs << "% This matlab file generated automatically by class Benchmark"
345 << endl << "% of the Blitz++ class library." << endl << endl;
346
347 ofs.setf(ios::scientific);
348
349 // This will be a lot simpler once Matlab-style output formatting
350 // of vectors & matrices is finished.
351
352 // ofs << "parm = " << parameters_ << ";" << endl << endl;
353
354 ofs << "parm = [ ";
355 unsigned i;
356 for (i=0; i < numParameters_; ++i)
357 ofs << setprecision(12) << double(parameters_(i)) << " ";
358 ofs << "]; " << endl << endl;
359
360 ofs << "Mf = [ ";
361 for (i=0; i < numParameters_; ++i)
362 {
363 for (unsigned j=0; j < numImplementations_; ++j)
364 {
365 ofs << setprecision(12) << getMflops(j,i) << " ";
366 }
367 if (i != numParameters_ - 1)
368 ofs << ";" << endl;
369 }
370 ofs << "] ;" << endl << endl;
371
372 ofs << graphType << "(parm,Mf), title('" << description_ << "'), " << endl
373 << " xlabel('" << parameterDescription_ << "'), "
374 << "ylabel('" << depvar_ << "')\n"
375 << "legend(";
376
377 for (unsigned j=0; j < numImplementations_; ++j)
378 {
379 ofs << "'" << implementationDescriptions_[j] << "'";
380 if (j != numImplementations_ - 1)
381 ofs << ", ";
382 }
383
384 ofs << ")" << endl;
385 }
386
387
388 template<typename P_parameter>
savePylabGraph(const char * filename,const char * graphType) const389 void BenchmarkExt<P_parameter>::savePylabGraph(const char* filename, const char* graphType) const
390 {
391 BZPRECONDITION(state_ == done);
392
393 ofstream ofs(filename);
394
395 assert(ofs.good());
396
397 ofs << "# This python file generated automatically by class Benchmark\n"
398 << "# of the Blitz++ class library.\n"
399 << "from pylab import *\nfrom numpy import *\n"
400 << "clf()\n";
401
402 ofs.setf(ios::scientific);
403
404 ofs << "legnames=[";
405 for (unsigned j=0; j < numImplementations_; ++j)
406 {
407 ofs << "'" << implementationDescriptions_[j] << "'";
408 if (j != numImplementations_ - 1)
409 ofs << ", ";
410 }
411
412 ofs << "]\n\nparm = array([ ";
413 unsigned i;
414 for (i=0; i < numParameters_; ++i)
415 ofs << setprecision(12) << double(parameters_(i)) << ", ";
416 ofs << "])\n\n";
417
418 ofs << "Mf = array([[ ";
419 for (i=0; i < numParameters_; ++i)
420 {
421 if(i>0) ofs << ", [ ";
422 for (unsigned j=0; j < numImplementations_; ++j)
423 {
424 ofs << setprecision(12) << getMflops(j,i);
425 if(j<numImplementations_-1) ofs << ", ";
426 }
427 ofs << "]";
428 }
429 ofs << "])" << endl << endl;
430 #ifdef BZ_HAVE_LIBPAPI
431 // add i/c and flops counters
432 ofs << "ic = array([[ ";
433 for (i=0; i < numParameters_; ++i)
434 {
435 if(i>0) ofs << ", [ ";
436 for (unsigned j=0; j < numImplementations_; ++j)
437 {
438 ofs << setprecision(12) << getinstrperc(j,i);
439 if(j<numImplementations_-1) ofs << ", ";
440 }
441 ofs << "]";
442 }
443 ofs << "])" << endl << endl;
444 ofs << "fc = array([[ ";
445 for (i=0; i < numParameters_; ++i)
446 {
447 if(i>0) ofs << ", [ ";
448 for (unsigned j=0; j < numImplementations_; ++j)
449 {
450 ofs << setprecision(12) << getflopsperc(j,i);
451 if(j<numImplementations_-1) ofs << ", ";
452 }
453 ofs << "]";
454 }
455 ofs << "])" << endl << endl;
456 #endif
457
458 ofs << graphType << "(parm,Mf)\ntitle('" << description_ << "')\n"
459 << "xlabel('" << parameterDescription_ << "')\n"
460 << "ylabel('" << depvar_ << "')\n";
461
462 ofs << "legend(legnames)\n";
463 }
464
465 }
466
467 #endif // BZ_BENCHEXT_CC
468