1 //////////////////////////////////////////////////////////////////////////////////////
2 // This file is distributed under the University of Illinois/NCSA Open Source License.
3 // See LICENSE file in top directory for details.
4 //
5 // Copyright (c) 2016 Jeongnim Kim and QMCPACK developers.
6 //
7 // File developed by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
8 // Cynthia Gu, zg1@ornl.gov, Oak Ridge National Laboratory
9 // Norbert Podhorszki, pnorbert@ornl.gov, Oak Ridge National Laboratory
10 // Paul R. C. Kent, kentpr@ornl.gov, Oak Ridge National Laboratory
11 // Jaron T. Krogel, krogeljt@ornl.gov, Oak Ridge National Laboratory
12 // Mark Dewing, markdewing@gmail.com, University of Illinois at Urbana-Champaign
13 // Mark A. Berrill, berrillma@ornl.gov, Oak Ridge National Laboratory
14 //
15 // File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
16 //////////////////////////////////////////////////////////////////////////////////////
17
18
19 #ifndef QMCPLUSPLUS_TRACEMANAGER_H
20 #define QMCPLUSPLUS_TRACEMANAGER_H
21
22
23 #if !defined(DISABLE_TRACEMANAGER)
24
25
26 #include <Configuration.h>
27 #include "OhmmsData/OhmmsElementBase.h"
28 #include "OhmmsData/AttributeSet.h"
29 #include "OhmmsPETE/OhmmsArray.h"
30 #include "Particle/ParticleSet.h"
31 #include "Utilities/IteratorUtility.h"
32 #include "Message/Communicate.h"
33 #include "hdf/hdf_archive.h"
34 #include "Message/OpenMP.h"
35 #include <map>
36 #include <set>
37 #include <algorithm>
38
39 namespace qmcplusplus
40 {
41 //#define TRACE_CHECK
42
43 const unsigned int DMAX = 4;
44 typedef long TraceInt;
45 typedef OHMMS_PRECISION TraceReal;
46 typedef std::complex<TraceReal> TraceComp;
47
48
49 struct TraceQuantity
50 {
51 std::string name;
52 bool default_quantity;
53 bool combined_quantity;
54 bool scalar_available;
55 bool array_available;
56 bool scalar_stream_requested;
57 bool array_stream_requested;
58 bool scalar_write_requested;
59 bool array_write_requested;
60 bool stream_scalar;
61 bool stream_array;
62 bool write_scalar;
63 bool write_array;
64
TraceQuantityTraceQuantity65 inline TraceQuantity()
66 {
67 default_quantity = false;
68 combined_quantity = false;
69 scalar_available = false;
70 array_available = false;
71 scalar_stream_requested = false;
72 array_stream_requested = false;
73 scalar_write_requested = false;
74 array_write_requested = false;
75 stream_scalar = false;
76 stream_array = false;
77 write_scalar = false;
78 write_array = false;
79 }
80
incorporateTraceQuantity81 inline void incorporate(const TraceQuantity& other)
82 {
83 if (name != other.name)
84 {
85 APP_ABORT("TraceQuantity::incorporate\n cannot merge quantities with differing names\n names: " + name + " " +
86 other.name);
87 }
88 default_quantity |= other.default_quantity;
89 combined_quantity |= other.combined_quantity;
90 scalar_available |= other.scalar_available;
91 array_available |= other.array_available;
92 scalar_stream_requested |= other.scalar_stream_requested;
93 array_stream_requested |= other.array_stream_requested;
94 }
95 };
96
97
98 //means of control of traces
99 // which quantities are available, streaming, writing, etc
100 // handles global (user + all observable) trace request
101 //medium of exchange of trace information between trace manager and observables
102 // handles individual trace requests from observables
103 struct TraceRequest
104 {
105 //switch to allow or disallow streams/writes of requested quantities
106 bool allow_streams;
107 bool allow_writes;
108
109 //switch to allow or disallow scalar/array streaming/writing
110 bool scalars_on;
111 bool arrays_on;
112
113 //all scalar/array quantities should be provided for streaming
114 bool stream_all_scalars;
115 bool stream_all_arrays;
116
117 //all scalar/array quantities should be provided for writing
118 bool write_all_scalars;
119 bool write_all_arrays;
120
121 //records whether default scalars/arrays should stream/write
122 bool streaming_default_scalars;
123 bool streaming_default_arrays;
124 bool writing_default_scalars;
125 bool writing_default_arrays;
126
127
128 //quantities made available or requested for streaming or writing
129 std::map<std::string, TraceQuantity> quantities;
130
131 //dependency lists for combined quantities
132 std::map<std::string, std::set<std::string>> combined_dependencies;
133
134 //used to screen checked out quantities for writing
135 std::string scalar_domain;
136
TraceRequestTraceRequest137 inline TraceRequest() { reset(); }
138
resetTraceRequest139 inline void reset()
140 {
141 allow_streams = false;
142 allow_writes = false;
143 scalars_on = false;
144 arrays_on = false;
145 stream_all_scalars = false;
146 stream_all_arrays = false;
147 write_all_scalars = false;
148 write_all_arrays = false;
149 streaming_default_scalars = false;
150 streaming_default_arrays = false;
151 writing_default_scalars = false;
152 writing_default_arrays = false;
153
154 quantities.clear();
155 combined_dependencies.clear();
156 }
157
set_scalar_domainTraceRequest158 inline void set_scalar_domain(const std::string& domain) { scalar_domain = domain; }
159
screen_sampleTraceRequest160 inline bool screen_sample(const std::string& domain, const std::string& name, bool& write)
161 {
162 bool scalar = domain == scalar_domain;
163 bool present = quantities.find(name) != quantities.end();
164 bool stream = false;
165 write = false;
166 if (present)
167 {
168 TraceQuantity& q = quantities[name];
169 if (scalar)
170 {
171 stream = q.stream_scalar;
172 write = q.write_scalar;
173 }
174 else
175 {
176 stream = q.stream_array;
177 write = q.write_array;
178 }
179 }
180 return stream;
181 }
182
183 //Contributor API (OperatorBase and others)
184 //declare that scalars are available for a quantity
185 inline void contribute_scalar(const std::string& name, bool default_quantity = false)
186 {
187 guarantee_presence(name);
188 quantities[name].scalar_available = true;
189 if (default_quantity)
190 quantities[name].default_quantity = true;
191 }
192
193 //declare that arrays are available for a quantity
194 inline void contribute_array(const std::string& name, bool default_quantity = false)
195 {
196 guarantee_presence(name);
197 quantities[name].array_available = true;
198 if (default_quantity)
199 quantities[name].default_quantity = true;
200 }
201
202 //declare that a combined quantity could be made available
203 inline void contribute_combined(const std::string& name,
204 std::vector<std::string>& deps,
205 bool scalar = false,
206 bool array = false,
207 bool default_quantity = false)
208 {
209 guarantee_presence(name, true);
210 TraceQuantity& q = quantities[name];
211 q.combined_quantity = true;
212 q.scalar_available = scalar;
213 q.array_available = array;
214 if (default_quantity)
215 q.default_quantity = true;
216 //combined_dependencies[name] = deps;
217 }
218
219 //declare that a scalar quantity is desired for streaming
220 inline void request_scalar(const std::string& name, bool write = false)
221 {
222 guarantee_presence(name);
223 quantities[name].scalar_stream_requested = true;
224 if (write)
225 quantities[name].scalar_write_requested = true;
226 }
227
228 //declare that a array quantity is desired for streaming
229 inline void request_array(const std::string& name, bool write = false)
230 {
231 guarantee_presence(name);
232 quantities[name].array_stream_requested = true;
233 if (write)
234 quantities[name].array_write_requested = true;
235 }
236
237 //query whether a scalar quantity should/will be streaming
streaming_scalarTraceRequest238 inline bool streaming_scalar(const std::string& name)
239 {
240 check_presence(name);
241 return quantities[name].stream_scalar;
242 }
243
244 //query whether a array quantity should/will be streaming
streaming_arrayTraceRequest245 inline bool streaming_array(const std::string& name)
246 {
247 check_presence(name);
248 return quantities[name].stream_array;
249 }
250
251 //query whether a quantity will be streaming in any fashion
streamingTraceRequest252 inline bool streaming(const std::string& name)
253 {
254 check_presence(name);
255 TraceQuantity& q = quantities[name];
256 return q.stream_scalar || q.stream_array;
257 }
258
259
260 //TraceManager API
261 //declare that scalar quantities are desired for streaming
262 inline void request_scalar(const std::set<std::string>& names, bool write = false)
263 {
264 std::set<std::string>::iterator name;
265 for (name = names.begin(); name != names.end(); ++name)
266 request_scalar(*name, write);
267 }
268
269 //declare that array quantities are desired for streaming
270 inline void request_array(const std::set<std::string>& names, bool write = false)
271 {
272 std::set<std::string>::iterator name;
273 for (name = names.begin(); name != names.end(); ++name)
274 request_array(*name, write);
275 }
276
277 //merge in all quantities from a contributor request
incorporateTraceRequest278 inline void incorporate(TraceRequest& other)
279 {
280 std::map<std::string, TraceQuantity>::iterator it;
281 for (it = other.quantities.begin(); it != other.quantities.end(); ++it)
282 {
283 const TraceQuantity& q = it->second;
284 if (quantity_present(q.name))
285 quantities[q.name].incorporate(q);
286 else
287 quantities[q.name] = q;
288 }
289 std::map<std::string, std::set<std::string>>::iterator d;
290 for (d = other.combined_dependencies.begin(); d != other.combined_dependencies.end(); ++d)
291 {
292 const std::string& name = d->first;
293 std::set<std::string>& deps = d->second;
294 if (combined_dependencies.find(name) != combined_dependencies.end())
295 combined_dependencies[name].insert(deps.begin(), deps.end());
296 else
297 combined_dependencies[name] = deps;
298 }
299 }
300
301 //balance requests with availability and turn streaming/writing on/off
determine_stream_writeTraceRequest302 inline void determine_stream_write()
303 {
304 std::map<std::string, TraceQuantity>::iterator it;
305 for (it = quantities.begin(); it != quantities.end(); ++it)
306 {
307 TraceQuantity& q = it->second;
308
309 q.stream_scalar =
310 allow_streams && scalars_on && q.scalar_available && (q.scalar_stream_requested || stream_all_scalars);
311
312 q.write_scalar = q.stream_scalar && (q.scalar_write_requested || write_all_scalars);
313
314 q.stream_array =
315 allow_streams && arrays_on && q.array_available && (q.array_stream_requested || stream_all_arrays);
316
317 q.write_array = q.stream_array && (q.array_write_requested || write_all_arrays);
318 }
319 // default quantities stream and write if any others do
320 streaming_default_scalars = false;
321 writing_default_scalars = false;
322 streaming_default_arrays = false;
323 writing_default_arrays = false;
324 for (it = quantities.begin(); it != quantities.end(); ++it)
325 {
326 TraceQuantity& q = it->second;
327 streaming_default_scalars |= q.stream_scalar;
328 writing_default_scalars |= q.write_scalar;
329 streaming_default_arrays |= q.stream_array;
330 writing_default_arrays |= q.write_array;
331 }
332 for (it = quantities.begin(); it != quantities.end(); ++it)
333 {
334 TraceQuantity& q = it->second;
335 if (q.default_quantity)
336 {
337 q.stream_scalar = streaming_default_scalars;
338 q.write_scalar = writing_default_scalars;
339 q.stream_array = streaming_default_arrays;
340 q.write_array = writing_default_arrays;
341 }
342 }
343 // if any combined quantities are streaming, their dependencies must also
344 for (it = quantities.begin(); it != quantities.end(); ++it)
345 {
346 TraceQuantity& q = it->second;
347 if (q.combined_quantity)
348 {
349 std::set<std::string>& deps = combined_dependencies[q.name];
350 std::set<std::string>::iterator name;
351 if (q.stream_scalar || q.stream_array)
352 {
353 for (name = deps.begin(); name != deps.end(); ++name)
354 {
355 check_presence(*name);
356 TraceQuantity& qd = quantities[*name];
357 qd.stream_scalar |= (qd.scalar_available && q.stream_scalar);
358 qd.stream_array |= (qd.array_available && q.stream_array);
359 }
360 }
361 }
362 }
363 combined_dependencies.clear();
364 }
365
366 //relay updated streaming information to contributor
relay_stream_infoTraceRequest367 inline void relay_stream_info(TraceRequest& other)
368 {
369 other.allow_streams = allow_streams;
370 other.allow_writes = allow_writes;
371 other.scalars_on = scalars_on;
372 other.arrays_on = arrays_on;
373 other.stream_all_scalars = stream_all_scalars;
374 other.stream_all_arrays = stream_all_arrays;
375 other.write_all_scalars = write_all_scalars;
376 other.write_all_arrays = write_all_arrays;
377 other.streaming_default_scalars = streaming_default_scalars;
378 other.streaming_default_arrays = streaming_default_arrays;
379 other.writing_default_scalars = writing_default_scalars;
380 other.writing_default_arrays = writing_default_arrays;
381 std::map<std::string, TraceQuantity>::iterator it;
382 for (it = other.quantities.begin(); it != other.quantities.end(); ++it)
383 {
384 TraceQuantity& q = it->second;
385 check_presence(q.name);
386 q = quantities[q.name];
387 }
388 other.combined_dependencies.clear();
389 }
390
reportTraceRequest391 inline void report()
392 {
393 //app_log()<<"\n TraceRequest"<< std::endl;
394 app_log() << " allow_streams = " << allow_streams << std::endl;
395 app_log() << " allow_writes = " << allow_writes << std::endl;
396 app_log() << " scalars_on = " << scalars_on << std::endl;
397 app_log() << " arrays_on = " << arrays_on << std::endl;
398 app_log() << " stream_all_scalars = " << stream_all_scalars << std::endl;
399 app_log() << " stream_all_arrays = " << stream_all_arrays << std::endl;
400 app_log() << " write_all_scalars = " << write_all_scalars << std::endl;
401 app_log() << " write_all_arrays = " << write_all_arrays << std::endl;
402 app_log() << " streaming_default_scalars = " << streaming_default_scalars << std::endl;
403 app_log() << " streaming_default_arrays = " << streaming_default_arrays << std::endl;
404 app_log() << " writing_default_scalars = " << writing_default_scalars << std::endl;
405 app_log() << " writing_default_arrays = " << writing_default_arrays << std::endl;
406
407 write_selected("scalars available", "scalar_available");
408 write_selected("arrays available", "array_available");
409
410 write_selected("scalar streams requested", "scalar_stream_requested");
411 write_selected("array streams requested", "array_stream_requested");
412
413 write_selected("scalar writes requested", "scalar_write_requested");
414 write_selected("array writes requested", "array_write_requested");
415
416 write_selected("scalar streams occurring", "stream_scalar");
417 write_selected("array streams occurring", "stream_array");
418
419 write_selected("scalar writes occurring", "write_scalar");
420 write_selected("array writes occurring", "write_array");
421 }
422
write_selectedTraceRequest423 inline void write_selected(const std::string& header, const std::string& selector)
424 {
425 app_log() << " " << header << ":";
426 int n = 0;
427 std::map<std::string, TraceQuantity>::iterator it;
428 for (it = quantities.begin(); it != quantities.end(); ++it)
429 {
430 TraceQuantity& q = it->second;
431 bool selected = false;
432 if (selector == "scalar_available")
433 selected = q.scalar_available;
434 else if (selector == "array_available")
435 selected = q.array_available;
436 else if (selector == "scalar_stream_requested")
437 selected = q.scalar_stream_requested;
438 else if (selector == "array_stream_requested")
439 selected = q.array_stream_requested;
440 else if (selector == "scalar_write_requested")
441 selected = q.scalar_write_requested;
442 else if (selector == "array_write_requested")
443 selected = q.array_write_requested;
444 else if (selector == "stream_scalar")
445 selected = q.stream_scalar;
446 else if (selector == "stream_array")
447 selected = q.stream_array;
448 else if (selector == "write_scalar")
449 selected = q.write_scalar;
450 else if (selector == "write_array")
451 selected = q.write_array;
452 else
453 APP_ABORT("TraceRequest::write_selected unrecognized selector: " + selector);
454 if (selected)
455 {
456 if (n % 5 == 0)
457 app_log() << std::endl << " ";
458 n++;
459 app_log() << " " << q.name;
460 }
461 }
462 app_log() << std::endl;
463 }
464
465
466 //private (internal) API
467 //query whether a quantity is present
quantity_presentTraceRequest468 inline bool quantity_present(const std::string& name) { return quantities.find(name) != quantities.end(); }
469
470 //create a quantity if it is not present
471 inline void guarantee_presence(const std::string& name, bool combined = false)
472 {
473 if (!quantity_present(name))
474 {
475 TraceQuantity q;
476 q.name = name;
477 quantities[name] = q;
478 }
479 if (combined)
480 if (combined_dependencies.find(name) == combined_dependencies.end())
481 {
482 std::set<std::string> stmp;
483 combined_dependencies[name] = stmp;
484 }
485 }
486
487 //abort if a quantity is not present
check_presenceTraceRequest488 inline void check_presence(const std::string& name)
489 {
490 if (!quantity_present(name))
491 {
492 APP_ABORT("TraceRequest::check_presence quantity " + name + " is not present");
493 }
494 }
495
496 //query whether any quantities are streaming
streamingTraceRequest497 inline bool streaming() { return streaming_default_scalars || streaming_default_arrays; }
498
499 //query whether any quantities are writing
writingTraceRequest500 inline bool writing() { return writing_default_scalars || writing_default_arrays; }
501
502 //query whether any scalar quantities are streaming
streaming_scalarsTraceRequest503 inline bool streaming_scalars() { return streaming_default_scalars; }
504
505 //query whether any array quantities are streaming
streaming_arraysTraceRequest506 inline bool streaming_arrays() { return streaming_default_arrays; }
507 };
508
509
510 template<typename T>
511 struct TraceSample
512 {
513 std::string domain;
514 std::string name;
515 int index;
516 bool array_trace;
517 int dimension;
518 int size;
519 int unit_size;
520 int data_size;
521 TinyVector<int, DMAX> shape;
522 std::vector<T>& sample;
523 bool write;
524 int buffer_start, buffer_end;
525 std::map<std::string, TraceInt> meta_int;
526 std::map<std::string, TraceReal> meta_real;
527 std::map<std::string, std::string> meta_string;
528 bool verbose;
529
TraceSampleTraceSample530 inline TraceSample(const std::string& sdomain,
531 const std::string& sname,
532 int sindex,
533 int sdim,
534 std::vector<T>& ssample)
535 : sample(ssample), verbose(false)
536 {
537 initialize(sdomain, sname, sindex, sdim);
538 }
539
540
TraceSampleTraceSample541 inline TraceSample(const std::string& sdomain,
542 const std::string& sname,
543 int sindex,
544 int sdim,
545 TinyVector<int, DMAX> sshape,
546 std::vector<T>& ssample)
547 : sample(ssample), verbose(false)
548 {
549 initialize(sdomain, sname, sindex, sdim);
550 shape = sshape;
551 size = sample.size();
552 check_shape();
553 }
554
555 inline virtual ~TraceSample() = default;
556
initializeTraceSample557 inline void initialize(const std::string& sdomain, const std::string& sname, int sindex, int sdim)
558 {
559 domain = sdomain, name = sname;
560 dimension = sdim;
561 index = sindex;
562 array_trace = false;
563 write = false;
564 buffer_start = -1;
565 buffer_end = -1;
566 }
567
568
set_unit_sizeTraceSample569 inline void set_unit_size(int usize) { unit_size = usize; }
570
571
set_data_sizeTraceSample572 inline void set_data_size() { data_size = size * unit_size; }
573
574
check_shapeTraceSample575 inline void check_shape()
576 {
577 bool correct_shape = true;
578 bool correct_dimension = dimension <= DMAX;
579 if (correct_dimension)
580 {
581 int tsize = 1;
582 for (int d = 0; d < dimension; ++d)
583 {
584 tsize *= shape[d];
585 correct_dimension = correct_dimension && shape[d] > 0;
586 }
587 correct_shape = tsize == size;
588 }
589 if (!correct_dimension)
590 APP_ABORT("TraceSample::check_shape dimension of sample array is incorrect");
591 if (!correct_shape)
592 APP_ABORT("TraceSample::check_shape shape and size of sample array do not match");
593 }
594
595
same_shapeTraceSample596 inline bool same_shape(TraceSample<T>* other)
597 {
598 bool same = dimension == other->dimension && size == other->size;
599 if (same)
600 for (int d = 0; d < dimension; ++d)
601 same = same && shape[d] == other->shape[d];
602 return same;
603 }
604
is_combinedTraceSample605 virtual bool is_combined() { return false; }
606
set_buffer_rangeTraceSample607 inline void set_buffer_range(int& bstart)
608 {
609 set_data_size();
610 if (write)
611 {
612 buffer_start = bstart;
613 buffer_end = bstart + data_size;
614 bstart = buffer_end;
615 }
616 }
617
618
sumTraceSample619 inline T sum()
620 {
621 T s(0);
622 for (int i = 0; i < sample.size(); ++i)
623 s += sample[i];
624 return s;
625 }
626
627 inline void write_summary(int ind = -1, std::string pad = " ")
628 {
629 std::string pad2 = pad + " ";
630 if (ind == -1)
631 app_log() << pad << " TraceSample " << name << std::endl;
632 else
633 app_log() << pad << ind << " TraceSample " << name << std::endl;
634 app_log() << pad2 << "domain = " << domain << std::endl;
635 app_log() << pad2 << "name = " << name << std::endl;
636 app_log() << pad2 << "index = " << index << std::endl;
637 app_log() << pad2 << "array_trace = " << array_trace << std::endl;
638 app_log() << pad2 << "dimension = " << dimension << std::endl;
639 app_log() << pad2 << "size = " << size << std::endl;
640 app_log() << pad2 << "unit_size = " << unit_size << std::endl;
641 app_log() << pad2 << "data_size = " << data_size << std::endl;
642 app_log() << pad2 << "shape = " << shape << std::endl;
643 app_log() << pad2 << "write = " << write << std::endl;
644 app_log() << pad2 << "buffer range = [" << buffer_start << "," << buffer_end << ")" << std::endl;
645 }
646 };
647
648
649 template<typename T>
650 struct CombinedTraceSample : public TraceSample<T>
651 {
652 bool combined;
653 std::vector<TraceReal> weights;
654 std::vector<TraceSample<T>*> components;
655
656
CombinedTraceSampleCombinedTraceSample657 inline CombinedTraceSample(const std::string& sdomain,
658 const std::string& sname,
659 int sindex,
660 int sdim,
661 std::vector<T>& ssample)
662 : TraceSample<T>(sdomain, sname, sindex, sdim, ssample)
663 {
664 reset();
665 }
666
667
CombinedTraceSampleCombinedTraceSample668 inline CombinedTraceSample(const std::string& sdomain,
669 const std::string& sname,
670 int sindex,
671 int sdim,
672 TinyVector<int, DMAX> sshape,
673 std::vector<T>& ssample)
674 : TraceSample<T>(sdomain, sname, sindex, sdim, sshape, ssample)
675 {
676 reset();
677 }
678
is_combinedCombinedTraceSample679 virtual bool is_combined() { return true; }
680
resetCombinedTraceSample681 inline void reset() { combined = false; }
682
683
add_componentCombinedTraceSample684 inline void add_component(TraceSample<T>* component, TraceReal weight)
685 {
686 if (components.size() == 0)
687 {
688 this->dimension = component->dimension;
689 this->size = component->size;
690 this->shape = component->shape;
691 this->data_size = component->data_size;
692 this->array_trace = component->array_trace;
693 this->sample.resize(component->size);
694 }
695 else if (!this->same_shape(component))
696 {
697 APP_ABORT("CombinedTraceSample::add_component attempted to add a different shape component\n my domain: " +
698 this->domain + "\n my name: " + this->name + "\n component domain: " + component->domain +
699 "\n component name: " + component->name);
700 }
701 else if (this->domain != component->domain)
702 APP_ABORT("CombinedTraceSample::add_component attempted to add a different domain component\n my domain: " +
703 this->domain + "\n my name: " + this->name + "\n component domain: " + component->domain +
704 "\n component name: " + component->name);
705 weights.push_back(weight);
706 components.push_back(component);
707 }
708
709
combineCombinedTraceSample710 inline void combine()
711 {
712 fill(this->sample.begin(), this->sample.end(), T(0));
713 for (int i = 0; i < components.size(); ++i)
714 {
715 T weight = weights[i];
716 std::vector<T>& component = components[i]->sample;
717 for (int j = 0; j < this->sample.size(); ++j)
718 this->sample[j] += weight * component[j];
719 }
720 combined = true;
721 }
722
723
724 inline void write_summary_combined(int ind, std::string pad = " ")
725 {
726 std::string pad2 = pad + " ";
727 std::string pad3 = pad2 + " ";
728 app_log() << pad << ind << " CombinedTraceSample " << this->name << std::endl;
729 app_log() << pad2 << "domain = " << this->domain << std::endl;
730 app_log() << pad2 << "ncomponents = " << components.size() << std::endl;
731 app_log() << pad2 << "components" << std::endl;
732 for (int i = 0; i < components.size(); ++i)
733 {
734 TraceSample<T>& c = *components[i];
735 app_log() << pad3 << c.name << " " << c.index << " " << weights[i] << std::endl;
736 }
737 app_log() << pad2 << "end components" << std::endl;
738 app_log() << pad2 << "vector address = " << (size_t) & this->sample << std::endl;
739 }
740 };
741
742
743 template<typename T>
TraceSample_comp(TraceSample<T> * left,TraceSample<T> * right)744 bool TraceSample_comp(TraceSample<T>* left, TraceSample<T>* right)
745 {
746 return left->data_size < right->data_size;
747 }
748
749
750 template<typename T>
751 struct TraceSamples
752 {
753 std::vector<TraceSample<T>*> samples;
754 std::map<std::string, std::map<std::string, int>> sample_indices;
755 std::vector<TraceSample<T>*> ordered_samples;
756 std::vector<CombinedTraceSample<T>*> combined_samples;
757 std::vector<std::vector<T>*> combined_sample_vectors;
758 bool verbose;
759
TraceSamplesTraceSamples760 inline TraceSamples() : verbose(false) {}
761
~TraceSamplesTraceSamples762 inline ~TraceSamples() { finalize(); }
763
764
set_verboseTraceSamples765 inline void set_verbose(bool v) { verbose = v; }
766
767
sizeTraceSamples768 inline int size() { return samples.size(); }
769
770
771 inline void assign_sample_index(const std::string& domain, const std::string& name, int index, std::string label = "")
772 {
773 if (sample_indices.count(domain) > 0 && sample_indices[domain].count(name) > 0)
774 {
775 APP_ABORT("TraceSamples::checkout " + label + " variable " + name + " already exists in domain " + domain);
776 }
777 else
778 {
779 sample_indices[domain][name] = index;
780 }
781 }
782
783 template<int D>
checkout_arrayTraceSamples784 inline Array<T, D>* checkout_array(const std::string& domain, const std::string& name, TinyVector<int, DMAX> shape)
785 {
786 int index = samples.size();
787 assign_sample_index(domain, name, index, "array");
788 Array<T, D>* a = new Array<T, D>(shape.data());
789 TraceSample<T>* s = new TraceSample<T>(domain, name, index, D, shape, a->storage());
790 samples.push_back(s);
791 if (verbose)
792 app_log() << "TraceSamples::checkout_array " << domain << " " << name << " " << index << std::endl;
793 return a;
794 }
795
796
797 template<int D>
checkout_arrayTraceSamples798 inline Array<T, D>* checkout_array(const ParticleSet& P, const std::string& name, TinyVector<int, DMAX> shape)
799 {
800 const std::string& domain = P.parentName();
801 int index = samples.size();
802 assign_sample_index(domain, name, index, "array");
803 Array<T, D>* a = new Array<T, D>(shape.data());
804 TraceSample<T>* s = new TraceSample<T>(domain, name, index, D, shape, a->storage());
805 samples.push_back(s);
806 s->array_trace = true;
807 if (verbose)
808 app_log() << "TraceSamples::checkout_array " << domain << " " << name << " " << index << std::endl;
809 return a;
810 }
811
812
get_traceTraceSamples813 inline TraceSample<T>* get_trace(const std::string& domain, const std::string& name)
814 {
815 TraceSample<T>* ts = NULL;
816 for (int i = 0; i < samples.size(); ++i)
817 {
818 TraceSample<T>& tsc = *samples[i];
819 if (tsc.domain == domain && tsc.name == name)
820 {
821 ts = &tsc;
822 break;
823 }
824 }
825 if (ts == NULL)
826 APP_ABORT("TraceSamples::get_trace failed to get trace for quantity " + name + " in domain " + domain);
827 return ts;
828 }
829
830
get_combined_traceTraceSamples831 inline CombinedTraceSample<T>* get_combined_trace(const std::string& domain, const std::string& name)
832 {
833 CombinedTraceSample<T>* ts = NULL;
834 for (int i = 0; i < combined_samples.size(); ++i)
835 {
836 CombinedTraceSample<T>& tsc = *combined_samples[i];
837 if (tsc.domain == domain && tsc.name == name)
838 {
839 ts = &tsc;
840 break;
841 }
842 }
843 if (ts == NULL)
844 APP_ABORT("TraceSamples::get_combined_trace failed to get trace for quantity " + name + " in domain " + domain);
845 return ts;
846 }
847
848
make_combined_traceTraceSamples849 inline bool make_combined_trace(const std::string& name,
850 std::vector<std::string>& names,
851 std::vector<TraceReal>& weights)
852 {
853 bool created = false;
854 if (names.size() != weights.size())
855 APP_ABORT("TraceSamples::make_combined_trace names and weights must be the same size");
856 std::map<std::string, std::map<std::string, int>>::iterator it;
857 for (it = sample_indices.begin(); it != sample_indices.end(); it++)
858 {
859 std::string domain = it->first;
860 std::map<std::string, int>& indices = it->second;
861 bool any_present = false;
862 for (int i = 0; i < names.size(); ++i)
863 any_present = any_present || indices.count(names[i]) > 0;
864 if (any_present)
865 {
866 int index = samples.size();
867 std::vector<T>* sample = new std::vector<T>;
868 CombinedTraceSample<T>* combined = new CombinedTraceSample<T>(domain, name, index, 0, *sample);
869 for (int i = 0; i < names.size(); ++i)
870 {
871 if (indices.count(names[i]) > 0)
872 {
873 TraceSample<T>* component = samples[indices[names[i]]];
874 combined->add_component(component, weights[i]);
875 }
876 }
877 assign_sample_index(domain, name, index);
878 samples.push_back(combined);
879 combined_samples.push_back(combined);
880 combined_sample_vectors.push_back(sample);
881 created = true;
882 }
883 }
884 return created;
885 }
886
887
set_unit_sizeTraceSamples888 inline void set_unit_size(int usize)
889 {
890 for (int i = 0; i < samples.size(); i++)
891 samples[i]->set_unit_size(usize);
892 }
893
894
screen_writesTraceSamples895 inline void screen_writes(TraceRequest& request)
896 {
897 for (int i = 0; i < samples.size(); i++)
898 {
899 TraceSample<T>& s = *samples[i];
900 bool stream = request.screen_sample(s.domain, s.name, s.write);
901 if (verbose)
902 app_log() << "TraceRequest screening " << s.name << " in domain " << s.domain << ". stream: " << stream
903 << " write: " << s.write << std::endl;
904 if (!stream && !s.is_combined())
905 app_log() << "warning: quantity " + s.name + " in domain " + s.domain +
906 " was not requested but is streaming anyway"
907 << std::endl;
908 }
909 }
910
911
order_by_sizeTraceSamples912 inline void order_by_size()
913 {
914 for (int i = 0; i < samples.size(); i++)
915 samples[i]->set_data_size();
916 ordered_samples.resize(samples.size());
917 copy(samples.begin(), samples.end(), ordered_samples.begin());
918 sort(ordered_samples.begin(), ordered_samples.end(), TraceSample_comp<T>);
919 }
920
921
set_buffer_rangesTraceSamples922 inline void set_buffer_ranges(int& starting_index)
923 {
924 for (int i = 0; i < ordered_samples.size(); i++)
925 {
926 TraceSample<T>& sample = *ordered_samples[i];
927 sample.set_buffer_range(starting_index);
928 }
929 }
930
931
total_sizeTraceSamples932 inline int total_size()
933 {
934 int s = 0;
935 for (int i = 0; i < samples.size(); i++)
936 s += samples[i]->sample.size() * samples[i]->unit_size;
937 return s;
938 }
939
940
min_buffer_indexTraceSamples941 inline int min_buffer_index()
942 {
943 int min_index = 2000000000;
944 for (int i = 0; i < samples.size(); i++)
945 min_index = std::min(min_index, samples[i]->buffer_start);
946 return min_index;
947 }
948
949
max_buffer_indexTraceSamples950 inline int max_buffer_index()
951 {
952 int max_index = -1;
953 for (int i = 0; i < samples.size(); i++)
954 max_index = std::max(max_index, samples[i]->buffer_end);
955 return max_index;
956 }
957
958
combine_samplesTraceSamples959 inline void combine_samples()
960 {
961 for (int i = 0; i < combined_samples.size(); ++i)
962 combined_samples[i]->combine();
963 }
964
965
reset_combined_samplesTraceSamples966 inline void reset_combined_samples()
967 {
968 for (int i = 0; i < combined_samples.size(); ++i)
969 combined_samples[i]->reset();
970 }
971
972
finalizeTraceSamples973 inline void finalize()
974 {
975 //note combined_samples pointers are a subset of those in samples
976 // and so only samples needs to be deleted
977 delete_iter(samples.begin(), samples.end());
978 delete_iter(combined_sample_vectors.begin(), combined_sample_vectors.end());
979 samples.resize(0);
980 ordered_samples.resize(0);
981 combined_samples.resize(0);
982 combined_sample_vectors.resize(0);
983 sample_indices.clear();
984 }
985
986
register_hdf_dataTraceSamples987 inline void register_hdf_data(hdf_archive& f)
988 {
989 std::map<std::string, std::map<std::string, int>>::iterator it;
990 std::map<std::string, int>::iterator it2;
991 for (it = sample_indices.begin(); it != sample_indices.end(); it++)
992 {
993 const std::string& domain = it->first;
994 std::map<std::string, int>& indices = it->second;
995 f.push(domain);
996 for (it2 = indices.begin(); it2 != indices.end(); ++it2)
997 {
998 const std::string& quantity = it2->first;
999 TraceSample<T>& sample = *samples[it2->second];
1000 if (sample.write)
1001 {
1002 f.push(quantity);
1003 f.write(sample.dimension, "dimension");
1004 f.write(sample.shape, "shape");
1005 f.write(sample.size, "size");
1006 f.write(sample.unit_size, "unit_size");
1007 f.write(sample.buffer_start, "row_start");
1008 f.write(sample.buffer_end, "row_end");
1009 f.pop();
1010 }
1011 }
1012 f.pop();
1013 }
1014 }
1015
1016
1017 inline void write_summary(std::string type, std::string pad = " ")
1018 {
1019 std::string pad2 = pad + " ";
1020 std::string pad3 = pad2 + " ";
1021 std::string pad4 = pad3 + " ";
1022 app_log() << pad << "TraceSamples<" << type << ">" << std::endl;
1023 app_log() << pad2 << "nsamples = " << samples.size() << std::endl;
1024 app_log() << pad2 << "ncombined_samples = " << combined_samples.size() << std::endl;
1025 app_log() << pad2 << "sample_indices" << std::endl;
1026 std::map<std::string, std::map<std::string, int>>::iterator it;
1027 std::map<std::string, int>::iterator it2;
1028 for (it = sample_indices.begin(); it != sample_indices.end(); it++)
1029 {
1030 const std::string& domain = it->first;
1031 std::map<std::string, int>& indices = it->second;
1032 app_log() << pad3 << domain << std::endl;
1033 for (it2 = indices.begin(); it2 != indices.end(); ++it2)
1034 app_log() << pad4 << it2->first << " = " << it2->second << std::endl;
1035 }
1036 app_log() << pad2 << "end sample_indices" << std::endl;
1037 app_log() << pad2 << "combined_sample_vectors = ";
1038 for (int i = 0; i < combined_sample_vectors.size(); ++i)
1039 app_log() << (size_t)combined_sample_vectors[i] << " ";
1040 app_log() << std::endl;
1041 app_log() << pad2 << "combined_samples" << std::endl;
1042 for (int i = 0; i < combined_samples.size(); ++i)
1043 combined_samples[i]->write_summary_combined(i, pad3);
1044 app_log() << pad2 << "end combined_samples" << std::endl;
1045 app_log() << pad2 << "samples" << std::endl;
1046 for (int i = 0; i < ordered_samples.size(); ++i)
1047 ordered_samples[i]->write_summary(i, pad3);
1048 //for(int i=0; i<samples.size(); ++i)
1049 // samples[i]->write_summary(i,pad3);
1050 app_log() << pad2 << "end samples" << std::endl;
1051 app_log() << pad << "end TraceSamples<" << type << ">" << std::endl;
1052 }
1053
1054
1055 inline void user_report(const std::string& type, const std::string& pad = " ")
1056 {
1057 std::string pad2 = pad + " ";
1058 app_log() << pad << type << " traces provided by estimators" << std::endl;
1059 std::map<std::string, std::map<std::string, int>>::iterator it;
1060 std::map<std::string, int>::iterator it2;
1061 for (it = sample_indices.begin(); it != sample_indices.end(); it++)
1062 {
1063 const std::string& domain = it->first;
1064 std::map<std::string, int>& indices = it->second;
1065 app_log() << pad2 << "domain " << domain << ": ";
1066 int n = 0;
1067 for (it2 = indices.begin(); it2 != indices.end(); ++it2)
1068 {
1069 if (n % 5 == 0)
1070 app_log() << std::endl << pad2 << " ";
1071 n++;
1072 const std::string& quantity = it2->first;
1073 app_log() << quantity << " ";
1074 }
1075 app_log() << std::endl;
1076 }
1077 }
1078 };
1079
1080
1081 template<typename T>
1082 struct TraceBuffer
1083 {
1084 bool has_complex;
1085 TraceSamples<T>* samples;
1086 TraceSamples<std::complex<T>>* complex_samples;
1087 std::string type;
1088 Array<T, 2> buffer;
1089 bool verbose;
1090
1091 //hdf variables
1092 std::string top;
1093 hsize_t dims[2];
1094 hsize_t hdf_file_pointer;
1095
1096
TraceBufferTraceBuffer1097 TraceBuffer() : samples(0), complex_samples(0), verbose(false)
1098 {
1099 type = "?";
1100 has_complex = false;
1101 reset();
1102 }
1103
1104
set_verboseTraceBuffer1105 inline void set_verbose(bool v) { verbose = v; }
1106
1107
set_typeTraceBuffer1108 inline void set_type(std::string stype)
1109 {
1110 type = stype;
1111 top = type + "_data";
1112 }
1113
1114
resetTraceBuffer1115 inline void reset() { buffer.resize(0, buffer.size(1)); }
1116
1117
set_samplesTraceBuffer1118 inline void set_samples(TraceSamples<T>& s) { samples = &s; }
1119
1120
set_samplesTraceBuffer1121 inline void set_samples(TraceSamples<std::complex<T>>& s)
1122 {
1123 complex_samples = &s;
1124 has_complex = true;
1125 }
1126
1127
make_combined_traceTraceBuffer1128 inline void make_combined_trace(const std::string& name,
1129 std::vector<std::string>& names,
1130 std::vector<TraceReal>& weights)
1131 {
1132 bool created_real = samples->make_combined_trace(name, names, weights);
1133 if (has_complex)
1134 {
1135 bool created_complex = complex_samples->make_combined_trace(name, names, weights);
1136 if (created_real && created_complex)
1137 APP_ABORT("TraceBuffer<" + type +
1138 ">::make_combined_trace\n cannot create real and complex combined traces for the same quantity\n "
1139 "attempted for quantity " +
1140 name);
1141 }
1142 }
1143
1144
order_and_resizeTraceBuffer1145 inline void order_and_resize()
1146 {
1147 //put the sample data in size order
1148 samples->set_unit_size(1);
1149 samples->order_by_size();
1150 if (has_complex)
1151 {
1152 complex_samples->set_unit_size(2);
1153 complex_samples->order_by_size();
1154 }
1155 //assign buffer ranges to each sample
1156 int sample_size = 0;
1157 samples->set_buffer_ranges(sample_size);
1158 if (has_complex)
1159 complex_samples->set_buffer_ranges(sample_size);
1160 #if defined(TRACE_CHECK)
1161 test_buffer_write(sample_size);
1162 #endif
1163 //resize the buffer
1164 int nsamples_init = 1;
1165 buffer.resize(nsamples_init, sample_size);
1166 }
1167
1168
same_asTraceBuffer1169 inline bool same_as(TraceBuffer<T>& ref) { return buffer.size(1) == ref.buffer.size(1); }
1170
1171
collect_sampleTraceBuffer1172 inline void collect_sample()
1173 {
1174 if (verbose)
1175 app_log() << " TraceBuffer<" << type << ">::collect_sample()" << std::endl;
1176 //make more room, if necessary
1177 int nrows = buffer.size(0);
1178 int row_size = buffer.size(1);
1179 if (row_size > 0)
1180 {
1181 //make space for the row, if necessary
1182 int current_row = nrows;
1183 nrows++;
1184 buffer.resize(nrows, row_size);
1185 if (verbose)
1186 app_log() << " increasing # of rows to " << nrows << std::endl;
1187 //combine samples
1188 samples->combine_samples();
1189 if (has_complex)
1190 complex_samples->combine_samples();
1191 //collect data from all samples into the buffer row
1192 int offset = current_row * row_size;
1193 {
1194 int boffset;
1195 std::vector<TraceSample<T>*>& ordered_samples = samples->ordered_samples;
1196 for (int s = 0; s < ordered_samples.size(); s++)
1197 {
1198 TraceSample<T>& tsample = *ordered_samples[s];
1199 if (tsample.write)
1200 {
1201 std::vector<T>& sample = tsample.sample;
1202 boffset = offset + tsample.buffer_start;
1203 for (int i = 0; i < sample.size(); ++i)
1204 {
1205 buffer(boffset + i) = sample[i];
1206 }
1207 }
1208 }
1209 }
1210 if (has_complex)
1211 {
1212 int boffset;
1213 std::vector<TraceSample<std::complex<T>>*>& ordered_samples = complex_samples->ordered_samples;
1214 for (int s = 0; s < ordered_samples.size(); s++)
1215 {
1216 TraceSample<std::complex<T>>& tsample = *ordered_samples[s];
1217 if (tsample.write)
1218 {
1219 std::vector<std::complex<T>>& sample = tsample.sample;
1220 boffset = offset + tsample.buffer_start;
1221 for (int i = 0, ib = 0; i < sample.size(); ++i, ib += 2)
1222 {
1223 buffer(boffset + ib) = sample[i].real();
1224 buffer(boffset + ib + 1) = sample[i].imag();
1225 }
1226 }
1227 }
1228 }
1229 //reset combined samples so they can be recombined on the next step
1230 samples->reset_combined_samples();
1231 if (has_complex)
1232 complex_samples->reset_combined_samples();
1233 #if defined(TRACE_CHECK)
1234 test_buffer_collect(current_row);
1235 #endif
1236 }
1237 }
1238
1239
writeTraceBuffer1240 inline void write() { APP_ABORT("TraceBuffer::write has not yet been implemented"); }
1241
1242
1243 inline void write_summary(std::string pad = " ")
1244 {
1245 std::string pad2 = pad + " ";
1246 app_log() << pad << "TraceBuffer<" << type << ">" << std::endl;
1247 app_log() << pad2 << "nrows = " << buffer.size(0) << std::endl;
1248 app_log() << pad2 << "row_size = " << buffer.size(1) << std::endl;
1249 app_log() << pad2 << "has_complex = " << has_complex << std::endl;
1250 samples->write_summary(type, pad2);
1251 if (has_complex)
1252 complex_samples->write_summary("complex " + type, pad2);
1253 app_log() << pad << "end TraceBuffer<" << type << ">" << std::endl;
1254 }
1255
1256
1257 inline void user_report(const std::string& pad = " ")
1258 {
1259 samples->user_report(type, pad);
1260 if (has_complex)
1261 complex_samples->user_report("complex " + type, pad);
1262 }
1263
register_hdf_dataTraceBuffer1264 inline void register_hdf_data(hdf_archive& f)
1265 {
1266 f.push(top);
1267 f.push("layout");
1268 samples->register_hdf_data(f);
1269 if (has_complex)
1270 complex_samples->register_hdf_data(f);
1271 f.pop();
1272 f.pop();
1273 if (!f.open_groups())
1274 APP_ABORT("TraceBuffer<" + type +
1275 ">::register_hdf_data() some hdf groups are still open at the end of registration");
1276 hdf_file_pointer = 0;
1277 }
1278
1279
write_hdfTraceBuffer1280 inline void write_hdf(hdf_archive& f) { write_hdf(f, hdf_file_pointer); }
1281
1282
write_hdfTraceBuffer1283 inline void write_hdf(hdf_archive& f, hsize_t& file_pointer)
1284 {
1285 if (verbose)
1286 app_log() << "TraceBuffer<" << type << ">::write_hdf() " << file_pointer << " " << buffer.size(0) << " "
1287 << buffer.size(1) << std::endl;
1288 dims[0] = buffer.size(0);
1289 dims[1] = buffer.size(1);
1290 if (dims[0] > 0)
1291 {
1292 f.push(top);
1293 h5d_append(f.top(), "traces", file_pointer, buffer.dim(), dims, buffer.data());
1294 f.pop();
1295 }
1296 f.flush();
1297 }
1298
1299
test_buffer_writeTraceBuffer1300 inline void test_buffer_write(int sample_size)
1301 {
1302 //check that the size is correct
1303 int ssize = samples->total_size();
1304 if (has_complex)
1305 ssize += complex_samples->total_size();
1306 if (sample_size != ssize)
1307 {
1308 app_log() << "sample_size = " << sample_size << "\ntotal_size = " << ssize << std::endl;
1309 APP_ABORT("TraceBuffer::test_buffer_write sample_size and total_size do not match");
1310 }
1311 //check that buffer indices fall in the expected range
1312 int nsamples = samples->size();
1313 int min_index = samples->min_buffer_index();
1314 int max_index = samples->max_buffer_index();
1315 if (has_complex)
1316 {
1317 nsamples += complex_samples->size();
1318 min_index = std::min(min_index, complex_samples->min_buffer_index());
1319 max_index = std::max(max_index, complex_samples->max_buffer_index());
1320 }
1321 if (nsamples > 0)
1322 {
1323 if (min_index != 0)
1324 APP_ABORT("TraceBuffer::test_buffer_write min_index!=0\n min_index=" << min_index);
1325 if (max_index != sample_size)
1326 APP_ABORT("TraceBuffer::test_buffer_write max_index!=sample_size");
1327 //check that no overlap exists in writes to buffer
1328 Array<int, 2> test_buffer;
1329 test_buffer.resize(1, sample_size);
1330 fill(test_buffer.begin(), test_buffer.end(), 0);
1331 int row = 0;
1332 int row_size = test_buffer.size(1);
1333 int offset = row * row_size;
1334 int* loc1 = &test_buffer(offset);
1335 int* loc2 = &test_buffer(row, 0);
1336 if (loc1 != loc2)
1337 APP_ABORT("TraceBuffer::test_buffer_write serialized buffer offset improperly computed");
1338 {
1339 int boffset;
1340 std::vector<TraceSample<T>*>& ordered_samples = samples->ordered_samples;
1341 for (int s = 0; s < ordered_samples.size(); s++)
1342 {
1343 TraceSample<T>& tsample = *ordered_samples[s];
1344 std::vector<T>& sample = tsample.sample;
1345 boffset = offset + tsample.buffer_start;
1346 for (int i = 0; i < sample.size(); ++i)
1347 test_buffer(boffset + i) = 1;
1348 }
1349 }
1350 if (has_complex)
1351 {
1352 int boffset;
1353 std::vector<TraceSample<std::complex<T>>*>& ordered_samples = complex_samples->ordered_samples;
1354 for (int s = 0; s < ordered_samples.size(); s++)
1355 {
1356 TraceSample<std::complex<T>>& tsample = *ordered_samples[s];
1357 std::vector<std::complex<T>>& sample = tsample.sample;
1358 boffset = offset + tsample.buffer_start;
1359 for (int i = 0, ib = 0; i < sample.size(); ++i, ib += tsample.unit_size)
1360 {
1361 test_buffer(boffset + ib) = 1;
1362 test_buffer(boffset + ib + 1) = 1;
1363 }
1364 }
1365 }
1366 //app_log()<<"test_buffer:"<< std::endl;
1367 //for(int i=0;i<row_size;++i)
1368 // app_log()<<" "<<i<<" "<<test_buffer(row,i)<< std::endl;
1369 for (int i = 0; i < row_size; ++i)
1370 if (!test_buffer(row, i))
1371 APP_ABORT("TraceBuffer::test_buffer_write write to row is not contiguous");
1372 }
1373 }
1374
1375
test_buffer_collectTraceBuffer1376 inline void test_buffer_collect(int current_row)
1377 {
1378 if (verbose)
1379 app_log() << "TraceBuffer::test_buffer_collect" << std::endl;
1380 std::string scalars = "scalars";
1381 std::map<std::string, std::map<std::string, int>>::iterator dom;
1382 std::map<std::string, int>::iterator var;
1383 std::map<std::string, std::map<std::string, int>>& domains = samples->sample_indices;
1384 std::vector<TraceSample<T>*>& tsamples = samples->samples;
1385 std::map<std::string, int>& scalar_vars = domains[scalars];
1386 for (var = scalar_vars.begin(); var != scalar_vars.end(); var++)
1387 {
1388 const std::string& name = var->first;
1389 TraceSample<T>& sample = *tsamples[var->second];
1390 T value = buffer(current_row, sample.buffer_start);
1391 T svalue = 0;
1392 bool any_present = false;
1393 for (dom = domains.begin(); dom != domains.end(); dom++)
1394 {
1395 const std::string& domain = dom->first;
1396 if (domain != scalars)
1397 {
1398 std::map<std::string, int>& vars = dom->second;
1399 if (vars.count(name) > 0)
1400 {
1401 any_present = true;
1402 TraceSample<T>& ssample = *tsamples[vars[name]];
1403 int start = ssample.buffer_start;
1404 int end = ssample.buffer_end;
1405 for (int i = start; i < end; i++)
1406 svalue += buffer(current_row, i);
1407 }
1408 }
1409 }
1410 if (any_present)
1411 {
1412 if (verbose)
1413 app_log() << " " << name << " " << value << " " << svalue << std::endl;
1414 }
1415 }
1416 }
1417 };
1418
1419
1420 class TraceManager
1421 {
1422 private:
1423 //collections of samples for a single walker step
1424 // the associated arrays will be updated following evaluate
1425 TraceSamples<TraceInt> int_samples;
1426 TraceSamples<TraceReal> real_samples;
1427 TraceSamples<TraceComp> comp_samples;
1428
1429 //buffers for storing samples
1430 // single row of buffer is a single sample from one walker
1431 // number of rows adjusts to accommodate walker samples
1432 TraceBuffer<TraceInt> int_buffer;
1433 TraceBuffer<TraceReal> real_buffer;
1434
1435 public:
1436 static double trace_tol;
1437
1438 TraceRequest request;
1439
1440 bool master_copy;
1441 std::string default_domain;
1442 bool method_allows_traces;
1443 bool streaming_traces;
1444 bool writing_traces;
1445 int throttle;
1446 bool verbose;
1447 std::string format;
1448 bool hdf_format;
1449 std::string file_root;
1450 Communicate* communicator;
1451 hdf_archive* hdf_file;
1452
verbose(false)1453 TraceManager(Communicate* comm = 0) : verbose(false), hdf_file(0)
1454 {
1455 reset_permissions();
1456 master_copy = true;
1457 communicator = comm;
1458 throttle = 1;
1459 format = "hdf";
1460 default_domain = "scalars";
1461 request.set_scalar_domain(default_domain);
1462 int_buffer.set_type("int");
1463 real_buffer.set_type("real");
1464 int_buffer.set_samples(int_samples);
1465 real_buffer.set_samples(real_samples);
1466 real_buffer.set_samples(comp_samples);
1467 }
1468
1469
makeClone()1470 inline TraceManager* makeClone()
1471 {
1472 if (verbose)
1473 app_log() << "TraceManager::makeClone " << master_copy << std::endl;
1474 if (!master_copy)
1475 APP_ABORT("TraceManager::makeClone only the master copy should call this function");
1476 TraceManager* tm = new TraceManager();
1477 tm->master_copy = false;
1478 tm->transfer_state_from(*this);
1479 tm->distribute();
1480 return tm;
1481 }
1482
1483
transfer_state_from(const TraceManager & tm)1484 inline void transfer_state_from(const TraceManager& tm)
1485 {
1486 method_allows_traces = tm.method_allows_traces;
1487 request = tm.request;
1488 streaming_traces = tm.streaming_traces;
1489 writing_traces = tm.writing_traces;
1490 throttle = tm.throttle;
1491 verbose = tm.verbose;
1492 format = tm.format;
1493 hdf_format = tm.hdf_format;
1494 default_domain = tm.default_domain;
1495 }
1496
1497
distribute()1498 inline void distribute()
1499 {
1500 int_samples.set_verbose(verbose);
1501 real_samples.set_verbose(verbose);
1502 comp_samples.set_verbose(verbose);
1503 int_buffer.set_verbose(verbose);
1504 real_buffer.set_verbose(verbose);
1505 }
1506
1507
reset_permissions()1508 inline void reset_permissions()
1509 {
1510 method_allows_traces = false;
1511 streaming_traces = false;
1512 writing_traces = false;
1513 verbose = false;
1514 hdf_format = false;
1515 request.reset();
1516 }
1517
1518
put(xmlNodePtr cur,bool allow_traces,std::string series_root)1519 inline void put(xmlNodePtr cur, bool allow_traces, std::string series_root)
1520 {
1521 reset_permissions();
1522 method_allows_traces = allow_traces;
1523 file_root = series_root;
1524 bool traces_requested = cur != NULL;
1525 streaming_traces = traces_requested && method_allows_traces;
1526 if (streaming_traces)
1527 {
1528 if (omp_get_thread_num() == 0)
1529 {
1530 app_log() << "\n TraceManager::put() " << master_copy << std::endl;
1531 app_log() << " traces requested : " << traces_requested << std::endl;
1532 app_log() << " method allows traces : " << method_allows_traces << std::endl;
1533 app_log() << " streaming traces : " << streaming_traces << std::endl;
1534 app_log() << std::endl;
1535 }
1536 //read trace attributes
1537 std::string writing = "yes";
1538 std::string scalar = "yes";
1539 std::string array = "yes";
1540 std::string scalar_defaults = "yes";
1541 std::string array_defaults = "yes";
1542 std::string verbose_write = "no";
1543 OhmmsAttributeSet attrib;
1544 attrib.add(writing, "write");
1545 attrib.add(scalar, "scalar");
1546 attrib.add(array, "array");
1547 attrib.add(scalar_defaults, "scalar_defaults");
1548 attrib.add(array_defaults, "array_defaults");
1549 attrib.add(format, "format");
1550 attrib.add(throttle, "throttle");
1551 attrib.add(verbose_write, "verbose");
1552 attrib.add(array, "particle"); //legacy
1553 attrib.add(array_defaults, "particle_defaults"); //legacy
1554 attrib.put(cur);
1555 writing_traces = writing == "yes";
1556 bool scalars_on = scalar == "yes";
1557 bool arrays_on = array == "yes";
1558 bool use_scalar_defaults = scalar_defaults == "yes";
1559 bool use_array_defaults = array_defaults == "yes";
1560 verbose = verbose_write == "yes";
1561 tolower(format);
1562 if (format == "hdf")
1563 {
1564 hdf_format = true;
1565 }
1566 else
1567 {
1568 APP_ABORT("TraceManager::put " + format + " is not a valid file format for traces\n valid options is: hdf");
1569 }
1570
1571 //read scalar and array elements
1572 // each requests that certain traces be computed
1573 std::set<std::string> scalar_requests;
1574 std::set<std::string> array_requests;
1575 xmlNodePtr element = cur->children;
1576 while (element != NULL)
1577 {
1578 std::string name((const char*)element->name);
1579 if (name == "scalar_traces")
1580 {
1581 std::string defaults = "no";
1582 OhmmsAttributeSet eattrib;
1583 eattrib.add(defaults, "defaults");
1584 eattrib.put(element);
1585 use_scalar_defaults = use_scalar_defaults && defaults == "yes";
1586 if (!use_scalar_defaults)
1587 {
1588 std::vector<std::string> scalar_list;
1589 putContent(scalar_list, element);
1590 scalar_requests.insert(scalar_list.begin(), scalar_list.end());
1591 }
1592 }
1593 else if (name == "array_traces" || name == "particle_traces")
1594 {
1595 std::string defaults = "no";
1596 OhmmsAttributeSet eattrib;
1597 eattrib.add(defaults, "defaults");
1598 eattrib.put(element);
1599 use_array_defaults = use_array_defaults && defaults == "yes";
1600 if (!use_array_defaults)
1601 {
1602 std::vector<std::string> array_list;
1603 putContent(array_list, element);
1604 array_requests.insert(array_list.begin(), array_list.end());
1605 }
1606 }
1607 else if (name != "text")
1608 {
1609 APP_ABORT("TraceManager::put " + name +
1610 " is not a valid sub-element of <trace/>\n valid options are: scalar_traces, array_traces");
1611 }
1612 element = element->next;
1613 }
1614
1615 writing_traces &= method_allows_traces;
1616
1617 //input user quantity requests into the traces request
1618 request.allow_streams = method_allows_traces;
1619 request.allow_writes = writing_traces;
1620 request.scalars_on = scalars_on;
1621 request.arrays_on = arrays_on;
1622 request.stream_all_scalars = use_scalar_defaults;
1623 request.stream_all_arrays = use_array_defaults;
1624 request.write_all_scalars = request.stream_all_scalars && writing_traces;
1625 request.write_all_arrays = request.stream_all_arrays && writing_traces;
1626 request.request_scalar(scalar_requests, writing_traces);
1627 request.request_array(array_requests, writing_traces);
1628
1629 //distribute verbosity level to buffer and sample objects
1630 distribute();
1631 }
1632
1633 //streaming_traces = false;
1634 //writing_traces = false;
1635 }
1636
1637
update_status()1638 inline void update_status()
1639 {
1640 streaming_traces = request.streaming();
1641 writing_traces = request.writing();
1642 }
1643
1644
screen_writes()1645 inline void screen_writes()
1646 {
1647 int_samples.screen_writes(request);
1648 real_samples.screen_writes(request);
1649 comp_samples.screen_writes(request);
1650 }
1651
initialize_traces()1652 inline void initialize_traces()
1653 {
1654 if (streaming_traces)
1655 {
1656 if (verbose)
1657 app_log() << "TraceManager::initialize_traces " << master_copy << std::endl;
1658 //organize trace samples and initialize buffers
1659 if (writing_traces)
1660 {
1661 int_buffer.order_and_resize();
1662 real_buffer.order_and_resize();
1663 }
1664 }
1665 }
1666
1667
finalize_traces()1668 inline void finalize_traces()
1669 {
1670 if (verbose)
1671 app_log() << "TraceManager::finalize_traces " << master_copy << std::endl;
1672 int_samples.finalize();
1673 real_samples.finalize();
1674 comp_samples.finalize();
1675 }
1676
1677
1678 //checkout functions to be used by any OperatorBase or Estimator
1679 // the array checked out should be updated during evaluate
1680 // object calling checkout is responsible for deleting the new array
1681
1682 // checkout integer arrays
1683 template<int D>
1684 inline Array<TraceInt, D>* checkout_int(const std::string& name, int n1 = 1, int n2 = 0, int n3 = 0, int n4 = 0)
1685 {
1686 return checkout_int<D>(default_domain, name, n1, n2, n3, n4);
1687 }
1688 template<int D>
1689 inline Array<TraceInt, D>* checkout_int(const std::string& domain,
1690 const std::string& name,
1691 int n1 = 1,
1692 int n2 = 0,
1693 int n3 = 0,
1694 int n4 = 0)
1695 {
1696 TinyVector<int, DMAX> shape(n1, n2, n3, n4);
1697 return int_samples.checkout_array<D>(domain, name, shape);
1698 }
1699 template<int D>
1700 inline Array<TraceInt, D>* checkout_int(const std::string& name,
1701 const ParticleSet& P,
1702 int n2 = 0,
1703 int n3 = 0,
1704 int n4 = 0)
1705 {
1706 TinyVector<int, DMAX> shape(P.getTotalNum(), n2, n3, n4);
1707 return int_samples.checkout_array<D>(P, name, shape);
1708 }
1709
1710
1711 // checkout real arrays
1712 template<int D>
1713 inline Array<TraceReal, D>* checkout_real(const std::string& name, int n1 = 1, int n2 = 0, int n3 = 0, int n4 = 0)
1714 {
1715 return checkout_real<D>(default_domain, name, n1, n2, n3, n4);
1716 }
1717 template<int D>
1718 inline Array<TraceReal, D>* checkout_real(const std::string& domain,
1719 const std::string& name,
1720 int n1 = 1,
1721 int n2 = 0,
1722 int n3 = 0,
1723 int n4 = 0)
1724 {
1725 TinyVector<int, DMAX> shape(n1, n2, n3, n4);
1726 return real_samples.checkout_array<D>(domain, name, shape);
1727 }
1728 template<int D>
1729 inline Array<TraceReal, D>* checkout_real(const std::string& name,
1730 const ParticleSet& P,
1731 int n2 = 0,
1732 int n3 = 0,
1733 int n4 = 0)
1734 {
1735 TinyVector<int, DMAX> shape(P.getTotalNum(), n2, n3, n4);
1736 return real_samples.checkout_array<D>(P, name, shape);
1737 }
1738
1739
1740 // checkout complex arrays
1741 template<int D>
1742 inline Array<TraceComp, D>* checkout_complex(const std::string& name, int n1 = 1, int n2 = 0, int n3 = 0, int n4 = 0)
1743 {
1744 return checkout_complex<D>(default_domain, name, n1, n2, n3, n4);
1745 }
1746 template<int D>
1747 inline Array<TraceComp, D>* checkout_complex(const std::string& domain,
1748 const std::string& name,
1749 int n1 = 1,
1750 int n2 = 0,
1751 int n3 = 0,
1752 int n4 = 0)
1753 {
1754 TinyVector<int, DMAX> shape(n1, n2, n3, n4);
1755 return comp_samples.checkout_array<D>(domain, name, shape);
1756 }
1757 template<int D>
1758 inline Array<TraceComp, D>* checkout_complex(const std::string& name,
1759 const ParticleSet& P,
1760 int n2 = 0,
1761 int n3 = 0,
1762 int n4 = 0)
1763 {
1764 TinyVector<int, DMAX> shape(P.getTotalNum(), n2, n3, n4);
1765 return comp_samples.checkout_array<D>(P, name, shape);
1766 }
1767
1768
1769 //get trace functions
1770 // used by estimators that require trace information
get_int_trace(const std::string & name)1771 inline TraceSample<TraceInt>* get_int_trace(const std::string& name) { return get_int_trace(default_domain, name); }
get_int_trace(const std::string & domain,const std::string & name)1772 inline TraceSample<TraceInt>* get_int_trace(const std::string& domain, const std::string& name)
1773 {
1774 return int_samples.get_trace(domain, name);
1775 }
get_int_trace(const ParticleSet & P,const std::string & name)1776 inline TraceSample<TraceInt>* get_int_trace(const ParticleSet& P, const std::string& name)
1777 {
1778 return int_samples.get_trace(P.parentName(), name);
1779 }
1780
get_real_trace(const std::string & name)1781 inline TraceSample<TraceReal>* get_real_trace(const std::string& name)
1782 {
1783 return get_real_trace(default_domain, name);
1784 }
get_real_trace(const std::string & domain,const std::string & name)1785 inline TraceSample<TraceReal>* get_real_trace(const std::string& domain, const std::string& name)
1786 {
1787 return real_samples.get_trace(domain, name);
1788 }
get_real_trace(const ParticleSet & P,const std::string & name)1789 inline TraceSample<TraceReal>* get_real_trace(const ParticleSet& P, const std::string& name)
1790 {
1791 return real_samples.get_trace(P.parentName(), name);
1792 }
1793
get_complex_trace(const std::string & name)1794 inline TraceSample<TraceComp>* get_complex_trace(const std::string& name)
1795 {
1796 return get_complex_trace(default_domain, name);
1797 }
get_complex_trace(const std::string & domain,const std::string & name)1798 inline TraceSample<TraceComp>* get_complex_trace(const std::string& domain, const std::string& name)
1799 {
1800 return comp_samples.get_trace(domain, name);
1801 }
get_complex_trace(const ParticleSet & P,const std::string & name)1802 inline TraceSample<TraceComp>* get_complex_trace(const ParticleSet& P, const std::string& name)
1803 {
1804 return comp_samples.get_trace(P.parentName(), name);
1805 }
1806
1807
get_int_combined_trace(const std::string & name)1808 inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const std::string& name)
1809 {
1810 return get_int_combined_trace(default_domain, name);
1811 }
get_int_combined_trace(const std::string & domain,const std::string & name)1812 inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const std::string& domain, const std::string& name)
1813 {
1814 return int_samples.get_combined_trace(domain, name);
1815 }
get_int_combined_trace(const ParticleSet & P,const std::string & name)1816 inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const ParticleSet& P, const std::string& name)
1817 {
1818 return int_samples.get_combined_trace(P.parentName(), name);
1819 }
1820
get_real_combined_trace(const std::string & name)1821 inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const std::string& name)
1822 {
1823 return get_real_combined_trace(default_domain, name);
1824 }
get_real_combined_trace(const std::string & domain,const std::string & name)1825 inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const std::string& domain, const std::string& name)
1826 {
1827 return real_samples.get_combined_trace(domain, name);
1828 }
get_real_combined_trace(const ParticleSet & P,const std::string & name)1829 inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const ParticleSet& P, const std::string& name)
1830 {
1831 return real_samples.get_combined_trace(P.parentName(), name);
1832 }
1833
get_complex_combined_trace(const std::string & name)1834 inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const std::string& name)
1835 {
1836 return get_complex_combined_trace(default_domain, name);
1837 }
get_complex_combined_trace(const std::string & domain,const std::string & name)1838 inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const std::string& domain, const std::string& name)
1839 {
1840 return comp_samples.get_combined_trace(domain, name);
1841 }
get_complex_combined_trace(const ParticleSet & P,const std::string & name)1842 inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const ParticleSet& P, const std::string& name)
1843 {
1844 return comp_samples.get_combined_trace(P.parentName(), name);
1845 }
1846
1847
1848 //create combined trace out of existing traces
make_combined_trace(const std::string & name,std::vector<std::string> & names)1849 inline void make_combined_trace(const std::string& name, std::vector<std::string>& names)
1850 {
1851 std::vector<TraceReal> weights;
1852 weights.resize(names.size());
1853 fill(weights.begin(), weights.end(), 1.0);
1854 make_combined_trace(name, names, weights);
1855 }
1856
make_combined_trace(const std::string & name,std::vector<std::string> & names,std::vector<TraceReal> & weights)1857 inline void make_combined_trace(const std::string& name,
1858 std::vector<std::string>& names,
1859 std::vector<TraceReal>& weights)
1860 {
1861 if (streaming_traces)
1862 {
1863 if (verbose)
1864 app_log() << "TraceManager::make_combined_trace " << master_copy << " " << name << std::endl;
1865 real_buffer.make_combined_trace(name, names, weights);
1866 }
1867 }
1868
1869
check_clones(std::vector<TraceManager * > & clones)1870 inline void check_clones(std::vector<TraceManager*>& clones)
1871 {
1872 if (writing_traces && clones.size() > 0)
1873 {
1874 if (verbose)
1875 app_log() << "TraceManager::check_clones" << std::endl;
1876 bool all_same = true;
1877 bool int_same;
1878 bool real_same;
1879 TraceManager& ref = *clones[0];
1880 for (int i = 0; i < clones.size(); ++i)
1881 {
1882 TraceManager& tm = *clones[i];
1883 int_same = tm.int_buffer.same_as(ref.int_buffer);
1884 real_same = tm.real_buffer.same_as(ref.real_buffer);
1885 all_same = all_same && int_same && real_same;
1886 }
1887 if (!all_same)
1888 {
1889 for (int i = 0; i < clones.size(); ++i)
1890 clones[i]->write_summary();
1891 APP_ABORT("TraceManager::check_clones trace buffer widths of clones do not match\n contiguous write is "
1892 "impossible\n this was first caused by clones contributing array traces from identical, but "
1893 "differently named, particlesets such as e, e2, e3 ... (fixed)\n please check the TraceManager "
1894 "summaries printed above");
1895 }
1896 }
1897 }
1898
1899
reset_buffers()1900 inline void reset_buffers()
1901 {
1902 if (writing_traces)
1903 {
1904 if (verbose)
1905 app_log() << "TraceManager::reset_buffers " << master_copy << std::endl;
1906 int_buffer.reset();
1907 real_buffer.reset();
1908 }
1909 }
1910
1911
1912 //store the full sample from a single walker step in buffers
buffer_sample(int current_step)1913 inline void buffer_sample(int current_step)
1914 {
1915 if (writing_traces && current_step % throttle == 0)
1916 {
1917 if (verbose)
1918 app_log() << " TraceManager::buffer_sample() " << master_copy << std::endl;
1919 int_buffer.collect_sample();
1920 real_buffer.collect_sample();
1921 }
1922 }
1923
1924
1925 //write buffered trace data to file
write_buffers(std::vector<TraceManager * > & clones,int block)1926 inline void write_buffers(std::vector<TraceManager*>& clones, int block)
1927 {
1928 if (master_copy)
1929 {
1930 if (writing_traces)
1931 {
1932 //double tstart = MPI_Wtime();
1933 if (verbose)
1934 app_log() << "TraceManager::write_buffers " << master_copy << std::endl;
1935 if (hdf_format)
1936 {
1937 write_buffers_hdf(clones);
1938 }
1939 }
1940 }
1941 else
1942 APP_ABORT("TraceManager::write_buffers should not be called from non-master copy");
1943 }
1944
1945
open_file(std::vector<TraceManager * > & clones)1946 inline void open_file(std::vector<TraceManager*>& clones)
1947 {
1948 if (master_copy)
1949 {
1950 if (writing_traces)
1951 {
1952 if (verbose)
1953 app_log() << "TraceManager::open_file " << master_copy << std::endl;
1954 if (verbose)
1955 clones[0]->write_summary();
1956 if (hdf_format)
1957 {
1958 open_hdf_file(clones);
1959 }
1960 }
1961 }
1962 else
1963 APP_ABORT("TraceManager::open_file should not be called from non-master copy");
1964 }
1965
1966
close_file()1967 inline void close_file()
1968 {
1969 if (master_copy)
1970 {
1971 if (writing_traces)
1972 {
1973 if (verbose)
1974 app_log() << "TraceManager::close_file " << master_copy << std::endl;
1975 if (hdf_format)
1976 {
1977 close_hdf_file();
1978 }
1979 }
1980 }
1981 else
1982 APP_ABORT("TraceManager::close_file should not be called from non-master copy");
1983 }
1984
1985
startRun(int blocks,std::vector<TraceManager * > & clones)1986 inline void startRun(int blocks, std::vector<TraceManager*>& clones)
1987 {
1988 if (verbose)
1989 app_log() << "TraceManager::startRun " << master_copy << std::endl;
1990 if (master_copy)
1991 {
1992 initialize_traces();
1993 check_clones(clones);
1994 open_file(clones);
1995 }
1996 else
1997 APP_ABORT("TraceManager::startRun should not be called from non-master copy");
1998 }
1999
2000
stopRun()2001 inline void stopRun()
2002 {
2003 if (verbose)
2004 app_log() << "TraceManager::stopRun " << master_copy << std::endl;
2005 if (master_copy)
2006 {
2007 close_file();
2008 finalize_traces();
2009 }
2010 else
2011 APP_ABORT("TraceManager::stopRun should not be called from non-master copy");
2012 }
2013
2014
startBlock(int nsteps)2015 inline void startBlock(int nsteps)
2016 {
2017 if (verbose)
2018 app_log() << "TraceManager::startBlock " << master_copy << std::endl;
2019 reset_buffers();
2020 }
2021
2022
stopBlock()2023 inline void stopBlock()
2024 {
2025 if (verbose)
2026 app_log() << "TraceManager::stopBlock " << master_copy << std::endl;
2027 }
2028
2029
2030 inline void write_summary(std::string pad = " ")
2031 {
2032 std::string pad2 = pad + " ";
2033 app_log() << std::endl;
2034 app_log() << pad << "TraceManager (detailed summary)" << std::endl;
2035 app_log() << pad2 << "master_copy = " << master_copy << std::endl;
2036 app_log() << pad2 << "method_allows_traces = " << method_allows_traces << std::endl;
2037 app_log() << pad2 << "streaming_traces = " << streaming_traces << std::endl;
2038 app_log() << pad2 << "writing_traces = " << writing_traces << std::endl;
2039 app_log() << pad2 << "format = " << format << std::endl;
2040 app_log() << pad2 << "hdf format = " << hdf_format << std::endl;
2041 app_log() << pad2 << "default_domain = " << default_domain << std::endl;
2042 int_buffer.write_summary(pad2);
2043 real_buffer.write_summary(pad2);
2044 app_log() << pad << "end TraceManager" << std::endl;
2045 }
2046
2047
2048 inline void user_report(std::string pad = " ")
2049 {
2050 std::string pad2 = pad + " ";
2051 std::string pad3 = pad2 + " ";
2052 app_log() << std::endl;
2053 app_log() << pad << "Traces report" << std::endl;
2054 request.report();
2055 app_log() << pad2 << "Type and domain breakdown of streaming quantities:" << std::endl;
2056 std::set<std::string>::iterator req;
2057 int_buffer.user_report(pad3);
2058 real_buffer.user_report(pad3);
2059 app_log() << std::endl;
2060 //if(verbose)
2061 // write_summary(pad);
2062 }
2063
2064 //hdf file operations
open_hdf_file(std::vector<TraceManager * > & clones)2065 inline void open_hdf_file(std::vector<TraceManager*>& clones)
2066 {
2067 if (clones.size() == 0)
2068 APP_ABORT("TraceManager::open_hdf_file no trace clones exist, cannot open file");
2069 int nprocs = communicator->size();
2070 int rank = communicator->rank();
2071 char ptoken[32];
2072 std::string file_name = file_root;
2073 if (nprocs > 1)
2074 {
2075 if (nprocs > 10000)
2076 sprintf(ptoken, ".p%05d", rank);
2077 else if (nprocs > 1000)
2078 sprintf(ptoken, ".p%04d", rank);
2079 else
2080 sprintf(ptoken, ".p%03d", rank);
2081 file_name += ptoken;
2082 }
2083 file_name += ".traces.h5";
2084 if (verbose)
2085 app_log() << "TraceManager::open_hdf_file opening traces hdf file " << file_name << std::endl;
2086 hdf_file = new hdf_archive(communicator, false);
2087 bool successful = hdf_file->create(file_name);
2088 if (!successful)
2089 APP_ABORT("TraceManager::open_hdf_file failed to open hdf file " + file_name);
2090 // only clones have active buffers and associated data
2091 TraceManager& tm = *clones[0];
2092 //tm.write_summary();
2093 tm.int_buffer.register_hdf_data(*hdf_file);
2094 tm.real_buffer.register_hdf_data(*hdf_file);
2095 }
2096
2097
write_buffers_hdf(std::vector<TraceManager * > & clones)2098 inline void write_buffers_hdf(std::vector<TraceManager*>& clones)
2099 {
2100 if (verbose)
2101 app_log() << "TraceManager::write_buffers_hdf " << master_copy << std::endl;
2102 for (int ip = 0; ip < clones.size(); ++ip)
2103 {
2104 TraceManager& tm = *clones[ip];
2105 tm.int_buffer.write_hdf(*hdf_file, int_buffer.hdf_file_pointer);
2106 tm.real_buffer.write_hdf(*hdf_file, real_buffer.hdf_file_pointer);
2107 }
2108 }
2109
close_hdf_file()2110 inline void close_hdf_file() { delete hdf_file; }
2111 };
2112
2113
2114 } // namespace qmcplusplus
2115
2116
2117 #else
2118
2119
2120 // make a vacuous class for TraceManager for lighter compilation
2121 // disabling TraceManager should not affect other runtime behavior
2122
2123 #include "Particle/ParticleSet.h"
2124
2125 namespace qmcplusplus
2126 {
2127 typedef long TraceInt;
2128 typedef double TraceReal;
2129 typedef std::complex<TraceReal> TraceComp;
2130
2131 struct TraceRequest
2132 {
2133 bool streaming_default_scalars;
2134
TraceRequestTraceRequest2135 inline TraceRequest() { streaming_default_scalars = false; }
2136
2137 //inline bool screen_sample(const std::string& domain,const std::string& name,bool& write) { return false; }
streaming_scalarTraceRequest2138 inline bool streaming_scalar(const std::string& name) { return false; }
streaming_arrayTraceRequest2139 inline bool streaming_array(const std::string& name) { return false; }
streamingTraceRequest2140 inline bool streaming(const std::string& name) { return false; }
2141 //inline bool quantity_present(const std::string& name) { return false; }
streamingTraceRequest2142 inline bool streaming() { return false; }
2143 //inline bool writing() { return false; }
2144 //inline bool streaming_scalars() { return false; }
2145 //inline bool streaming_arrays() { return false; }
2146
2147
2148 //inline void set_scalar_domain(const std::string& domain) { }
2149 inline void request_scalar(const std::string& name, bool write = false) {}
2150 inline void request_array(const std::string& name, bool write = false) {}
2151 //inline void request_scalar(const std::set<std::string>& names,bool write=false) { }
2152 //inline void request_array(const std::set<std::string>& names,bool write=false) { }
incorporateTraceRequest2153 inline void incorporate(TraceRequest& other) {}
determine_stream_writeTraceRequest2154 inline void determine_stream_write() {}
relay_stream_infoTraceRequest2155 inline void relay_stream_info(TraceRequest& other) {}
2156 //inline void report() { }
2157 //inline void write_selected(const std::string& header,const std::string& selector) { }
2158 //inline void guarantee_presence(const std::string& name,bool combined=false) { }
2159 //inline void check_presence(const std::string& name) { }
2160 inline void contribute_scalar(const std::string& name, bool default_quantity = false) {}
2161 inline void contribute_array(const std::string& name, bool default_quantity = false) {}
2162 inline void contribute_combined(const std::string& name,
2163 std::vector<std::string>& deps,
2164 bool scalar = false,
2165 bool array = false,
2166 bool default_quantity = false)
2167 {}
2168 };
2169
2170
2171 template<typename T>
2172 struct TraceSample
2173 {
2174 std::vector<T>& sample;
2175 };
2176
2177
2178 template<typename T>
2179 struct CombinedTraceSample : public TraceSample<T>
2180 {
combineCombinedTraceSample2181 inline void combine() {}
2182 };
2183
2184
2185 struct TraceManager
2186 {
2187 TraceRequest request;
2188 bool streaming_traces;
2189
2190
2191 TraceManager(Communicate* comm = 0) { streaming_traces = false; }
2192
makeCloneTraceManager2193 inline TraceManager* makeClone() { return new TraceManager(); }
2194
transfer_state_fromTraceManager2195 inline void transfer_state_from(const TraceManager& tm) {}
2196 //inline void distribute() { }
2197 //inline void reset_permissions() { }
putTraceManager2198 inline void put(xmlNodePtr cur, bool allow_traces, std::string series_root) {}
update_statusTraceManager2199 inline void update_status() {}
screen_writesTraceManager2200 inline void screen_writes() {}
initialize_tracesTraceManager2201 inline void initialize_traces() {}
finalize_tracesTraceManager2202 inline void finalize_traces() {}
2203
2204 template<int D>
2205 inline Array<TraceInt, D>* checkout_int(const std::string& name, int n1 = 1, int n2 = 0, int n3 = 0, int n4 = 0)
2206 {
2207 return 0;
2208 }
2209 //template<int D> inline Array<TraceInt,D>* checkout_int( const std::string& domain,const std::string& name, int n1=1,int n2=0,int n3=0,int n4=0) { return 0; }
2210 template<int D>
2211 inline Array<TraceInt, D>* checkout_int(const std::string& name,
2212 const ParticleSet& P,
2213 int n2 = 0,
2214 int n3 = 0,
2215 int n4 = 0)
2216 {
2217 return 0;
2218 }
2219 template<int D>
2220 inline Array<TraceReal, D>* checkout_real(const std::string& name, int n1 = 1, int n2 = 0, int n3 = 0, int n4 = 0)
2221 {
2222 return 0;
2223 }
2224 //template<int D> inline Array<TraceReal,D>* checkout_real( const std::string& domain,const std::string& name, int n1=1,int n2=0,int n3=0,int n4=0) { return 0; }
2225 template<int D>
2226 inline Array<TraceReal, D>* checkout_real(const std::string& name,
2227 const ParticleSet& P,
2228 int n2 = 0,
2229 int n3 = 0,
2230 int n4 = 0)
2231 {
2232 return 0;
2233 }
2234 //template<int D> inline Array<TraceComp,D>* checkout_complex(const std::string& name, int n1=1,int n2=0,int n3=0,int n4=0) { return 0; }
2235 //template<int D> inline Array<TraceComp,D>* checkout_complex(const std::string& domain,const std::string& name, int n1=1,int n2=0,int n3=0,int n4=0) { return 0; }
2236 template<int D>
2237 inline Array<TraceComp, D>* checkout_complex(const std::string& name,
2238 const ParticleSet& P,
2239 int n2 = 0,
2240 int n3 = 0,
2241 int n4 = 0)
2242 {
2243 return 0;
2244 }
2245
2246 //inline TraceSample<TraceInt>* get_int_trace(const std::string& name) { return 0; }
2247 //inline TraceSample<TraceInt>* get_int_trace(const std::string& domain, const std::string& name) { return 0; }
2248 //inline TraceSample<TraceInt>* get_int_trace(const ParticleSet& P, const std::string& name) { return 0; }
get_real_traceTraceManager2249 inline TraceSample<TraceReal>* get_real_trace(const std::string& name) { return 0; }
2250 //inline TraceSample<TraceReal>* get_real_trace(const std::string& domain, const std::string& name) { return 0; }
get_real_traceTraceManager2251 inline TraceSample<TraceReal>* get_real_trace(const ParticleSet& P, const std::string& name) { return 0; }
2252 //inline TraceSample<TraceComp>* get_complex_trace(const std::string& name) { return 0; }
2253 //inline TraceSample<TraceComp>* get_complex_trace(const std::string& domain, const std::string& name) { return 0; }
get_complex_traceTraceManager2254 inline TraceSample<TraceComp>* get_complex_trace(const ParticleSet& P, const std::string& name) { return 0; }
2255
2256 //inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const std::string& name) { return 0; }
2257 //inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const std::string& domain, const std::string& name) { return 0; }
2258 //inline CombinedTraceSample<TraceInt>* get_int_combined_trace(const ParticleSet& P, const std::string& name) { return 0; }
2259 //inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const std::string& name) { return 0; }
2260 //inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const std::string& domain, const std::string& name) { return 0; }
get_real_combined_traceTraceManager2261 inline CombinedTraceSample<TraceReal>* get_real_combined_trace(const ParticleSet& P, const std::string& name)
2262 {
2263 return 0;
2264 }
2265 //inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const std::string& name) { return 0; }
2266 //inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const std::string& domain, const std::string& name) { return 0; }
2267 //inline CombinedTraceSample<TraceComp>* get_complex_combined_trace(const ParticleSet& P, const std::string& name) { return 0; }
2268
make_combined_traceTraceManager2269 inline void make_combined_trace(const std::string& name, std::vector<std::string>& names) {}
2270 //inline void make_combined_trace(const std::string& name,std::vector<std::string>& names,std::vector<TraceReal>& weights) { }
2271 //inline void check_clones(std::vector<TraceManager*>& clones) { }
2272 //inline void reset_buffers() { }
buffer_sampleTraceManager2273 inline void buffer_sample(int current_step) {}
write_buffersTraceManager2274 inline void write_buffers(std::vector<TraceManager*>& clones, int block) {}
2275 //inline void open_file(std::vector<TraceManager*>& clones) { }
2276 //inline void close_file() { }
startRunTraceManager2277 inline void startRun(int blocks, std::vector<TraceManager*>& clones) {}
stopRunTraceManager2278 inline void stopRun() {}
startBlockTraceManager2279 inline void startBlock(int nsteps) {}
stopBlockTraceManager2280 inline void stopBlock() {}
2281 //inline void write_summary( std::string pad=" ") { }
2282 inline void user_report(std::string pad = " ") {}
2283 };
2284
2285
2286 } // namespace qmcplusplus
2287
2288 #endif
2289
2290
2291 #endif
2292