1 #include "IODebugTrack.hpp"
2 #include "moab/Range.hpp"
3 #include <iostream>
4 #include <vector>
5 #include <assert.h>
6
7 #ifdef MOAB_HAVE_MPI
8 # include "moab_mpi.h"
9 #endif
10
11 const char PFX[] = ">>> ";
12
13 namespace moab {
14
IODebugTrack(bool enabled,const std::string & name,std::ostream & output_stream,unsigned long table_size)15 IODebugTrack::IODebugTrack( bool enabled,
16 const std::string& name,
17 std::ostream& output_stream,
18 unsigned long table_size )
19 : enableOutput(enabled),
20 tableName(name),
21 ostr(output_stream),
22 maxSize(table_size),
23 haveMPI(false)
24 {
25 #ifdef MOAB_HAVE_MPI
26 MPI_Comm_rank( MPI_COMM_WORLD, &mpiRank );
27 #else
28 mpiRank = 0;
29 #endif
30 }
31
32
IODebugTrack(bool enabled,const std::string & name,unsigned long table_size)33 IODebugTrack::IODebugTrack( bool enabled,
34 const std::string& name,
35 unsigned long table_size )
36 : enableOutput(enabled),
37 tableName(name),
38 ostr(std::cerr),
39 maxSize(table_size)
40 {
41 mpiRank = 0;
42 haveMPI = false;
43 #ifdef MOAB_HAVE_MPI
44 int have_init = 0;
45 MPI_Initialized(&have_init);
46 if (have_init) {
47 haveMPI = true;
48 MPI_Comm_rank( MPI_COMM_WORLD, &mpiRank );
49 }
50 #endif
51 }
52
~IODebugTrack()53 IODebugTrack::~IODebugTrack()
54 {
55 if (!enableOutput || mpiRank) // only root prints gap summary
56 return;
57
58 if (dataSet.empty()) {
59 ostr << PFX << tableName << " : No Data Written!!!!" << std::endl;
60 return;
61 }
62
63 std::list<DRange>::const_iterator i;
64 if (!maxSize) {
65 for (i = dataSet.begin(); i != dataSet.end(); ++i)
66 if (i->end >= maxSize)
67 maxSize = i->end + 1;
68 }
69 Range processed;
70 Range::iterator h = processed.begin();
71 bool wrote_zero = false;
72 for (i = dataSet.begin(); i != dataSet.end(); ++i) {
73 // ranges cannot contain zero
74 assert(i->begin <= i->end);
75 if (i->begin)
76 h = processed.insert( h, i->begin, i->end );
77 else {
78 wrote_zero = true;
79 if (i->end)
80 h = processed.insert( h, i->begin+1, i->end );
81 }
82 }
83
84 // ranges cannot contain zero
85 Range unprocessed;
86 if (maxSize > 1)
87 unprocessed.insert( 1, maxSize - 1 );
88 unprocessed = subtract( unprocessed, processed );
89 if (unprocessed.empty())
90 return;
91
92 Range::const_pair_iterator j;
93 for (j = unprocessed.const_pair_begin(); j != unprocessed.const_pair_end(); ++j) {
94 unsigned long b = j->first;
95 unsigned long e = j->second;
96 if (b == 1 && !wrote_zero)
97 b = 0;
98
99 ostr << PFX << tableName << " : range not read/written: ["
100 << b << "," << e << "]" << std::endl;
101 ostr.flush();
102 }
103 }
104
record_io(unsigned long begin,unsigned long count)105 void IODebugTrack::record_io( unsigned long begin, unsigned long count )
106 {
107 if (enableOutput && count) {
108 DRange ins = { begin, begin+count-1, static_cast<long unsigned>(mpiRank) };
109 record_io( ins );
110 }
111 }
112
record_io(DRange ins)113 void IODebugTrack::record_io( DRange ins )
114 {
115 if (!enableOutput)
116 return;
117
118 // only root should get non-local data
119 assert(!mpiRank || ins.rank == (unsigned)mpiRank);
120 assert( ins.begin <= ins.end );
121
122 // test for out-of-bounds write
123 if (maxSize && ins.end >= maxSize)
124 ostr << ": Out of bounds write on rank " << mpiRank
125 << ": [" << ins.begin << "," << ins.end << "] >= " << maxSize
126 << std::endl;
127
128 // test for overlap with all existing ranges
129 std::list<DRange>::iterator i;
130 for (i = dataSet.begin(); i != dataSet.end(); ++i) {
131 if (i->end >= ins.begin && i->begin <= ins.end) { // if overlap
132 ostr << PFX << tableName;
133 if (i->rank == ins.rank) {
134 if (mpiRank == (int)ins.rank)
135 ostr << ": Local overwrite on rank " << mpiRank;
136
137 // otherwise should have been logged on remote proc, do nothing here
138 }
139 else
140 ostr << ": Conflicting write for ranks " << i->rank << " and " << ins.rank;
141
142 ostr << ": [" << i->begin << "," << i->end << "] and [" << ins.begin
143 << "," << ins.end << "]" << std::endl;
144 ostr.flush();
145 }
146 }
147
148 dataSet.push_back( ins );
149 }
150
all_reduce()151 void IODebugTrack::all_reduce()
152 {
153 #ifdef MOAB_HAVE_MPI
154 if (!enableOutput || !haveMPI)
155 return;
156
157 int commsize;
158 MPI_Comm_size( MPI_COMM_WORLD, &commsize);
159 int count = 3*dataSet.size();
160 std::vector<int> displs(commsize), counts(commsize);
161 MPI_Gather( &count, 1, MPI_INT,
162 &counts[0], 1, MPI_INT,
163 0, MPI_COMM_WORLD );
164 displs[0] = 0;
165 for (int i = 1; i < commsize; ++i)
166 displs[i] = displs[i-1] + counts[i-1];
167 int total = (displs.back() + counts.back()) / 3;
168 count /= 3;
169
170 std::vector<DRange> send(dataSet.size()), recv(total);
171 std::copy( dataSet.begin(), dataSet.end(), send.begin() );
172 MPI_Gatherv( (void*)&send[0], 3*send.size(), MPI_UNSIGNED_LONG,
173 (void*)&recv[0], &counts[0], &displs[0], MPI_UNSIGNED_LONG,
174 0, MPI_COMM_WORLD );
175
176 if (0 == mpiRank) {
177 for (int i = count; i < total; ++i)
178 record_io( recv[i] );
179 }
180 else {
181 dataSet.clear();
182 }
183 #endif
184 }
185
186
187
188 } // namespace moab
189