1 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 // vi: set et ts=4 sw=2 sts=2:
3 /****************************************************************************/
4 /* */
5 /* File: cplmsg.c */
6 /* */
7 /* Purpose: ddd object transfer: */
8 /* last messages in order to create coupling consistency. */
9 /* */
10 /* Author: Klaus Birken */
11 /* Rechenzentrum Uni Stuttgart */
12 /* Universitaet Stuttgart */
13 /* Allmandring 30 */
14 /* 70550 Stuttgart */
15 /* internet: birken@rus.uni-stuttgart.de */
16 /* */
17 /* History: 960712 kb created */
18 /* 960718 kb introduced lowcomm-layer (sets of messages) */
19 /* */
20 /* Remarks: */
21 /* */
22 /****************************************************************************/
23
24 /****************************************************************************/
25 /* */
26 /* include files */
27 /* system include files */
28 /* application include files */
29 /* */
30 /****************************************************************************/
31
32 /* standard C library */
33 #include <config.h>
34 #include <cstdlib>
35 #include <cstdio>
36
37 #include <forward_list>
38 #include <iomanip>
39 #include <iostream>
40 #include <sstream>
41
42 #include <dune/common/stdstreams.hh>
43
44 #include <dune/uggrid/parallel/ddd/dddcontext.hh>
45
46 #include <dune/uggrid/parallel/ddd/dddi.h>
47 #include "xfer.h"
48
49
50 USING_UG_NAMESPACE
51 using namespace PPIF;
52
53 START_UGDIM_NAMESPACE
54
55 /****************************************************************************/
56 /* */
57 /* data structures */
58 /* */
59 /****************************************************************************/
60
61 /* CPLMSG: complete description of message on sender side */
62
63 struct CPLMSG
64 {
65 DDD_PROC proc;
66
67 XIDelCpl **xferDelCpl = nullptr;
68 int nDelCpl = 0;
69
70 XIModCpl **xferModCpl = nullptr;
71 int nModCpl = 0;
72
73 XIAddCpl **xferAddCpl = nullptr;
74 int nAddCpl = 0;
75
76
77 /* lowcomm message handle */
78 LC_MSGHANDLE msg_h;
79
CPLMSGCPLMSG80 CPLMSG(DDD_PROC dest)
81 : proc(dest)
82 {}
83 };
84
85 using CplmsgList = std::forward_list<CPLMSG>;
86
87 /****************************************************************************/
88 /* */
89 /* routines */
90 /* */
91 /****************************************************************************/
92
93
CplMsgInit(DDD::DDDContext & context)94 void CplMsgInit(DDD::DDDContext& context)
95 {
96 auto& ctx = context.cplmsgContext();
97
98 ctx.cplmsg_t = LC_NewMsgType(context, "CplMsg");
99 ctx.delcpl_id = LC_NewMsgTable("DelCpl", ctx.cplmsg_t, sizeof(TEDelCpl));
100 ctx.modcpl_id = LC_NewMsgTable("ModCpl", ctx.cplmsg_t, sizeof(TEModCpl));
101 ctx.addcpl_id = LC_NewMsgTable("AddCpl", ctx.cplmsg_t, sizeof(TEAddCpl));
102 }
103
104
CplMsgExit(DDD::DDDContext &)105 void CplMsgExit(DDD::DDDContext&)
106 {}
107
108
109 /****************************************************************************/
110
111 static
112 std::pair<int, CplmsgList>
PrepareCplMsgs(DDD::DDDContext & context,XIDelCpl ** itemsDC,int nDC,XIModCpl ** itemsMC,int nMC,XIAddCpl ** itemsAC,int nAC)113 PrepareCplMsgs (
114 DDD::DDDContext& context,
115 XIDelCpl **itemsDC, int nDC,
116 XIModCpl **itemsMC, int nMC,
117 XIAddCpl **itemsAC, int nAC)
118 {
119 auto& ctx = context.cplmsgContext();
120 const auto procs = context.procs();
121
122 # if DebugCplMsg<=3
123 Dune::dverb << "PrepareCplMsgs, nXIDelCpl=" << nDC << " nXIModCpl=" << nMC
124 << " nXIAddCpl=" << nAC << "\n";
125 # endif
126
127 CplmsgList msgs;
128 int nMsgs = 0;
129
130 auto msgForProc = [&](DDD_PROC dest) -> CPLMSG& {
131 if (msgs.empty() or msgs.front().proc != dest) {
132 msgs.emplace_front(dest);
133 ++nMsgs;
134 }
135 return msgs.front();
136 };
137
138 /*
139 run through all tables simultaneously,
140 each time a new proc-nr is encountered in one of these
141 tables, create a new CPLMSG item.
142
143 (the lists have been sorted according to proc-nr previously.)
144 */
145
146 int iDC=0, iMC=0, iAC=0;
147 while (iDC<nDC || iMC<nMC || iAC<nAC)
148 {
149 DDD_PROC pDC = (iDC<nDC) ? itemsDC[iDC]->to : procs;
150 DDD_PROC pMC = (iMC<nMC) ? itemsMC[iMC]->to : procs;
151 DDD_PROC pAC = (iAC<nAC) ? itemsAC[iAC]->to : procs;
152
153 /* check DelCpl-items */
154 if (pDC<=pMC && pDC<=pAC && pDC<procs)
155 {
156 int i;
157
158 CPLMSG& xm = msgForProc(pDC);
159 xm.xferDelCpl = itemsDC+iDC;
160 for(i=iDC; i<nDC && itemsDC[i]->to==pDC; i++)
161 ;
162
163 xm.nDelCpl = i-iDC;
164 iDC = i;
165 }
166
167 /* check ModCpl-items */
168 if (pMC<=pDC && pMC<=pAC && pMC<procs)
169 {
170 int i;
171
172 CPLMSG& xm = msgForProc(pMC);
173 xm.xferModCpl = itemsMC+iMC;
174 for(i=iMC; i<nMC && itemsMC[i]->to==pMC; i++)
175 ;
176
177 xm.nModCpl = i-iMC;
178 iMC = i;
179 }
180
181 /* check AddCpl-items */
182 if (pAC<=pDC && pAC<=pMC && pAC<procs)
183 {
184 int i;
185
186 CPLMSG& xm = msgForProc(pAC);
187 xm.xferAddCpl = itemsAC+iAC;
188 for(i=iAC; i<nAC && itemsAC[i]->to==pAC; i++)
189 ;
190
191 xm.nAddCpl = i-iAC;
192 iAC = i;
193 }
194
195 if (pDC==procs) iDC = nDC;
196 if (pMC==procs) iMC = nMC;
197 if (pAC==procs) iAC = nAC;
198 }
199
200
201 /* initiate send messages */
202 for(auto& xm : msgs)
203 {
204 /* create new send message */
205 xm.msg_h = LC_NewSendMsg(context, ctx.cplmsg_t, xm.proc);
206
207 /* init tables inside message */
208 LC_SetTableSize(xm.msg_h, ctx.delcpl_id, xm.nDelCpl);
209 LC_SetTableSize(xm.msg_h, ctx.modcpl_id, xm.nModCpl);
210 LC_SetTableSize(xm.msg_h, ctx.addcpl_id, xm.nAddCpl);
211
212 /* prepare message for sending away */
213 LC_MsgPrepareSend(context, xm.msg_h);
214 }
215
216 return {nMsgs, std::move(msgs)};
217 }
218
219
CplMsgSend(DDD::DDDContext & context,const CplmsgList & msgs)220 static void CplMsgSend(DDD::DDDContext& context, const CplmsgList& msgs)
221 {
222 auto& ctx = context.cplmsgContext();
223
224 for(const auto& msg : msgs)
225 {
226 TEDelCpl *arrayDC = (TEDelCpl *)LC_GetPtr(msg.msg_h, ctx.delcpl_id);
227 TEModCpl *arrayMC = (TEModCpl *)LC_GetPtr(msg.msg_h, ctx.modcpl_id);
228 TEAddCpl *arrayAC = (TEAddCpl *)LC_GetPtr(msg.msg_h, ctx.addcpl_id);
229
230 /* copy data into message */
231 for(int i=0; i < msg.nDelCpl; i++)
232 {
233 arrayDC[i] = msg.xferDelCpl[i]->te;
234 }
235 for(int i=0; i < msg.nModCpl; i++)
236 {
237 arrayMC[i] = msg.xferModCpl[i]->te;
238 }
239 for(int i=0; i < msg.nAddCpl; i++)
240 {
241 arrayAC[i] = msg.xferAddCpl[i]->te;
242 }
243
244 /* schedule message for send */
245 LC_MsgSend(context, msg.msg_h);
246 }
247 }
248
249
250
251 /****************************************************************************/
252
253
CplMsgUnpackSingle(DDD::DDDContext & context,LC_MSGHANDLE xm,DDD_HDR * localCplObjs,int nLCO)254 static void CplMsgUnpackSingle (DDD::DDDContext& context, LC_MSGHANDLE xm,
255 DDD_HDR *localCplObjs, int nLCO)
256 {
257 auto& ctx = context.cplmsgContext();
258
259 TEDelCpl *theDelCpl;
260 TEModCpl *theModCpl;
261 TEAddCpl *theAddCpl;
262 int i, j, nDelCpl, nModCpl, nAddCpl;
263 DDD_PROC proc = LC_MsgGetProc(xm);
264
265 /* get number and address of del-items */
266 nDelCpl = (int) LC_GetTableLen(xm, ctx.delcpl_id);
267 nModCpl = (int) LC_GetTableLen(xm, ctx.modcpl_id);
268 nAddCpl = (int) LC_GetTableLen(xm, ctx.addcpl_id);
269 theDelCpl = (TEDelCpl *) LC_GetPtr(xm, ctx.delcpl_id);
270 theModCpl = (TEModCpl *) LC_GetPtr(xm, ctx.modcpl_id);
271 theAddCpl = (TEAddCpl *) LC_GetPtr(xm, ctx.addcpl_id);
272
273
274 /* modify couplings according to mod-list */
275 for(i=0, j=0; i<nModCpl; i++)
276 {
277 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theModCpl[i].gid))
278 j++;
279
280 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theModCpl[i].gid))
281 {
282 ModCoupling(context, localCplObjs[j], proc, theModCpl[i].prio);
283 }
284 }
285
286
287 /* delete couplings according to del-list */
288 for(i=0, j=0; i<nDelCpl; i++)
289 {
290 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theDelCpl[i].gid))
291 j++;
292
293 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theDelCpl[i].gid))
294 {
295 DelCoupling(context, localCplObjs[j], proc);
296 }
297 }
298
299
300 /* add couplings according to add-list */
301 for(i=0, j=0; i<nAddCpl; i++)
302 {
303 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theAddCpl[i].gid))
304 j++;
305
306 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theAddCpl[i].gid))
307 {
308 AddCoupling(context, localCplObjs[j], theAddCpl[i].proc, theAddCpl[i].prio);
309 }
310 }
311 }
312
313
314 /****************************************************************************/
315
316
CplMsgDisplay(DDD::DDDContext & context,const char * comment,LC_MSGHANDLE xm)317 static void CplMsgDisplay (DDD::DDDContext& context, const char *comment, LC_MSGHANDLE xm)
318 {
319 using std::setw;
320
321 std::ostream& out = std::cout;
322 auto& ctx = context.cplmsgContext();
323 TEDelCpl *theDelCpl;
324 TEModCpl *theModCpl;
325 TEAddCpl *theAddCpl;
326 int i, proc = LC_MsgGetProc(xm);
327 int lenDelCpl = (int) LC_GetTableLen(xm, ctx.delcpl_id);
328 int lenModCpl = (int) LC_GetTableLen(xm, ctx.modcpl_id);
329 int lenAddCpl = (int) LC_GetTableLen(xm, ctx.addcpl_id);
330
331 std::ostringstream prefixStream;
332 prefixStream
333 << " " << setw(3) << context.me() << "-" << comment << "-" << setw(3) << proc << " ";
334 const std::string& prefix = prefixStream.str();
335
336 /* get table addresses inside message */
337 theDelCpl = (TEDelCpl *) LC_GetPtr(xm, ctx.delcpl_id);
338 theModCpl = (TEModCpl *) LC_GetPtr(xm, ctx.modcpl_id);
339 theAddCpl = (TEAddCpl *) LC_GetPtr(xm, ctx.addcpl_id);
340
341
342 out << prefix << " 04 DelCpl.size=" << setw(5) << lenDelCpl << "\n";
343 out << prefix << " 05 ModCpl.size=" << setw(5) << lenModCpl << "\n";
344 out << prefix << " 06 AddCpl.size=" << setw(5) << lenAddCpl << "\n";
345
346 for(i=0; i<lenDelCpl; i++)
347 out << prefix << " 14 delcpl " << setw(4) << i << " - "
348 << theDelCpl[i].gid << "\n";
349
350 for(i=0; i<lenModCpl; i++)
351 out << prefix << " 15 modcpl " << setw(4) << i << " - "
352 << theModCpl[i].gid << " " << setw(3) << theModCpl[i].prio << "\n";
353
354 for(i=0; i<lenAddCpl; i++)
355 out << prefix << " 16 addcpl " << setw(4) << i << " - "
356 << theAddCpl[i].gid << " " << setw(4) << theAddCpl[i].proc
357 << " " << setw(3) << theAddCpl[i].prio << "\n";
358 }
359
360
361 /****************************************************************************/
362
363
364 /*
365 ...
366
367 localCplObjs is a sorted list of all objects with coupling lists.
368 */
369
CommunicateCplMsgs(DDD::DDDContext & context,XIDelCpl ** itemsDC,int nDC,XIModCpl ** itemsMC,int nMC,XIAddCpl ** itemsAC,int nAC,DDD_HDR * localCplObjs,int nLCO)370 void CommunicateCplMsgs (
371 DDD::DDDContext& context,
372 XIDelCpl **itemsDC, int nDC,
373 XIModCpl **itemsMC, int nMC,
374 XIAddCpl **itemsAC, int nAC,
375 DDD_HDR *localCplObjs, int nLCO)
376 {
377 auto& ctx = context.cplmsgContext();
378
379 /* accumulate messages (one for each partner) */
380 CplmsgList sendMsgs;
381 int nSendMsgs;
382 std::tie(nSendMsgs, sendMsgs) = PrepareCplMsgs(context,
383 itemsDC, nDC,
384 itemsMC, nMC,
385 itemsAC, nAC);
386
387 /* init communication topology */
388 int nRecvMsgs = LC_Connect(context, ctx.cplmsg_t);
389
390 /* build and send messages */
391 CplMsgSend(context, sendMsgs);
392
393
394 #if DebugCplMsg>2
395 if (DDD_GetOption(context, OPT_DEBUG_XFERMESGS)==OPT_ON)
396 #endif
397 {
398 for (const auto& msg : sendMsgs)
399 CplMsgDisplay(context, "CS", msg.msg_h);
400 }
401
402
403 /* display information about send-messages on lowcomm-level */
404 if (DDD_GetOption(context, OPT_INFO_XFER) & XFER_SHOW_MSGSALL)
405 {
406 DDD_SyncAll(context);
407 if (context.isMaster())
408 Dune::dwarn << "DDD XFER_SHOW_MSGSALL: CplMsg.Send\n";
409 LC_PrintSendMsgs(context);
410 }
411
412
413 /* communicate set of messages (send AND receive) */
414 LC_MSGHANDLE* recvMsgs = LC_Communicate(context);
415
416
417 /* display information about recv-messages on lowcomm-level */
418 if (DDD_GetOption(context, OPT_INFO_XFER) & XFER_SHOW_MSGSALL)
419 {
420 DDD_SyncAll(context);
421 if (context.isMaster())
422 Dune::dwarn << "DDD XFER_SHOW_MSGSALL: CplMsg.Recv\n";
423 LC_PrintRecvMsgs(context);
424 }
425
426
427 for(int i=0; i<nRecvMsgs; i++)
428 {
429 CplMsgUnpackSingle(context, recvMsgs[i], localCplObjs, nLCO);
430
431 /*
432 if (DDD_GetOption(context, OPT_DEBUG_XFERMESGS)==OPT_ON)
433 CplMsgDisplay(context, "CR", recvMsgs[i]);
434 */
435 }
436
437
438 /* cleanup low-comm layer */
439 LC_Cleanup(context);
440 }
441
442
443 /****************************************************************************/
444
445 END_UGDIM_NAMESPACE
446