1 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 // vi: set et ts=4 sw=2 sts=2:
3 /****************************************************************************/
4 /* */
5 /* File: mpi-ppif.c */
6 /* */
7 /* Purpose: parallel processor interface */
8 /* Provides a portable interface to message passing MIMD */
9 /* architectures. PPIF is divided into three parts: */
10 /* */
11 /* (1) Administration */
12 /* (2) Communication */
13 /* (3) Miscellaneous */
14 /* */
15 /* The interface assumes that the parallel machine has */
16 /* the following properties: */
17 /* */
18 /* (1) it is physically connected at least as a 2 or 3 dim. array*/
19 /* (2) it has a fast virtual channel communication mechanism */
20 /* (3) it has an asynchronous communication mechanism */
21 /* */
22 /* MPI Implementation */
23 /* */
24 /* Author: Jens Boenisch */
25 /* Rechenzentrum Uni Stuttgart */
26 /* Universitaet Stuttgart */
27 /* Allmandring 3a */
28 /* 7000 Stuttgart 80 */
29 /* internet: boenisch@rus.uni-stuttgart.de */
30 /* */
31 /* History: 17 Aug 1992, begin */
32 /* 18 Feb 1993, Indigo version */
33 /* 02 Jun 1993, Paragon OSF/1 version */
34 /* 14 Sep 1995, MPI version */
35 /* 29 Jan 2003, pV3 concentrator support */
36 /* */
37 /* Remarks: */
38 /* */
39 /****************************************************************************/
40
41 /****************************************************************************/
42 /* */
43 /* include files */
44 /* system include files */
45 /* application include files */
46 /* */
47 /****************************************************************************/
48
49 /* standard C library */
50 #include <config.h>
51 #include <sys/types.h>
52 #include <cstdio>
53 #include <cstdlib>
54 #include <ctime>
55 #include <cmath>
56
57 #include <mpi.h>
58
59 #include "ppif.h"
60 #include "ppifcontext.hh"
61
62 using namespace PPIF;
63
64 /****************************************************************************/
65 /* */
66 /* defines in the following order */
67 /* */
68 /* compile time constants defining static data size (i.e. arrays) */
69 /* other constants */
70 /* macros */
71 /* */
72 /****************************************************************************/
73
74 #define MAXT 15 /* maximum number of downtree nodes max */
75 /* log2(P) */
76
77 #define ID_TREE 101 /* channel id: tree */
78
79 #define PPIF_SUCCESS 0 /* Return value for success */
80 #define PPIF_FAILURE 1 /* Return value for failure */
81
82 /****************************************************************************/
83 /* */
84 /* data structures */
85 /* */
86 /****************************************************************************/
87
88 namespace PPIF {
89
90 struct VChannel
91 {
92 int p;
93 int chanid;
94 };
95
96 struct Msg
97 {
98 MPI_Request req;
99 };
100
101 } /* namespace PPIF */
102
103 /****************************************************************************/
104 /* */
105 /* definition of static variables */
106 /* */
107 /****************************************************************************/
108
109
110
111 /****************************************************************************/
112 /* */
113 /* definition of exported global variables */
114 /* */
115 /****************************************************************************/
116
117 /* id's */
118 int PPIF::me; /* my processor id */
119 int PPIF::master; /* id of master processor */
120 int PPIF::procs; /* number of processors in the network */
121
122 /****************************************************************************/
123 /* */
124 /* routines for handling virtual channels */
125 /* */
126 /****************************************************************************/
127
NewVChan(int p,int id)128 static VChannelPtr NewVChan (int p, int id)
129
130 {
131 VChannelPtr myChan = new PPIF::VChannel;
132
133 myChan->p = p;
134 myChan->chanid = id;
135
136 return myChan;
137 }
138
139
DeleteVChan(VChannelPtr myChan)140 static void DeleteVChan (VChannelPtr myChan)
141
142 {
143 delete myChan;
144 }
145
146 static std::shared_ptr<PPIF::PPIFContext> ppifContext_;
147
ppifContext(const std::shared_ptr<PPIFContext> & context)148 void PPIF::ppifContext(const std::shared_ptr<PPIFContext>& context)
149 {
150 ppifContext_ = context;
151
152 me = context->me();
153 master = context->master();
154 procs = context->procs();
155 }
156
ppifContext(std::nullptr_t)157 void PPIF::ppifContext(std::nullptr_t)
158 {
159 ppifContext_ = nullptr;
160
161 me = 0;
162 master = 0;
163 procs = 1;
164 }
165
ppifContext()166 const std::shared_ptr<PPIF::PPIFContext>& PPIF::ppifContext()
167 {
168 return ppifContext_;
169 }
170
171 /****************************************************************************/
172 /* */
173 /* Function: InitPPIF */
174 /* */
175 /* Purpose: initialize parallel processor interface */
176 /* set exported variables, allocate tree communication structure */
177 /* */
178 /* Input: void */
179 /* */
180 /* Output: int 0: ok */
181 /* int!=0: error */
182 /* */
183 /****************************************************************************/
184
185 /*
186 Factor N into two integers that are as close together as possible
187 */
188
Factor(int N,int * pn,int * pm)189 static void Factor (int N, int *pn, int *pm)
190
191 {
192 int n = (int)ceil (sqrt ((double) N));
193 int m = (int)floor (sqrt ((double) N));
194
195 while (n*m != N)
196 {
197 if (n*m < N) n++;
198 else m--;
199 }
200
201 *pn = n; *pm = m;
202 }
203
204
InitPPIF(PPIFContext & context)205 void PPIF::InitPPIF(PPIFContext& context)
206 {
207 const auto me = context.me();
208 const auto procs = context.procs();
209
210 context.dims_[2] = 1;
211 Factor(procs, &context.dims_[0], &context.dims_[1]);
212
213 /* tree configuration */
214
215 auto& degree = context.degree_;
216 auto& uptree = context.uptree_;
217 auto& downtree = context.downtree_;
218
219 degree = 0;
220 const int sonl = 2*me + 1;
221 const int sonr = 2*me + 2;
222
223 if (sonl<procs)
224 {
225 degree++;
226 if (! downtree[0]) /* InitPPIF is being called for the first time */
227 downtree[0] = NewVChan(sonl,ID_TREE);
228 }
229 else
230 {
231 downtree[0] = NULL;
232 }
233
234 if (sonr<procs)
235 {
236 degree++;
237 if (! downtree[1]) /* InitPPIF is being called for the first time */
238 downtree[1] = NewVChan(sonr,ID_TREE);
239 }
240 else
241 {
242 downtree[1] = NULL;
243 }
244
245 if (me>0)
246 {
247 if (! uptree) /* InitPPIF is being called for the first time */
248 uptree = NewVChan((me-1)/2,ID_TREE);
249 }
250 else
251 {
252 uptree = NULL;
253 }
254
255 auto& slvcnt = context.slvcnt_;
256
257 int succ=1;
258 for(int i=0; i<degree; i++)
259 {
260 MPI_Recv ((void *) &(slvcnt[i]), (int) sizeof(int), MPI_BYTE,
261 downtree[i]->p, ID_TREE, context.comm(), MPI_STATUS_IGNORE);
262 succ += slvcnt[i];
263 }
264 if (me>0)
265 {
266 MPI_Send ((void *) &succ, (int) sizeof(int), MPI_BYTE, (int)(me-1)/2, ID_TREE, context.comm());
267 }
268 }
269
InitPPIF(int *,char ***)270 int PPIF::InitPPIF (int *, char ***)
271 {
272 auto context = ppifContext();
273 if (not context)
274 context = std::make_shared<PPIFContext>();
275 ppifContext(context);
276
277 return PPIF_SUCCESS;
278 }
279
ExitPPIF(PPIF::PPIFContext & context)280 void PPIF::ExitPPIF(PPIF::PPIFContext& context)
281 {
282 /* Deallocate tree structure */
283 DeleteVChan(context.uptree_);
284 context.uptree_ = nullptr;
285 /* I currently think that only the first two entries of downtree can contain
286 * valied entries, but I am not entirely sure. */
287 DeleteVChan(context.downtree_[0]);
288 DeleteVChan(context.downtree_[1]);
289 context.downtree_[0] = context.downtree_[1] = nullptr;
290 }
291
ExitPPIF()292 int PPIF::ExitPPIF ()
293 {
294 if (ppifContext()) {
295 ppifContext(nullptr);
296 }
297
298 return PPIF_SUCCESS;
299 }
300
301 /****************************************************************************/
302 /* */
303 /* Tree oriented functions */
304 /* */
305 /****************************************************************************/
306
Broadcast(const PPIFContext & context,void * data,int size)307 int PPIF::Broadcast (const PPIFContext& context, void *data, int size)
308 {
309 if (MPI_SUCCESS != MPI_Bcast (data, size, MPI_BYTE, context.master(), context.comm()) )
310 return (PPIF_FAILURE);
311
312 return (PPIF_SUCCESS);
313 }
314
Concentrate(const PPIFContext & context,void * data,int size)315 int PPIF::Concentrate (const PPIFContext& context, void *data, int size)
316 {
317 if (not context.isMaster())
318 if (SendSync (context, context.uptree(), data, size) < 0) return (PPIF_FAILURE);
319
320 return (PPIF_SUCCESS);
321 }
322
GetConcentrate(const PPIFContext & context,int slave,void * data,int size)323 int PPIF::GetConcentrate(const PPIFContext& context, int slave, void *data, int size)
324 {
325 if (slave < context.degree())
326 if (RecvSync (context, context.downtree()[slave], data, size) < 0) return (PPIF_FAILURE);
327
328 return (PPIF_SUCCESS);
329 }
330
Spread(const PPIFContext & context,int slave,void * data,int size)331 int PPIF::Spread(const PPIFContext& context, int slave, void *data, int size)
332 {
333 if (slave < context.degree())
334 if (SendSync(context, context.downtree()[slave], data, size) < 0) return (PPIF_FAILURE);
335
336 return (PPIF_SUCCESS);
337 }
338
GetSpread(const PPIFContext & context,void * data,int size)339 int PPIF::GetSpread(const PPIFContext& context, void *data, int size)
340 {
341 if (not context.isMaster())
342 if (RecvSync(context, context.uptree(), data, size) < 0) return (PPIF_FAILURE);
343
344 return (PPIF_SUCCESS);
345 }
346
Synchronize(const PPIFContext & context)347 int PPIF::Synchronize(const PPIFContext& context)
348 {
349 if (MPI_SUCCESS != MPI_Barrier (context.comm()) ) return (PPIF_FAILURE);
350
351 return (PPIF_SUCCESS);
352 }
353
354 /****************************************************************************/
355 /* */
356 /* Synchronous communication */
357 /* */
358 /****************************************************************************/
359
ConnSync(const PPIFContext &,int p,int id)360 VChannelPtr PPIF::ConnSync(const PPIFContext&, int p, int id)
361 {
362 return NewVChan(p, id);
363 }
364
DiscSync(const PPIFContext &,VChannelPtr v)365 int PPIF::DiscSync(const PPIFContext&, VChannelPtr v)
366 {
367 DeleteVChan(v);
368
369 return (0);
370 }
371
SendSync(const PPIFContext & context,VChannelPtr v,void * data,int size)372 int PPIF::SendSync(const PPIFContext& context, VChannelPtr v, void *data, int size)
373 {
374 if (MPI_SUCCESS == MPI_Ssend (data, size, MPI_BYTE,
375 v->p, v->chanid, context.comm()) )
376 return (size);
377 else
378 return (-1);
379 }
380
RecvSync(const PPIFContext & context,VChannelPtr v,void * data,int size)381 int PPIF::RecvSync(const PPIFContext& context, VChannelPtr v, void *data, int size)
382 {
383 int count = -1;
384 MPI_Status status;
385
386 if (MPI_SUCCESS == MPI_Recv (data, size, MPI_BYTE,
387 v->p, v->chanid, context.comm(), &status) )
388 MPI_Get_count (&status, MPI_BYTE, &count);
389
390 return (count);
391 }
392
393 /****************************************************************************/
394 /* */
395 /* Asynchronous communication */
396 /* */
397 /****************************************************************************/
398
ConnASync(const PPIFContext &,int p,int id)399 VChannelPtr PPIF::ConnASync(const PPIFContext&, int p, int id)
400 {
401 return NewVChan(p, id);
402 }
403
InfoAConn(const PPIFContext &,VChannelPtr v)404 int PPIF::InfoAConn(const PPIFContext&, VChannelPtr v)
405 {
406 return (v ? 1 : -1);
407 }
408
DiscASync(const PPIFContext &,VChannelPtr v)409 int PPIF::DiscASync(const PPIFContext&, VChannelPtr v)
410 {
411 DeleteVChan (v);
412 return (PPIF_SUCCESS);
413 }
414
InfoADisc(const PPIFContext &,VChannelPtr v)415 int PPIF::InfoADisc(const PPIFContext&, VChannelPtr v)
416 {
417 return (true);
418 }
419
SendASync(const PPIFContext & context,VChannelPtr v,void * data,int size,int * error)420 msgid PPIF::SendASync(const PPIFContext& context, VChannelPtr v, void *data, int size, int *error)
421 {
422 msgid m = new PPIF::Msg;
423
424 if (m)
425 {
426 if (MPI_SUCCESS == MPI_Isend (data, size, MPI_BYTE,
427 v->p, v->chanid, context.comm(), &m->req) )
428 {
429 *error = false;
430 return m;
431 }
432 }
433
434 *error = true;
435 return NULL;
436 }
437
RecvASync(const PPIFContext & context,VChannelPtr v,void * data,int size,int * error)438 msgid PPIF::RecvASync(const PPIFContext& context, VChannelPtr v, void *data, int size, int *error)
439 {
440 msgid m = new PPIF::Msg;
441
442 if (m)
443 {
444 if (MPI_SUCCESS == MPI_Irecv (data, size, MPI_BYTE,
445 v->p, v->chanid, context.comm(), &m->req) )
446 {
447 *error = false;
448 return m;
449 }
450 }
451
452 *error = true;
453 return (NULL);
454 }
455
InfoASend(const PPIFContext &,VChannelPtr v,msgid m)456 int PPIF::InfoASend(const PPIFContext&, VChannelPtr v, msgid m)
457 {
458 int complete;
459
460 if (m)
461 {
462 if (MPI_SUCCESS == MPI_Test (&m->req, &complete, MPI_STATUS_IGNORE) )
463 {
464 if (complete)
465 delete m;
466
467 return (complete); /* complete is true for completed send, false otherwise */
468 }
469 }
470
471 return (-1); /* return -1 for FAILURE */
472 }
473
InfoARecv(const PPIFContext &,VChannelPtr v,msgid m)474 int PPIF::InfoARecv(const PPIFContext&, VChannelPtr v, msgid m)
475 {
476 int complete;
477
478 if (m)
479 {
480 if (MPI_SUCCESS == MPI_Test (&m->req, &complete, MPI_STATUS_IGNORE) )
481 {
482 if (complete)
483 delete m;
484
485 return (complete); /* complete is true for completed receive, false otherwise */
486 }
487 }
488
489 return (-1); /* return -1 for FAILURE */
490 }
491