1 //
2 // messimpl.cc
3 //
4 // Copyright (C) 1996 Limit Point Systems, Inc.
5 //
6 // Author: Curtis Janssen <cljanss@limitpt.com>
7 // Maintainer: LPS
8 //
9 // This file is part of the SC Toolkit.
10 //
11 // The SC Toolkit is free software; you can redistribute it and/or modify
12 // it under the terms of the GNU Library General Public License as published by
13 // the Free Software Foundation; either version 2, or (at your option)
14 // any later version.
15 //
16 // The SC Toolkit is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 // GNU Library General Public License for more details.
20 //
21 // You should have received a copy of the GNU Library General Public License
22 // along with the SC Toolkit; see the file COPYING.LIB. If not, write to
23 // the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
24 //
25 // The U.S. Government is granted a limited license as per AL 91-7.
26 //
27
28 #include <string.h>
29
30 #include <util/misc/formio.h>
31 #include <util/misc/exenv.h>
32
33 #include <util/group/message.h>
34
35 #include <util/group/topology.h>
36 #include <util/group/hcube.h>
37 #ifdef HAVE_NX
38 # include <util/group/messpgon.h>
39 #endif
40 #ifdef HAVE_MPI
41 # define MPICH_SKIP_MPICXX
42 # include <mpi.h>
43 # include <util/group/messmpi.h>
44 #endif
45
46 #define DEFAULT_GOP_MAX 320000
47
48 using namespace std;
49 using namespace sc;
50
51 static ClassDesc MessageGrp_cd(
52 typeid(MessageGrp),"MessageGrp",1,"public DescribedClass",
53 0, 0, 0);
54
MessageGrp(const Ref<KeyVal> & keyval)55 MessageGrp::MessageGrp(const Ref<KeyVal>& keyval):
56 me_(-1),
57 n_(-1),
58 index_to_classdesc_(0)
59 {
60 gop_max_ = keyval->intvalue("gop_max");
61 if (keyval->error() != KeyVal::OK) gop_max_ = DEFAULT_GOP_MAX;
62 debug_ = keyval->booleanvalue("debug");
63 }
64
MessageGrp()65 MessageGrp::MessageGrp():
66 me_(-1),
67 n_(-1),
68 index_to_classdesc_(0)
69 {
70 gop_max_ = DEFAULT_GOP_MAX;
71 debug_ = 0;
72 }
73
~MessageGrp()74 MessageGrp::~MessageGrp()
75 {
76 if (index_to_classdesc_) delete[] index_to_classdesc_;
77 }
78
79 static Ref<MessageGrp> default_messagegrp;
80
81 void
set_default_messagegrp(const Ref<MessageGrp> & grp)82 MessageGrp::set_default_messagegrp(const Ref<MessageGrp>& grp)
83 {
84 default_messagegrp = grp;
85 }
86
87 MessageGrp*
get_default_messagegrp()88 MessageGrp::get_default_messagegrp()
89 {
90 if (default_messagegrp.null()) {
91 #if defined(HAVE_MPI) && defined(DEFAULT_MPI)
92 default_messagegrp = new MPIMessageGrp;
93 #else
94 default_messagegrp = new ProcMessageGrp;
95 #endif
96 }
97 return default_messagegrp.pointer();
98 }
99
100 MessageGrp *
initial_messagegrp(int & argc,char ** & argv)101 MessageGrp::initial_messagegrp(int &argc, char** &argv)
102 {
103 MessageGrp *grp = 0;
104
105 char *keyval_string = 0;
106
107 // see if a message group is given on the command line
108 if (argc && argv) {
109 for (int i=0; i<argc; i++) {
110 if (argv[i] && !strcmp(argv[i], "-messagegrp")) {
111 char *messagegrp_string = argv[i];
112 i++;
113 if (i >= argc) {
114 ExEnv::errn() << "-messagegrp must be following by an argument"
115 << endl;
116 abort();
117 }
118 keyval_string = argv[i];
119 // move the messagegrp arguments to the end of argv
120 int j;
121 for (j=i+1; j<argc; j++) {
122 argv[j-2] = argv[j];
123 }
124 argv[j++] = messagegrp_string;
125 argv[j++] = keyval_string;
126 // decrement argc to hide the last two arguments
127 argc -= 2;
128 break;
129 }
130 }
131 }
132
133 if (!keyval_string) {
134 // find out if the environment gives the containing message group
135 keyval_string = getenv("MESSAGEGRP");
136 if (keyval_string) {
137 if (!strncmp("MESSAGEGRP=", keyval_string, 11)) {
138 keyval_string = strchr(keyval_string, '=');
139 }
140 if (*keyval_string == '=') keyval_string++;
141 }
142 }
143
144 // if keyval input for a message group was found, then
145 // create it.
146 if (keyval_string) {
147 if (keyval_string[0] == '\0') return 0;
148 //ExEnv::outn() << "Creating MessageGrp from \"" << keyval_string << "\"" << endl;
149 Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
150 strkv->parse_string(keyval_string);
151 Ref<DescribedClass> dc = strkv->describedclassvalue();
152 grp = dynamic_cast<MessageGrp*>(dc.pointer());
153 if (dc.null()) {
154 ExEnv::errn() << "initial_messagegrp: couldn't find a MessageGrp in "
155 << keyval_string << endl;
156 abort();
157 }
158 else if (!grp) {
159 ExEnv::errn() << "initial_messagegrp: wanted MessageGrp but got "
160 << dc->class_name() << endl;
161 abort();
162 }
163 // prevent an accidental delete
164 grp->reference();
165 strkv = 0;
166 dc = 0;
167 // accidental delete not a problem anymore since all smart pointers
168 // to grp are dead
169 grp->dereference();
170 return grp;
171 }
172
173 #if defined(HAVE_MPI)
174 int mpiinited;
175 #ifdef ALWAYS_USE_MPI
176 bool always_use_mpi = true;
177 #else
178 bool always_use_mpi = false;
179 #endif
180 MPI_Initialized(&mpiinited);
181 if (mpiinited || always_use_mpi) {
182 grp = new MPIMessageGrp(&argc,&argv);
183 return grp;
184 }
185 #endif
186
187 return 0;
188 }
189
190 void
initialize(int me,int n)191 MessageGrp::initialize(int me, int n)
192 {
193 // This member is called by a CTOR and, ultimately, causes
194 // 'this' to be converted into a temporary Ref which causes
195 // this to be deleted (very bad), so reference 'this'
196 // (and dereference this down below).
197 this->reference();
198
199 if (topology_.null()) {
200 topology_ = new HypercubeTopology();
201 }
202
203 int i;
204 std::map<std::string,ClassDescP>::iterator J;
205
206 me_ = me;
207 n_ = n;
208
209 // get all of the classes known on this node
210 std::map<std::string,ClassDescP>& classes = ClassDesc::all();
211
212 // Keeps count of how many classes are known.
213 int iclass = 0;
214
215 for (i=0; i<n; i++) {
216 if (i==me) {
217 // Find out how many of my classes are not yet in the
218 // classdesc to index map.
219 int n_new_class = 0;
220 int buffer_size = 0;
221 for (J=classes.begin(); J!=classes.end(); J++) {
222 if (classdesc_to_index_.find(J->second) == classdesc_to_index_.end()) {
223 n_new_class++;
224 buffer_size += strlen(J->second->name()) + 1;
225 }
226 }
227 char* buffer = new char[buffer_size];
228 char* currentbuffer = buffer;
229 for (J=classes.begin(); J!=classes.end(); J++) {
230 if (classdesc_to_index_.find(J->second) == classdesc_to_index_.end()) {
231 classdesc_to_index_[J->second] = iclass;
232 iclass++;
233 #ifdef DEBUG
234 ExEnv::outn() << scprintf("node %d adding class %d = \"%s\"\n",
235 me, iclass, J->second->name());
236 #endif
237 strcpy(currentbuffer,J->second->name());
238 currentbuffer += strlen(J->second->name()) + 1;
239 }
240 }
241 #ifdef DEBUG
242 ExEnv::outn() << scprintf("node %d bcast n_new_class = %d\n",me,n_new_class);
243 #endif
244 bcast(&n_new_class,1,i);
245 #ifdef DEBUG
246 ExEnv::outn() << scprintf("node %d finished bcast\n",me);
247 #endif
248 if (n_new_class) {
249 bcast(&buffer_size,1,i);
250 bcast(buffer,buffer_size,i);
251 }
252 delete[] buffer;
253 }
254 else {
255 int j;
256 // Get new classnames and indices from node i.
257 int n_new_class;
258 #ifdef DEBUG
259 ExEnv::outn() << scprintf("node %d begin recv bcast\n",me);
260 #endif
261 bcast(&n_new_class,1,i);
262 #ifdef DEBUG
263 ExEnv::outn() << scprintf("node %d recv bcast n_new_class = %d\n",
264 me,n_new_class);
265 #endif
266 if (n_new_class) {
267 int buffer_size;
268 bcast(&buffer_size,1,i);
269 char* buffer = new char[buffer_size];
270 char* currentbuffer = buffer;
271 bcast(buffer,buffer_size,i);
272 for (j=0; j<n_new_class; j++) {
273 ClassDescP cd = ClassDesc::name_to_class_desc(currentbuffer);
274 if (cd) {
275 #ifdef DEBUG
276 ExEnv::outn() << scprintf("node %d adding \"%s\"\n",
277 me, currentbuffer);
278 #endif
279 classdesc_to_index_[cd] = iclass;
280 }
281 #ifdef DEBUG
282 else {
283 ExEnv::outn() << scprintf("node %d failed to add \"%s\"\n",
284 me, currentbuffer);
285 }
286 #endif
287 iclass++;
288 // advance the currentbuffer to the next classname
289 while(*currentbuffer != '\0') currentbuffer++;
290 currentbuffer++;
291 }
292 delete[] buffer;
293 }
294 }
295 }
296 nclass_ = iclass;
297
298 // Construct the mapping of index to classdesc.
299 index_to_classdesc_ = new ClassDescP[iclass];
300 for (i=0; i<nclass_; i++) {
301 index_to_classdesc_[i] = 0;
302 }
303 for (J=classes.begin(); J!=classes.end(); J++) {
304 if (classdesc_to_index_.find(J->second) != classdesc_to_index_.end()) {
305 index_to_classdesc_[classdesc_to_index_[J->second]] = J->second;
306 }
307 }
308
309 this->dereference();
310 }
311
312 // Sequential send routines
313
314 void
send(int target,const double * data,int ndata)315 MessageGrp::send(int target, const double* data, int ndata)
316 {
317 raw_send(target, data, ndata*sizeof(double));
318 }
319 void
send(int target,const short * data,int ndata)320 MessageGrp::send(int target, const short* data, int ndata)
321 {
322 raw_send(target, data, ndata*sizeof(short));
323 }
324 void
send(int target,const long * data,int ndata)325 MessageGrp::send(int target, const long* data, int ndata)
326 {
327 raw_send(target, data, ndata*sizeof(long));
328 }
329 void
send(int target,const float * data,int ndata)330 MessageGrp::send(int target, const float* data, int ndata)
331 {
332 raw_send(target, data, ndata*sizeof(float));
333 }
334 void
send(int target,const unsigned int * data,int ndata)335 MessageGrp::send(int target, const unsigned int* data, int ndata)
336 {
337 raw_send(target, data, ndata*sizeof(int));
338 }
339 void
send(int target,const int * data,int ndata)340 MessageGrp::send(int target, const int* data, int ndata)
341 {
342 raw_send(target, data, ndata*sizeof(int));
343 }
344 void
send(int target,const char * data,int ndata)345 MessageGrp::send(int target, const char* data, int ndata)
346 {
347 raw_send(target, data, ndata);
348 }
349 void
send(int target,const unsigned char * data,int ndata)350 MessageGrp::send(int target, const unsigned char* data, int ndata)
351 {
352 raw_send(target, data, ndata);
353 }
354 void
send(int target,const signed char * data,int ndata)355 MessageGrp::send(int target, const signed char* data, int ndata)
356 {
357 raw_send(target, data, ndata);
358 }
359
360 // Sequential receive routines
361
362 void
recv(int sender,double * data,int ndata)363 MessageGrp::recv(int sender, double* data, int ndata)
364 {
365 raw_recv(sender, data, ndata*sizeof(double));
366 }
367 void
recv(int sender,short * data,int ndata)368 MessageGrp::recv(int sender, short* data, int ndata)
369 {
370 raw_recv(sender, data, ndata*sizeof(short));
371 }
372 void
recv(int sender,long * data,int ndata)373 MessageGrp::recv(int sender, long* data, int ndata)
374 {
375 raw_recv(sender, data, ndata*sizeof(long));
376 }
377 void
recv(int sender,float * data,int ndata)378 MessageGrp::recv(int sender, float* data, int ndata)
379 {
380 raw_recv(sender, data, ndata*sizeof(float));
381 }
382 void
recv(int sender,unsigned int * data,int ndata)383 MessageGrp::recv(int sender, unsigned int* data, int ndata)
384 {
385 raw_recv(sender, data, ndata*sizeof(int));
386 }
387 void
recv(int sender,int * data,int ndata)388 MessageGrp::recv(int sender, int* data, int ndata)
389 {
390 raw_recv(sender, data, ndata*sizeof(int));
391 }
392 void
recv(int sender,char * data,int ndata)393 MessageGrp::recv(int sender, char* data, int ndata)
394 {
395 raw_recv(sender, data, ndata);
396 }
397 void
recv(int sender,unsigned char * data,int ndata)398 MessageGrp::recv(int sender, unsigned char* data, int ndata)
399 {
400 raw_recv(sender, data, ndata);
401 }
402 void
recv(int sender,signed char * data,int ndata)403 MessageGrp::recv(int sender, signed char* data, int ndata)
404 {
405 raw_recv(sender, data, ndata);
406 }
407
408 // Typed send routines
409
410 void
sendt(int target,int type,const double * data,int ndata)411 MessageGrp::sendt(int target, int type, const double* data, int ndata)
412 {
413 raw_sendt(target, type, data, ndata*sizeof(double));
414 }
415 void
sendt(int target,int type,const short * data,int ndata)416 MessageGrp::sendt(int target, int type, const short* data, int ndata)
417 {
418 raw_sendt(target, type, data, ndata*sizeof(short));
419 }
420 void
sendt(int target,int type,const long * data,int ndata)421 MessageGrp::sendt(int target, int type, const long* data, int ndata)
422 {
423 raw_sendt(target, type, data, ndata*sizeof(long));
424 }
425 void
sendt(int target,int type,const float * data,int ndata)426 MessageGrp::sendt(int target, int type, const float* data, int ndata)
427 {
428 raw_sendt(target, type, data, ndata*sizeof(float));
429 }
430 void
sendt(int target,int type,const unsigned int * data,int ndata)431 MessageGrp::sendt(int target, int type, const unsigned int* data, int ndata)
432 {
433 raw_sendt(target, type, data, ndata*sizeof(int));
434 }
435 void
sendt(int target,int type,const int * data,int ndata)436 MessageGrp::sendt(int target, int type, const int* data, int ndata)
437 {
438 raw_sendt(target, type, data, ndata*sizeof(int));
439 }
440 void
sendt(int target,int type,const char * data,int ndata)441 MessageGrp::sendt(int target, int type, const char* data, int ndata)
442 {
443 raw_sendt(target, type, data, ndata);
444 }
445 void
sendt(int target,int type,const unsigned char * data,int ndata)446 MessageGrp::sendt(int target, int type, const unsigned char* data, int ndata)
447 {
448 raw_sendt(target, type, data, ndata);
449 }
450 void
sendt(int target,int type,const signed char * data,int ndata)451 MessageGrp::sendt(int target, int type, const signed char* data, int ndata)
452 {
453 raw_sendt(target, type, data, ndata);
454 }
455
456 // Typed receive routines
457
458 void
recvt(int type,double * data,int ndata)459 MessageGrp::recvt(int type, double* data, int ndata)
460 {
461 raw_recvt(type, data, ndata*sizeof(double));
462 }
463 void
recvt(int type,short * data,int ndata)464 MessageGrp::recvt(int type, short* data, int ndata)
465 {
466 raw_recvt(type, data, ndata*sizeof(short));
467 }
468 void
recvt(int type,long * data,int ndata)469 MessageGrp::recvt(int type, long* data, int ndata)
470 {
471 raw_recvt(type, data, ndata*sizeof(long));
472 }
473 void
recvt(int type,float * data,int ndata)474 MessageGrp::recvt(int type, float* data, int ndata)
475 {
476 raw_recvt(type, data, ndata*sizeof(float));
477 }
478 void
recvt(int type,unsigned int * data,int ndata)479 MessageGrp::recvt(int type, unsigned int* data, int ndata)
480 {
481 raw_recvt(type, data, ndata*sizeof(int));
482 }
483 void
recvt(int type,int * data,int ndata)484 MessageGrp::recvt(int type, int* data, int ndata)
485 {
486 raw_recvt(type, data, ndata*sizeof(int));
487 }
488 void
recvt(int type,char * data,int ndata)489 MessageGrp::recvt(int type, char* data, int ndata)
490 {
491 raw_recvt(type, data, ndata);
492 }
493 void
recvt(int type,unsigned char * data,int ndata)494 MessageGrp::recvt(int type, unsigned char* data, int ndata)
495 {
496 raw_recvt(type, data, ndata);
497 }
498 void
recvt(int type,signed char * data,int ndata)499 MessageGrp::recvt(int type, signed char* data, int ndata)
500 {
501 raw_recvt(type, data, ndata);
502 }
503
504 // Broadcast operations
505
506 void
bcast(double * data,int ndata,int from)507 MessageGrp::bcast(double*data, int ndata, int from)
508 {
509 raw_bcast(data, ndata*sizeof(double), from);
510 }
511 void
bcast(short * data,int ndata,int from)512 MessageGrp::bcast(short*data, int ndata, int from)
513 {
514 raw_bcast(data, ndata*sizeof(short), from);
515 }
516 void
bcast(long * data,int ndata,int from)517 MessageGrp::bcast(long*data, int ndata, int from)
518 {
519 raw_bcast(data, ndata*sizeof(long), from);
520 }
521 void
bcast(float * data,int ndata,int from)522 MessageGrp::bcast(float*data, int ndata, int from)
523 {
524 raw_bcast(data, ndata*sizeof(float), from);
525 }
526 void
bcast(unsigned int * data,int ndata,int from)527 MessageGrp::bcast(unsigned int*data, int ndata, int from)
528 {
529 raw_bcast(data, ndata*sizeof(int), from);
530 }
531 void
bcast(int * data,int ndata,int from)532 MessageGrp::bcast(int*data, int ndata, int from)
533 {
534 raw_bcast(data, ndata*sizeof(int), from);
535 }
536 void
bcast(char * data,int ndata,int from)537 MessageGrp::bcast(char*data, int ndata, int from)
538 {
539 raw_bcast(data, ndata, from);
540 }
541 void
bcast(unsigned char * data,int ndata,int from)542 MessageGrp::bcast(unsigned char*data, int ndata, int from)
543 {
544 raw_bcast(data, ndata, from);
545 }
546 void
bcast(signed char * data,int ndata,int from)547 MessageGrp::bcast(signed char*data, int ndata, int from)
548 {
549 raw_bcast(data, ndata, from);
550 }
551
552 // Global classdesc indices
553
554 int
classdesc_to_index(const ClassDesc * cdptr)555 MessageGrp::classdesc_to_index(const ClassDesc* cdptr)
556 {
557 if (classdesc_to_index_.find((ClassDesc*)cdptr) != classdesc_to_index_.end()) {
558 return classdesc_to_index_[(ClassDesc*)cdptr];
559 }
560 else {
561 return -1;
562 }
563 }
564
565 const ClassDesc*
index_to_classdesc(int index)566 MessageGrp::index_to_classdesc(int index)
567 {
568 if (index < 0 || index >= nclass_) {
569 return 0;
570 }
571 else {
572 return index_to_classdesc_[index];
573 }
574 }
575
576 void
raw_bcast(void * data,int nbyte,int from)577 MessageGrp::raw_bcast(void* data, int nbyte, int from)
578 {
579 int nbyte_actual = nbyte;
580 int tgop_max = nbyte;
581 if (gop_max_ != 0) {
582 tgop_max = gop_max_;
583 gop_max_ = 0;
584 bcast(nbyte_actual,from);
585 gop_max_ = tgop_max;
586 }
587 for (int idat=0; idat<nbyte_actual; idat+=tgop_max) {
588 int ndat = (idat+tgop_max>nbyte_actual)?(nbyte_actual-idat):tgop_max;
589 Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, from));
590 for (i->forwards(); !i->done(); i->next()) {
591 if (i->send()) {
592 raw_send(i->sendto(), &((char*)data)[idat], ndat);
593 }
594 if (i->recv()) {
595 raw_recv(i->recvfrom(), &((char*)data)[idat], ndat);
596 }
597 }
598 }
599 }
600
601 void
sync()602 MessageGrp::sync()
603 {
604 Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, 0));
605
606 for (i->backwards(); !i->done(); i->next()) {
607 if (i->send()) {
608 raw_send(i->sendto(), 0, 0);
609 }
610 if (i->recv()) {
611 raw_recv(i->recvfrom(), 0, 0);
612 }
613 }
614
615 for (i->forwards(); !i->done(); i->next()) {
616 if (i->send()) {
617 raw_send(i->sendto(), 0, 0);
618 }
619 if (i->recv()) {
620 raw_recv(i->recvfrom(), 0, 0);
621 }
622 }
623 }
624
625 void
collect(const double * part,const int * lengths,double * whole)626 MessageGrp::collect(const double *part, const int *lengths, double *whole)
627 {
628 raw_collect(part,lengths,whole,sizeof(double));
629 }
630
631 void
raw_collect(const void * part,const int * lengths,void * whole,int bytes_per_datum)632 MessageGrp::raw_collect(const void *part, const int *lengths, void *whole,
633 int bytes_per_datum)
634 {
635 int offset = 0;
636 for (int i=0; i<n_; i++) {
637 int nbytes = lengths[i];
638 if (i==me_) memcpy(&((char*)whole)[offset], part, nbytes);
639 raw_bcast(&((char*)whole)[offset], nbytes, i);
640 offset += nbytes;
641 }
642 }
643
644 /////////////////////////////////////////////////////////////////////////////
645
646 // Local Variables:
647 // mode: c++
648 // c-file-style: "CLJ"
649 // End:
650