1 /*****************************************************************************
2  * Copyright (c) 2019 FrontISTR Commons
3  * This software is released under the MIT License, see LICENSE.txt
4  *****************************************************************************/
5 
6 #include "hecmw_vis_comm_util.h"
7 
8 #include "hecmw_vis_mem_util.h"
9 #include "hecmw_malloc.h"
10 
11 int HECMW_ANY_TAG = -1;
12 
whole_copy_array(int * recv_num,int * global_recv_num,int mynode,int pesize,HECMW_Comm repart_comm)13 void whole_copy_array(int *recv_num, int *global_recv_num, int mynode,
14                       int pesize, HECMW_Comm repart_comm) {
15   int i, j;
16   int *tmp_recv;
17   HECMW_Status stat;
18 
19   if (mynode == 0) {
20     for (j = 0; j < pesize + 1; j++) global_recv_num[j] = recv_num[j];
21     tmp_recv = (int *)HECMW_calloc(pesize + 1, sizeof(int));
22     if (tmp_recv == NULL) HECMW_vis_memory_exit("tmp_recv");
23     for (i = 1; i < pesize; i++) {
24       HECMW_Recv(tmp_recv, pesize + 1, HECMW_INT, i, HECMW_ANY_TAG, repart_comm,
25                  &stat);
26       for (j                                  = 0; j < pesize + 1; j++)
27         global_recv_num[i * (pesize + 1) + j] = tmp_recv[j];
28     }
29     for (i = 1; i < pesize; i++)
30       HECMW_Send(global_recv_num, (pesize + 1) * pesize, HECMW_INT, i, 0,
31                  repart_comm);
32     HECMW_free(tmp_recv);
33   } else {
34     HECMW_Send(recv_num, pesize + 1, HECMW_INT, 0, 0, repart_comm);
35     HECMW_Recv(global_recv_num, (pesize + 1) * pesize, HECMW_INT, 0,
36                HECMW_ANY_TAG, repart_comm, &stat);
37   }
38   return;
39 }
40 
stack_part_send_recv(int neibpetot,int * neibpe,int * stack_import,int * stack_export,HECMW_Comm repart_comm,int my_rank)41 int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import,
42                          int *stack_export, HECMW_Comm repart_comm,
43                          int my_rank) {
44   HECMW_Status *sta1, *sta2;
45   HECMW_Request *req1, *req2;
46 
47   int nflag = 0;
48   int neib;
49   int num;
50 
51   if (nflag == 0) {
52     sta1 =
53         (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
54     if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
55     sta2 =
56         (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
57     if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
58     if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
59                                               sizeof(HECMW_Request))) == NULL)
60       HECMW_vis_memory_exit("HECMW_STATUS: req1");
61     if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
62                                               sizeof(HECMW_Request))) == NULL)
63       HECMW_vis_memory_exit("HECMW_STATUS: req2");
64     nflag = 1;
65   }
66 
67   for (neib = 0; neib < neibpetot; neib++) {
68     num = stack_import[neib];
69     HECMW_Isend(&num, 1, HECMW_INT, neibpe[neib], 0, repart_comm, &req1[neib]);
70   }
71   for (neib = 0; neib < neibpetot; neib++) {
72     HECMW_Irecv(&stack_export[neib], 1, HECMW_INT, neibpe[neib], 0, repart_comm,
73                 &req2[neib]);
74     /*	fprintf(stderr, "PE %d recv %d from %d\n", my_rank, stack_export[neib],
75      * neib);
76      */
77   }
78   HECMW_Barrier(repart_comm);
79   HECMW_free(sta1);
80   HECMW_free(sta2);
81   HECMW_free(req1);
82   HECMW_free(req2);
83 
84   return 1;
85 }
86 
stack_whole_send_recv(int pesize,int * stack_export,int * stack_import,HECMW_Comm repart_comm,int my_rank)87 int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import,
88                           HECMW_Comm repart_comm, int my_rank) {
89   HECMW_Status stat;
90   int tmp_int;
91   int i, j;
92 
93   /*
94     HECMW_Status	*sta1, *sta2;
95 HECMW_Request	*req1, *req2;
96 
97 int	nflag = 0;
98 int	neib;
99 int	istart, num, num1;
100 int	k;
101 
102 stack_export[0]=0;
103 stack_export[my_rank+1]=stack_import[my_rank+1]-stack_import[my_rank];
104 
105 if (nflag == 0) {
106 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
107 sizeof(HECMW_Status));
108 if (sta1 == NULL)
109           HECMW_vis_memory_exit("HECMW_STATUS: stat1");
110 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
111 sizeof(HECMW_Status));
112 if (sta2 == NULL)
113           HECMW_vis_memory_exit("HECMW_STATUS: stat2");
114 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
115   == NULL)
116       HECMW_vis_memory_exit("HECMW_STATUS: req1");
117 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
118   == NULL)
119      HECMW_vis_memory_exit("HECMW_STATUS: req2");
120 nflag = 1;
121 }
122 
123 for(neib=0;neib<pesize;neib++) {
124     if(neib!=my_rank) {
125     num=stack_import[neib+1]-stack_import[neib];
126 HECMW_Isend(&num, 1, HECMW_INT,
127         neib, 0, repart_comm, &req1[neib]);
128   fprintf(stderr, "pe %d send %d to %d\n", my_rank, num, neib);
129 }
130 }
131 for (neib = 0; neib < pesize; neib++) {
132     if(neib!=my_rank) {
133 HECMW_Irecv(&stack_export[neib+1], 1, HECMW_INT,
134         neib, 0, repart_comm, &req2[neib]);
135 
136   fprintf(stderr, "pe %d recv %d from %d\n", my_rank, stack_export[neib+1],
137 neib);
138 
139 
140     }
141 }
142 
143 
144 HECMW_Barrier(repart_comm);
145 
146 HECMW_free(sta1);
147 HECMW_free(sta2);
148 HECMW_free(req1);
149 HECMW_free(req2);
150    */
151   for (i = 0; i < pesize; i++) {
152     if (i != my_rank) {
153       tmp_int = stack_export[i + 1] - stack_export[i];
154       HECMW_Send(&tmp_int, 1, HECMW_INT, i, 0, repart_comm);
155     } else if (i == my_rank) {
156       stack_import[i + 1] = stack_export[i + 1] - stack_export[i];
157       for (j = 0; j < pesize; j++) {
158         if (j != my_rank) {
159           HECMW_Recv(&stack_import[j + 1], 1, HECMW_INT, j, HECMW_ANY_TAG,
160                      repart_comm, &stat);
161         }
162       }
163       stack_import[0] = 0;
164       for (j            = 1; j < pesize + 1; j++)
165         stack_import[j] = stack_import[j] + stack_import[j - 1];
166     }
167   }
168 
169   HECMW_Barrier(repart_comm);
170 
171   return 1;
172 }
173 
int_part_send_recv(int n,int neibpetot,int * neibpe,int * stack_import,int * nod_import,int * stack_export,int * nod_export,int * x,HECMW_Comm repart_comm,int my_rank)174 int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
175                        int *nod_import, int *stack_export, int *nod_export,
176                        int *x, HECMW_Comm repart_comm, int my_rank) {
177   /* Important:: node ID in nod_import and nod_export are all starting from 1 */
178 
179   HECMW_Status *sta1, *sta2;
180   HECMW_Request *req1, *req2;
181 
182   int nflag = 0;
183   int neib;
184   int inum;
185   int k;
186   int *ws, *wr;
187 
188   ws = (int *)HECMW_calloc(n, sizeof(int));
189   wr = (int *)HECMW_calloc(n, sizeof(int));
190   if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
191   if (nflag == 0) {
192     sta1 =
193         (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
194     if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
195     sta2 =
196         (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
197     if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
198     if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
199                                               sizeof(HECMW_Request))) == NULL)
200       HECMW_vis_memory_exit("HECMW_STATUS: req1");
201     if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
202                                               sizeof(HECMW_Request))) == NULL)
203       HECMW_vis_memory_exit("HECMW_STATUS: req2");
204     nflag = 1;
205   }
206   /* SEND */
207   for (neib = 0; neib < neibpetot; neib++) {
208     /*    if (neib != 0) istart = stack_export[neib - 1];
209 else istart = 0;
210      */
211     inum = stack_export[neib + 1] - stack_export[neib];
212 
213     for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
214       ws[k] = x[nod_export[k] - 1];
215     }
216     HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neibpe[neib], 0,
217                 repart_comm, &req1[neib]);
218   }
219 
220   /* RECEIVE */
221   for (neib = 0; neib < neibpetot; neib++) {
222     inum = stack_import[neib + 1] - stack_import[neib];
223     HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neibpe[neib], 0,
224                 repart_comm, &req2[neib]);
225   }
226   HECMW_Barrier(repart_comm);
227   /*
228 HECMW_Waitall(neibpetot, req2, sta2);
229    */
230   for (neib = 0; neib < neibpetot; neib++) {
231     /*    if (neib != 0) istart = stack_import[neib-1];
232 else istart = 0;
233      */
234     inum = stack_import[neib + 1] - stack_import[neib];
235     for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
236       x[nod_import[k] - 1] = wr[k];
237     }
238   }
239   HECMW_Barrier(repart_comm);
240   /*
241 HECMW_Waitall(neibpetot, req1, sta1);
242    */
243   HECMW_free(sta1);
244   HECMW_free(sta2);
245   HECMW_free(req1);
246   HECMW_free(req2);
247   HECMW_free(ws);
248   HECMW_free(wr);
249   return 1;
250 }
251 
double_part_send_recv(int n,int neibpetot,int * neibpe,int * stack_import,int * nod_import,int * stack_export,int * nod_export,double * x,HECMW_Comm repart_comm,int my_rank)252 int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
253                           int *nod_import, int *stack_export, int *nod_export,
254                           double *x, HECMW_Comm repart_comm, int my_rank) {
255   HECMW_Status *sta1, *sta2;
256   HECMW_Request *req1, *req2;
257 
258   int nflag = 0;
259   int neib;
260   int inum;
261   int k;
262   double *ws, *wr;
263 
264   ws = (double *)HECMW_calloc(n, sizeof(double));
265   wr = (double *)HECMW_calloc(n, sizeof(double));
266   if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
267   if (nflag == 0) {
268     sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
269                                         sizeof(HECMW_Status));
270     if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
271     sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
272                                         sizeof(HECMW_Status));
273     if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta12");
274     if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
275                                               sizeof(HECMW_Request))) == NULL)
276       HECMW_vis_memory_exit("send_recv: req1");
277     if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
278                                               sizeof(HECMW_Request))) == NULL)
279       HECMW_vis_memory_exit("send_recv: req2");
280     nflag = 1;
281   }
282   /* SEND */
283   for (neib = 0; neib < neibpetot; neib++) {
284     /*    if (neib != 0) istart = stack_export[neib - 1];
285 else istart = 0;
286      */
287     inum = stack_export[neib + 1] - stack_export[neib];
288 
289     for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
290       ws[k] = x[nod_export[k] - 1];
291     }
292     if (inum > 0) {
293       /*	fprintf(stderr, "PE %d send %d node to PE %d\n", my_rank, inum,
294        * neibpe[neib]);
295        */
296       HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
297                   repart_comm, &req1[neib]);
298     }
299   }
300 
301   /* RECEIVE */
302   for (neib = 0; neib < neibpetot; neib++) {
303     inum = stack_import[neib + 1] - stack_import[neib];
304     if (inum > 0) {
305       /*		fprintf(stderr, "PE %d recieve %d node from PE %d\n",
306        * my_rank, inum, neibpe[neib]);
307        */
308       HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
309                   repart_comm, &req2[neib]);
310     }
311   }
312   HECMW_Barrier(repart_comm);
313   /*  HECMW_Waitall(neibpetot, req2, sta2);
314    */
315   for (neib = 0; neib < neibpetot; neib++) {
316     /*    if (neib != 0) istart = stack_import[neib-1];
317 else istart = 0;
318      */
319     inum = stack_import[neib + 1] - stack_import[neib];
320     for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
321       x[nod_import[k] - 1] = wr[k];
322     }
323   }
324   HECMW_Barrier(repart_comm);
325 
326   /*  HECMW_Waitall(neibpetot, req1, sta1);
327    */
328   HECMW_free(sta1);
329   HECMW_free(sta2);
330   HECMW_free(req1);
331   HECMW_free(req2);
332   HECMW_free(ws);
333   HECMW_free(wr);
334   return 1;
335 }
336 
int_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * nod_import,int * stack_export,int * nod_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)337 void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
338                          int *nod_import, int *stack_export, int *nod_export,
339                          int *x, int *y, HECMW_Comm repart_comm, int my_rank) {
340   /* Important:: node ID in nod_import and nod_export are all starting from 0 */
341 
342   /*  HECMW_Status	*sta1, *sta2;
343    */
344   HECMW_Request *req1, *req2;
345 
346   int nflag = 0;
347   int neib;
348   int inum;
349   int k;
350   int *ws, *wr;
351 
352   ws = (int *)HECMW_calloc(n1, sizeof(int));
353   wr = (int *)HECMW_calloc(n2, sizeof(int));
354   if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("ws, wr");
355   if (nflag == 0) {
356     /*    sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
357 sizeof(HECMW_Status));
358 if (sta1 == NULL) {
359 fprintf(stderr, "Not enough memory\n");
360 exit(1);
361 }
362 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
363 if (sta2 == NULL) {
364 fprintf(stderr, "Not enough memory\n");
365 exit(1);
366 }
367      */
368     if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
369         NULL)
370       HECMW_vis_memory_exit("send_recv: req1");
371     if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
372         NULL)
373       HECMW_vis_memory_exit("send_recv: req2");
374     nflag = 1;
375   }
376   /* SEND */
377   for (neib = 0; neib < pesize; neib++) {
378     /*    if (neib != 0) istart = stack_export[neib - 1];
379 else istart = 0;
380      */
381     if (neib != my_rank) {
382       inum = stack_export[neib + 1] - stack_export[neib];
383 
384       for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
385         ws[k] = x[nod_export[k]];
386       }
387       HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neib, 0,
388                   repart_comm, &req1[neib]);
389     }
390   }
391 
392   /* RECEIVE */
393   for (neib = 0; neib < pesize; neib++) {
394     /*    if (neib != 0) istart = stack_import[neib-1];
395 else istart = 0;
396      */
397     inum = stack_import[neib + 1] - stack_import[neib];
398     if (neib != my_rank)
399       HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neib, 0,
400                   repart_comm, &req2[neib]);
401   }
402   HECMW_Barrier(repart_comm);
403   /*
404 HECMW_Waitall(neibpetot, req2, sta2);
405    */
406   for (neib = 0; neib < pesize; neib++) {
407     /*    if (neib != 0) istart = stack_import[neib-1];
408 else istart = 0;
409      */
410     inum = stack_import[neib + 1] - stack_import[neib];
411     if (neib != my_rank) {
412       for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
413         y[nod_import[k]] = wr[k];
414     } else {
415       for (k = 0; k < inum; k++)
416         /*		for (k = stack_import[neib]; k < stack_import[neib]+inum;
417          * k++)
418          */
419         y[nod_import[stack_import[neib] + k]] =
420             x[nod_export[stack_export[neib] + k]];
421     }
422   }
423   HECMW_Barrier(repart_comm);
424   /*
425 HECMW_Waitall(neibpetot, req1, sta1);
426 
427 HECMW_free(sta1);
428 HECMW_free(sta2);
429    */
430   HECMW_free(req1);
431   HECMW_free(req2);
432   HECMW_free(ws);
433   HECMW_free(wr);
434   return;
435 }
436 
int2_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)437 void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
438                           int *stack_export, int *x, int *y,
439                           HECMW_Comm repart_comm, int my_rank) {
440   /* Important:: node ID in nod_import and nod_export are all starting from 0 */
441 
442   HECMW_Status *sta1, *sta2;
443   HECMW_Request *req1, *req2;
444 
445   int nflag = 0;
446   int neib;
447   int inum;
448   int k;
449 
450   if (nflag == 0) {
451     sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
452                                         sizeof(HECMW_Status));
453     if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
454     sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
455                                         sizeof(HECMW_Status));
456     if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
457 
458     if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
459         NULL)
460       HECMW_vis_memory_exit("send_recv: req1");
461     if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
462         NULL)
463       HECMW_vis_memory_exit("send_recv: req2");
464     nflag = 1;
465   }
466   /* SEND */
467   for (neib = 0; neib < pesize; neib++) {
468     if (neib != my_rank) {
469       inum = stack_export[neib + 1] - stack_export[neib];
470       if (inum != 0) {
471         HECMW_Isend(&x[stack_export[neib]], inum, HECMW_INT, neib, 0,
472                     repart_comm, &req1[neib]);
473         /*		  fprintf(stderr, "sending from PE %d to PE %d nodes
474          * %d\n", my_rank, neib, inum);
475          */
476       }
477     }
478   }
479 
480   /* RECEIVE */
481   for (neib = 0; neib < pesize; neib++) {
482     /*    if (neib != 0) istart = stack_import[neib-1];
483 else istart = 0;
484      */
485     inum = stack_import[neib + 1] - stack_import[neib];
486     if ((neib != my_rank) && (inum != 0)) {
487       HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_INT, neib, 0, repart_comm,
488                   &req2[neib]);
489       /*	fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
490        * neib, inum);
491        */
492     }
493   }
494 
495   HECMW_Waitall(pesize, req2, sta2);
496 
497   inum = stack_import[my_rank + 1] - stack_import[my_rank];
498   for (k                         = 0; k < inum; k++)
499     y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
500 
501   HECMW_Waitall(pesize, req1, sta1);
502 
503   HECMW_free(sta1);
504   HECMW_free(sta2);
505 
506   HECMW_free(req1);
507   HECMW_free(req2);
508   return;
509 }
510 
double2_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,double * x,double * y,HECMW_Comm repart_comm,int my_rank)511 void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
512                              int *stack_export, double *x, double *y,
513                              HECMW_Comm repart_comm, int my_rank) {
514   /* Important:: node ID in nod_import and nod_export are all starting from 0 */
515 
516   HECMW_Status *sta1, *sta2;
517   HECMW_Request *req1, *req2;
518 
519   int nflag = 0;
520   int neib;
521   int inum;
522   int k;
523 
524   if (nflag == 0) {
525     sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
526                                         sizeof(HECMW_Status));
527     if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
528     sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
529                                         sizeof(HECMW_Status));
530     if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
531 
532     if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
533         NULL)
534       HECMW_vis_memory_exit("send_recv: req1");
535     if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
536         NULL)
537       HECMW_vis_memory_exit("send_recv: req2");
538     nflag = 1;
539   }
540   /* SEND */
541   for (neib = 0; neib < pesize; neib++) {
542     if (neib != my_rank) {
543       inum = stack_export[neib + 1] - stack_export[neib];
544       if (inum != 0) {
545         /*		  fprintf(stderr, "sending from PE %d to PE %d nodes
546          * %d\n", my_rank, neib, inum);
547          */
548         HECMW_Isend(&x[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
549                     repart_comm, &req1[neib]);
550       }
551     }
552   }
553 
554   /* RECEIVE */
555   for (neib = 0; neib < pesize; neib++) {
556     /*    if (neib != 0) istart = stack_import[neib-1];
557 else istart = 0;
558      */
559     inum = stack_import[neib + 1] - stack_import[neib];
560     if ((neib != my_rank) && (inum != 0)) {
561       /*	fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
562        * neib, inum);
563        */
564       HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_DOUBLE, neib, 0,
565                   repart_comm, &req2[neib]);
566     }
567   }
568 
569   HECMW_Waitall(pesize, req2, sta2);
570 
571   inum = stack_import[my_rank + 1] - stack_import[my_rank];
572   for (k                         = 0; k < inum; k++)
573     y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
574 
575   HECMW_Waitall(pesize, req1, sta1);
576 
577   HECMW_free(sta1);
578   HECMW_free(sta2);
579 
580   HECMW_free(req1);
581   HECMW_free(req2);
582   return;
583 }
584 
int3_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)585 void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
586                           int *stack_export, int *x, int *y,
587                           HECMW_Comm repart_comm, int my_rank) {
588   /* Important:: node ID in nod_import and nod_export are all starting from 0 */
589 
590   HECMW_Status stat;
591 
592   int neib;
593   int inum;
594   int i, j, k;
595 
596   for (i = 0; i < pesize; i++) {
597     if (i != my_rank) {
598       inum = stack_export[i + 1] - stack_export[i];
599       if (inum != 0)
600         HECMW_Send(&x[stack_export[neib]], inum, HECMW_INT, i, 0, repart_comm);
601     } else if (i == my_rank) {
602       inum = stack_import[my_rank + 1] - stack_import[my_rank];
603       for (k                         = 0; k < inum; k++)
604         y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
605       for (j = 0; j < pesize; j++) {
606         if (j != my_rank) {
607           inum = stack_import[j + 1] - stack_import[j];
608           if (inum != 0)
609 
610             HECMW_Recv(&y[stack_import[j]], inum, HECMW_INT, j, HECMW_ANY_TAG,
611                        repart_comm, &stat);
612         }
613       }
614     }
615   }
616 
617   HECMW_Barrier(repart_comm);
618 
619   return;
620 }
621 
double_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * nod_import,int * stack_export,int * nod_export,double * x,double * y,HECMW_Comm repart_comm,int my_rank)622 void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
623                             int *nod_import, int *stack_export, int *nod_export,
624                             double *x, double *y, HECMW_Comm repart_comm,
625                             int my_rank) {
626   /*  HECMW_Status	*sta1, *sta2;
627    */
628   HECMW_Request *req1, *req2;
629 
630   int nflag = 0;
631   int neib;
632   int inum;
633   int k;
634   double *ws, *wr;
635 
636   ws = (double *)HECMW_calloc(n1, sizeof(int));
637   wr = (double *)HECMW_calloc(n2, sizeof(int));
638   if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws,wr");
639   if (nflag == 0) {
640     /*    sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
641 sizeof(HECMW_Status));
642 if (sta1 == NULL) {
643 fprintf(stderr, "Not enough memory\n");
644 exit(1);
645 }
646 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
647 if (sta2 == NULL) {
648 fprintf(stderr, "Not enough memory\n");
649 exit(1);
650 }
651      */
652     if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
653         NULL)
654       HECMW_vis_memory_exit("send_recv: req1");
655     if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
656         NULL)
657       HECMW_vis_memory_exit("send_recv: req2");
658     nflag = 1;
659   }
660   /* SEND */
661   for (neib = 0; neib < pesize; neib++) {
662     /*    if (neib != 0) istart = stack_export[neib - 1];
663 else istart = 0;
664      */
665     if (neib != my_rank) {
666       inum = stack_export[neib + 1] - stack_export[neib];
667 
668       for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
669         ws[k] = x[nod_export[k] - 1];
670       }
671       HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
672                   repart_comm, &req1[neib]);
673     }
674   }
675 
676   /* RECEIVE */
677   for (neib = 0; neib < pesize; neib++) {
678     /*    if (neib != 0) istart = stack_import[neib-1];
679 else istart = 0;
680      */
681 
682     inum = stack_import[neib + 1] - stack_import[neib];
683     if (neib != my_rank) {
684       for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
685         y[k] = wr[k];
686     } else {
687       for (k = 0; k < inum; k++)
688         /*		for (k = stack_import[neib]; k < stack_import[neib]+inum;
689          * k++)
690          */
691         y[stack_import[neib] + k] = x[stack_export[neib] + k];
692     }
693   }
694 
695   HECMW_Barrier(repart_comm);
696   /*
697 HECMW_Waitall(neibpetot, req1, sta1);
698 
699 HECMW_free(sta1);
700 HECMW_free(sta2);
701    */
702   HECMW_free(req1);
703   HECMW_free(req2);
704   HECMW_free(ws);
705   HECMW_free(wr);
706   return;
707 }
708