1 /*****************************************************************************
2 * Copyright (c) 2019 FrontISTR Commons
3 * This software is released under the MIT License, see LICENSE.txt
4 *****************************************************************************/
5
6 #include "hecmw_vis_comm_util.h"
7
8 #include "hecmw_vis_mem_util.h"
9 #include "hecmw_malloc.h"
10
11 int HECMW_ANY_TAG = -1;
12
whole_copy_array(int * recv_num,int * global_recv_num,int mynode,int pesize,HECMW_Comm repart_comm)13 void whole_copy_array(int *recv_num, int *global_recv_num, int mynode,
14 int pesize, HECMW_Comm repart_comm) {
15 int i, j;
16 int *tmp_recv;
17 HECMW_Status stat;
18
19 if (mynode == 0) {
20 for (j = 0; j < pesize + 1; j++) global_recv_num[j] = recv_num[j];
21 tmp_recv = (int *)HECMW_calloc(pesize + 1, sizeof(int));
22 if (tmp_recv == NULL) HECMW_vis_memory_exit("tmp_recv");
23 for (i = 1; i < pesize; i++) {
24 HECMW_Recv(tmp_recv, pesize + 1, HECMW_INT, i, HECMW_ANY_TAG, repart_comm,
25 &stat);
26 for (j = 0; j < pesize + 1; j++)
27 global_recv_num[i * (pesize + 1) + j] = tmp_recv[j];
28 }
29 for (i = 1; i < pesize; i++)
30 HECMW_Send(global_recv_num, (pesize + 1) * pesize, HECMW_INT, i, 0,
31 repart_comm);
32 HECMW_free(tmp_recv);
33 } else {
34 HECMW_Send(recv_num, pesize + 1, HECMW_INT, 0, 0, repart_comm);
35 HECMW_Recv(global_recv_num, (pesize + 1) * pesize, HECMW_INT, 0,
36 HECMW_ANY_TAG, repart_comm, &stat);
37 }
38 return;
39 }
40
stack_part_send_recv(int neibpetot,int * neibpe,int * stack_import,int * stack_export,HECMW_Comm repart_comm,int my_rank)41 int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import,
42 int *stack_export, HECMW_Comm repart_comm,
43 int my_rank) {
44 HECMW_Status *sta1, *sta2;
45 HECMW_Request *req1, *req2;
46
47 int nflag = 0;
48 int neib;
49 int num;
50
51 if (nflag == 0) {
52 sta1 =
53 (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
54 if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
55 sta2 =
56 (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
57 if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
58 if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
59 sizeof(HECMW_Request))) == NULL)
60 HECMW_vis_memory_exit("HECMW_STATUS: req1");
61 if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
62 sizeof(HECMW_Request))) == NULL)
63 HECMW_vis_memory_exit("HECMW_STATUS: req2");
64 nflag = 1;
65 }
66
67 for (neib = 0; neib < neibpetot; neib++) {
68 num = stack_import[neib];
69 HECMW_Isend(&num, 1, HECMW_INT, neibpe[neib], 0, repart_comm, &req1[neib]);
70 }
71 for (neib = 0; neib < neibpetot; neib++) {
72 HECMW_Irecv(&stack_export[neib], 1, HECMW_INT, neibpe[neib], 0, repart_comm,
73 &req2[neib]);
74 /* fprintf(stderr, "PE %d recv %d from %d\n", my_rank, stack_export[neib],
75 * neib);
76 */
77 }
78 HECMW_Barrier(repart_comm);
79 HECMW_free(sta1);
80 HECMW_free(sta2);
81 HECMW_free(req1);
82 HECMW_free(req2);
83
84 return 1;
85 }
86
stack_whole_send_recv(int pesize,int * stack_export,int * stack_import,HECMW_Comm repart_comm,int my_rank)87 int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import,
88 HECMW_Comm repart_comm, int my_rank) {
89 HECMW_Status stat;
90 int tmp_int;
91 int i, j;
92
93 /*
94 HECMW_Status *sta1, *sta2;
95 HECMW_Request *req1, *req2;
96
97 int nflag = 0;
98 int neib;
99 int istart, num, num1;
100 int k;
101
102 stack_export[0]=0;
103 stack_export[my_rank+1]=stack_import[my_rank+1]-stack_import[my_rank];
104
105 if (nflag == 0) {
106 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
107 sizeof(HECMW_Status));
108 if (sta1 == NULL)
109 HECMW_vis_memory_exit("HECMW_STATUS: stat1");
110 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
111 sizeof(HECMW_Status));
112 if (sta2 == NULL)
113 HECMW_vis_memory_exit("HECMW_STATUS: stat2");
114 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
115 == NULL)
116 HECMW_vis_memory_exit("HECMW_STATUS: req1");
117 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
118 == NULL)
119 HECMW_vis_memory_exit("HECMW_STATUS: req2");
120 nflag = 1;
121 }
122
123 for(neib=0;neib<pesize;neib++) {
124 if(neib!=my_rank) {
125 num=stack_import[neib+1]-stack_import[neib];
126 HECMW_Isend(&num, 1, HECMW_INT,
127 neib, 0, repart_comm, &req1[neib]);
128 fprintf(stderr, "pe %d send %d to %d\n", my_rank, num, neib);
129 }
130 }
131 for (neib = 0; neib < pesize; neib++) {
132 if(neib!=my_rank) {
133 HECMW_Irecv(&stack_export[neib+1], 1, HECMW_INT,
134 neib, 0, repart_comm, &req2[neib]);
135
136 fprintf(stderr, "pe %d recv %d from %d\n", my_rank, stack_export[neib+1],
137 neib);
138
139
140 }
141 }
142
143
144 HECMW_Barrier(repart_comm);
145
146 HECMW_free(sta1);
147 HECMW_free(sta2);
148 HECMW_free(req1);
149 HECMW_free(req2);
150 */
151 for (i = 0; i < pesize; i++) {
152 if (i != my_rank) {
153 tmp_int = stack_export[i + 1] - stack_export[i];
154 HECMW_Send(&tmp_int, 1, HECMW_INT, i, 0, repart_comm);
155 } else if (i == my_rank) {
156 stack_import[i + 1] = stack_export[i + 1] - stack_export[i];
157 for (j = 0; j < pesize; j++) {
158 if (j != my_rank) {
159 HECMW_Recv(&stack_import[j + 1], 1, HECMW_INT, j, HECMW_ANY_TAG,
160 repart_comm, &stat);
161 }
162 }
163 stack_import[0] = 0;
164 for (j = 1; j < pesize + 1; j++)
165 stack_import[j] = stack_import[j] + stack_import[j - 1];
166 }
167 }
168
169 HECMW_Barrier(repart_comm);
170
171 return 1;
172 }
173
int_part_send_recv(int n,int neibpetot,int * neibpe,int * stack_import,int * nod_import,int * stack_export,int * nod_export,int * x,HECMW_Comm repart_comm,int my_rank)174 int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
175 int *nod_import, int *stack_export, int *nod_export,
176 int *x, HECMW_Comm repart_comm, int my_rank) {
177 /* Important:: node ID in nod_import and nod_export are all starting from 1 */
178
179 HECMW_Status *sta1, *sta2;
180 HECMW_Request *req1, *req2;
181
182 int nflag = 0;
183 int neib;
184 int inum;
185 int k;
186 int *ws, *wr;
187
188 ws = (int *)HECMW_calloc(n, sizeof(int));
189 wr = (int *)HECMW_calloc(n, sizeof(int));
190 if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
191 if (nflag == 0) {
192 sta1 =
193 (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
194 if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
195 sta2 =
196 (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
197 if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
198 if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
199 sizeof(HECMW_Request))) == NULL)
200 HECMW_vis_memory_exit("HECMW_STATUS: req1");
201 if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
202 sizeof(HECMW_Request))) == NULL)
203 HECMW_vis_memory_exit("HECMW_STATUS: req2");
204 nflag = 1;
205 }
206 /* SEND */
207 for (neib = 0; neib < neibpetot; neib++) {
208 /* if (neib != 0) istart = stack_export[neib - 1];
209 else istart = 0;
210 */
211 inum = stack_export[neib + 1] - stack_export[neib];
212
213 for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
214 ws[k] = x[nod_export[k] - 1];
215 }
216 HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neibpe[neib], 0,
217 repart_comm, &req1[neib]);
218 }
219
220 /* RECEIVE */
221 for (neib = 0; neib < neibpetot; neib++) {
222 inum = stack_import[neib + 1] - stack_import[neib];
223 HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neibpe[neib], 0,
224 repart_comm, &req2[neib]);
225 }
226 HECMW_Barrier(repart_comm);
227 /*
228 HECMW_Waitall(neibpetot, req2, sta2);
229 */
230 for (neib = 0; neib < neibpetot; neib++) {
231 /* if (neib != 0) istart = stack_import[neib-1];
232 else istart = 0;
233 */
234 inum = stack_import[neib + 1] - stack_import[neib];
235 for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
236 x[nod_import[k] - 1] = wr[k];
237 }
238 }
239 HECMW_Barrier(repart_comm);
240 /*
241 HECMW_Waitall(neibpetot, req1, sta1);
242 */
243 HECMW_free(sta1);
244 HECMW_free(sta2);
245 HECMW_free(req1);
246 HECMW_free(req2);
247 HECMW_free(ws);
248 HECMW_free(wr);
249 return 1;
250 }
251
double_part_send_recv(int n,int neibpetot,int * neibpe,int * stack_import,int * nod_import,int * stack_export,int * nod_export,double * x,HECMW_Comm repart_comm,int my_rank)252 int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
253 int *nod_import, int *stack_export, int *nod_export,
254 double *x, HECMW_Comm repart_comm, int my_rank) {
255 HECMW_Status *sta1, *sta2;
256 HECMW_Request *req1, *req2;
257
258 int nflag = 0;
259 int neib;
260 int inum;
261 int k;
262 double *ws, *wr;
263
264 ws = (double *)HECMW_calloc(n, sizeof(double));
265 wr = (double *)HECMW_calloc(n, sizeof(double));
266 if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
267 if (nflag == 0) {
268 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
269 sizeof(HECMW_Status));
270 if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
271 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
272 sizeof(HECMW_Status));
273 if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta12");
274 if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
275 sizeof(HECMW_Request))) == NULL)
276 HECMW_vis_memory_exit("send_recv: req1");
277 if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
278 sizeof(HECMW_Request))) == NULL)
279 HECMW_vis_memory_exit("send_recv: req2");
280 nflag = 1;
281 }
282 /* SEND */
283 for (neib = 0; neib < neibpetot; neib++) {
284 /* if (neib != 0) istart = stack_export[neib - 1];
285 else istart = 0;
286 */
287 inum = stack_export[neib + 1] - stack_export[neib];
288
289 for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
290 ws[k] = x[nod_export[k] - 1];
291 }
292 if (inum > 0) {
293 /* fprintf(stderr, "PE %d send %d node to PE %d\n", my_rank, inum,
294 * neibpe[neib]);
295 */
296 HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
297 repart_comm, &req1[neib]);
298 }
299 }
300
301 /* RECEIVE */
302 for (neib = 0; neib < neibpetot; neib++) {
303 inum = stack_import[neib + 1] - stack_import[neib];
304 if (inum > 0) {
305 /* fprintf(stderr, "PE %d recieve %d node from PE %d\n",
306 * my_rank, inum, neibpe[neib]);
307 */
308 HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
309 repart_comm, &req2[neib]);
310 }
311 }
312 HECMW_Barrier(repart_comm);
313 /* HECMW_Waitall(neibpetot, req2, sta2);
314 */
315 for (neib = 0; neib < neibpetot; neib++) {
316 /* if (neib != 0) istart = stack_import[neib-1];
317 else istart = 0;
318 */
319 inum = stack_import[neib + 1] - stack_import[neib];
320 for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
321 x[nod_import[k] - 1] = wr[k];
322 }
323 }
324 HECMW_Barrier(repart_comm);
325
326 /* HECMW_Waitall(neibpetot, req1, sta1);
327 */
328 HECMW_free(sta1);
329 HECMW_free(sta2);
330 HECMW_free(req1);
331 HECMW_free(req2);
332 HECMW_free(ws);
333 HECMW_free(wr);
334 return 1;
335 }
336
int_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * nod_import,int * stack_export,int * nod_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)337 void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
338 int *nod_import, int *stack_export, int *nod_export,
339 int *x, int *y, HECMW_Comm repart_comm, int my_rank) {
340 /* Important:: node ID in nod_import and nod_export are all starting from 0 */
341
342 /* HECMW_Status *sta1, *sta2;
343 */
344 HECMW_Request *req1, *req2;
345
346 int nflag = 0;
347 int neib;
348 int inum;
349 int k;
350 int *ws, *wr;
351
352 ws = (int *)HECMW_calloc(n1, sizeof(int));
353 wr = (int *)HECMW_calloc(n2, sizeof(int));
354 if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("ws, wr");
355 if (nflag == 0) {
356 /* sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
357 sizeof(HECMW_Status));
358 if (sta1 == NULL) {
359 fprintf(stderr, "Not enough memory\n");
360 exit(1);
361 }
362 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
363 if (sta2 == NULL) {
364 fprintf(stderr, "Not enough memory\n");
365 exit(1);
366 }
367 */
368 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
369 NULL)
370 HECMW_vis_memory_exit("send_recv: req1");
371 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
372 NULL)
373 HECMW_vis_memory_exit("send_recv: req2");
374 nflag = 1;
375 }
376 /* SEND */
377 for (neib = 0; neib < pesize; neib++) {
378 /* if (neib != 0) istart = stack_export[neib - 1];
379 else istart = 0;
380 */
381 if (neib != my_rank) {
382 inum = stack_export[neib + 1] - stack_export[neib];
383
384 for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
385 ws[k] = x[nod_export[k]];
386 }
387 HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neib, 0,
388 repart_comm, &req1[neib]);
389 }
390 }
391
392 /* RECEIVE */
393 for (neib = 0; neib < pesize; neib++) {
394 /* if (neib != 0) istart = stack_import[neib-1];
395 else istart = 0;
396 */
397 inum = stack_import[neib + 1] - stack_import[neib];
398 if (neib != my_rank)
399 HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neib, 0,
400 repart_comm, &req2[neib]);
401 }
402 HECMW_Barrier(repart_comm);
403 /*
404 HECMW_Waitall(neibpetot, req2, sta2);
405 */
406 for (neib = 0; neib < pesize; neib++) {
407 /* if (neib != 0) istart = stack_import[neib-1];
408 else istart = 0;
409 */
410 inum = stack_import[neib + 1] - stack_import[neib];
411 if (neib != my_rank) {
412 for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
413 y[nod_import[k]] = wr[k];
414 } else {
415 for (k = 0; k < inum; k++)
416 /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
417 * k++)
418 */
419 y[nod_import[stack_import[neib] + k]] =
420 x[nod_export[stack_export[neib] + k]];
421 }
422 }
423 HECMW_Barrier(repart_comm);
424 /*
425 HECMW_Waitall(neibpetot, req1, sta1);
426
427 HECMW_free(sta1);
428 HECMW_free(sta2);
429 */
430 HECMW_free(req1);
431 HECMW_free(req2);
432 HECMW_free(ws);
433 HECMW_free(wr);
434 return;
435 }
436
int2_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)437 void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
438 int *stack_export, int *x, int *y,
439 HECMW_Comm repart_comm, int my_rank) {
440 /* Important:: node ID in nod_import and nod_export are all starting from 0 */
441
442 HECMW_Status *sta1, *sta2;
443 HECMW_Request *req1, *req2;
444
445 int nflag = 0;
446 int neib;
447 int inum;
448 int k;
449
450 if (nflag == 0) {
451 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
452 sizeof(HECMW_Status));
453 if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
454 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
455 sizeof(HECMW_Status));
456 if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
457
458 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
459 NULL)
460 HECMW_vis_memory_exit("send_recv: req1");
461 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
462 NULL)
463 HECMW_vis_memory_exit("send_recv: req2");
464 nflag = 1;
465 }
466 /* SEND */
467 for (neib = 0; neib < pesize; neib++) {
468 if (neib != my_rank) {
469 inum = stack_export[neib + 1] - stack_export[neib];
470 if (inum != 0) {
471 HECMW_Isend(&x[stack_export[neib]], inum, HECMW_INT, neib, 0,
472 repart_comm, &req1[neib]);
473 /* fprintf(stderr, "sending from PE %d to PE %d nodes
474 * %d\n", my_rank, neib, inum);
475 */
476 }
477 }
478 }
479
480 /* RECEIVE */
481 for (neib = 0; neib < pesize; neib++) {
482 /* if (neib != 0) istart = stack_import[neib-1];
483 else istart = 0;
484 */
485 inum = stack_import[neib + 1] - stack_import[neib];
486 if ((neib != my_rank) && (inum != 0)) {
487 HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_INT, neib, 0, repart_comm,
488 &req2[neib]);
489 /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
490 * neib, inum);
491 */
492 }
493 }
494
495 HECMW_Waitall(pesize, req2, sta2);
496
497 inum = stack_import[my_rank + 1] - stack_import[my_rank];
498 for (k = 0; k < inum; k++)
499 y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
500
501 HECMW_Waitall(pesize, req1, sta1);
502
503 HECMW_free(sta1);
504 HECMW_free(sta2);
505
506 HECMW_free(req1);
507 HECMW_free(req2);
508 return;
509 }
510
double2_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,double * x,double * y,HECMW_Comm repart_comm,int my_rank)511 void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
512 int *stack_export, double *x, double *y,
513 HECMW_Comm repart_comm, int my_rank) {
514 /* Important:: node ID in nod_import and nod_export are all starting from 0 */
515
516 HECMW_Status *sta1, *sta2;
517 HECMW_Request *req1, *req2;
518
519 int nflag = 0;
520 int neib;
521 int inum;
522 int k;
523
524 if (nflag == 0) {
525 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
526 sizeof(HECMW_Status));
527 if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
528 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
529 sizeof(HECMW_Status));
530 if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
531
532 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
533 NULL)
534 HECMW_vis_memory_exit("send_recv: req1");
535 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
536 NULL)
537 HECMW_vis_memory_exit("send_recv: req2");
538 nflag = 1;
539 }
540 /* SEND */
541 for (neib = 0; neib < pesize; neib++) {
542 if (neib != my_rank) {
543 inum = stack_export[neib + 1] - stack_export[neib];
544 if (inum != 0) {
545 /* fprintf(stderr, "sending from PE %d to PE %d nodes
546 * %d\n", my_rank, neib, inum);
547 */
548 HECMW_Isend(&x[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
549 repart_comm, &req1[neib]);
550 }
551 }
552 }
553
554 /* RECEIVE */
555 for (neib = 0; neib < pesize; neib++) {
556 /* if (neib != 0) istart = stack_import[neib-1];
557 else istart = 0;
558 */
559 inum = stack_import[neib + 1] - stack_import[neib];
560 if ((neib != my_rank) && (inum != 0)) {
561 /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
562 * neib, inum);
563 */
564 HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_DOUBLE, neib, 0,
565 repart_comm, &req2[neib]);
566 }
567 }
568
569 HECMW_Waitall(pesize, req2, sta2);
570
571 inum = stack_import[my_rank + 1] - stack_import[my_rank];
572 for (k = 0; k < inum; k++)
573 y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
574
575 HECMW_Waitall(pesize, req1, sta1);
576
577 HECMW_free(sta1);
578 HECMW_free(sta2);
579
580 HECMW_free(req1);
581 HECMW_free(req2);
582 return;
583 }
584
int3_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * stack_export,int * x,int * y,HECMW_Comm repart_comm,int my_rank)585 void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
586 int *stack_export, int *x, int *y,
587 HECMW_Comm repart_comm, int my_rank) {
588 /* Important:: node ID in nod_import and nod_export are all starting from 0 */
589
590 HECMW_Status stat;
591
592 int neib;
593 int inum;
594 int i, j, k;
595
596 for (i = 0; i < pesize; i++) {
597 if (i != my_rank) {
598 inum = stack_export[i + 1] - stack_export[i];
599 if (inum != 0)
600 HECMW_Send(&x[stack_export[neib]], inum, HECMW_INT, i, 0, repart_comm);
601 } else if (i == my_rank) {
602 inum = stack_import[my_rank + 1] - stack_import[my_rank];
603 for (k = 0; k < inum; k++)
604 y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
605 for (j = 0; j < pesize; j++) {
606 if (j != my_rank) {
607 inum = stack_import[j + 1] - stack_import[j];
608 if (inum != 0)
609
610 HECMW_Recv(&y[stack_import[j]], inum, HECMW_INT, j, HECMW_ANY_TAG,
611 repart_comm, &stat);
612 }
613 }
614 }
615 }
616
617 HECMW_Barrier(repart_comm);
618
619 return;
620 }
621
double_whole_send_recv(int n1,int n2,int pesize,int * stack_import,int * nod_import,int * stack_export,int * nod_export,double * x,double * y,HECMW_Comm repart_comm,int my_rank)622 void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
623 int *nod_import, int *stack_export, int *nod_export,
624 double *x, double *y, HECMW_Comm repart_comm,
625 int my_rank) {
626 /* HECMW_Status *sta1, *sta2;
627 */
628 HECMW_Request *req1, *req2;
629
630 int nflag = 0;
631 int neib;
632 int inum;
633 int k;
634 double *ws, *wr;
635
636 ws = (double *)HECMW_calloc(n1, sizeof(int));
637 wr = (double *)HECMW_calloc(n2, sizeof(int));
638 if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws,wr");
639 if (nflag == 0) {
640 /* sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
641 sizeof(HECMW_Status));
642 if (sta1 == NULL) {
643 fprintf(stderr, "Not enough memory\n");
644 exit(1);
645 }
646 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
647 if (sta2 == NULL) {
648 fprintf(stderr, "Not enough memory\n");
649 exit(1);
650 }
651 */
652 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
653 NULL)
654 HECMW_vis_memory_exit("send_recv: req1");
655 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
656 NULL)
657 HECMW_vis_memory_exit("send_recv: req2");
658 nflag = 1;
659 }
660 /* SEND */
661 for (neib = 0; neib < pesize; neib++) {
662 /* if (neib != 0) istart = stack_export[neib - 1];
663 else istart = 0;
664 */
665 if (neib != my_rank) {
666 inum = stack_export[neib + 1] - stack_export[neib];
667
668 for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
669 ws[k] = x[nod_export[k] - 1];
670 }
671 HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
672 repart_comm, &req1[neib]);
673 }
674 }
675
676 /* RECEIVE */
677 for (neib = 0; neib < pesize; neib++) {
678 /* if (neib != 0) istart = stack_import[neib-1];
679 else istart = 0;
680 */
681
682 inum = stack_import[neib + 1] - stack_import[neib];
683 if (neib != my_rank) {
684 for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
685 y[k] = wr[k];
686 } else {
687 for (k = 0; k < inum; k++)
688 /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
689 * k++)
690 */
691 y[stack_import[neib] + k] = x[stack_export[neib] + k];
692 }
693 }
694
695 HECMW_Barrier(repart_comm);
696 /*
697 HECMW_Waitall(neibpetot, req1, sta1);
698
699 HECMW_free(sta1);
700 HECMW_free(sta2);
701 */
702 HECMW_free(req1);
703 HECMW_free(req2);
704 HECMW_free(ws);
705 HECMW_free(wr);
706 return;
707 }
708