1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #include "mpidimpl.h"
7 #include "ch4r_proc.h"
8
9 static int alloc_globals_for_avtid(int avtid);
10 static int free_globals_for_avtid(int avtid);
11 static int get_next_avtid(int *avtid);
12 static int free_avtid(int avtid);
13
MPIDIU_get_node_id(MPIR_Comm * comm,int rank,int * id_p)14 int MPIDIU_get_node_id(MPIR_Comm * comm, int rank, int *id_p)
15 {
16 int mpi_errno = MPI_SUCCESS;
17 int avtid = 0, lpid = 0;
18
19 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_NODE_ID);
20 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_NODE_ID);
21
22 MPIDIU_comm_rank_to_pid(comm, rank, &lpid, &avtid);
23 *id_p = MPIDI_global.node_map[avtid][lpid];
24
25 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_NODE_ID);
26 return mpi_errno;
27 }
28
MPIDIU_get_max_node_id(MPIR_Comm * comm,int * max_id_p)29 int MPIDIU_get_max_node_id(MPIR_Comm * comm, int *max_id_p)
30 {
31 int mpi_errno = MPI_SUCCESS;
32
33 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_MAX_NODE_ID);
34 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_MAX_NODE_ID);
35
36 *max_id_p = MPIDI_global.max_node_id;
37
38 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_MAX_NODE_ID);
39 return mpi_errno;
40 }
41
MPIDIU_build_nodemap(int myrank,MPIR_Comm * comm,int sz,int * out_nodemap,int * sz_out)42 int MPIDIU_build_nodemap(int myrank, MPIR_Comm * comm, int sz, int *out_nodemap, int *sz_out)
43 {
44 int mpi_errno = MPI_SUCCESS;
45
46 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_BUILD_NODEMAP);
47 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_BUILD_NODEMAP);
48
49 /* The nodemap is built in MPIR_pmi_init. Runtime rebuilding node_map not supported */
50 MPIR_Assert(0);
51
52 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_BUILD_NODEMAP);
53 return mpi_errno;
54 }
55
MPIDIU_get_n_avts(void)56 int MPIDIU_get_n_avts(void)
57 {
58 int ret;
59
60 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_N_AVTS);
61 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_N_AVTS);
62
63 ret = MPIDI_global.avt_mgr.n_avts;
64
65 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_N_AVTS);
66 return ret;
67 }
68
MPIDIU_get_max_n_avts(void)69 int MPIDIU_get_max_n_avts(void)
70 {
71 int ret;
72
73 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_MAX_N_AVTS);
74 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_MAX_N_AVTS);
75
76 ret = MPIDI_global.avt_mgr.max_n_avts;
77
78 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_MAX_N_AVTS);
79 return ret;
80 }
81
MPIDIU_get_avt_size(int avtid)82 int MPIDIU_get_avt_size(int avtid)
83 {
84 int ret;
85
86 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_AVT_SIZE);
87 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_AVT_SIZE);
88
89 ret = MPIDI_av_table[avtid]->size;
90
91 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_AVT_SIZE);
92 return ret;
93 }
94
alloc_globals_for_avtid(int avtid)95 static int alloc_globals_for_avtid(int avtid)
96 {
97 int mpi_errno = MPI_SUCCESS;
98 int *new_node_map = NULL;
99 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_ALLOC_GLOBALS_FOR_AVTID);
100 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_ALLOC_GLOBALS_FOR_AVTID);
101
102 new_node_map = (int *) MPL_malloc(MPIDI_av_table[avtid]->size * sizeof(int), MPL_MEM_ADDRESS);
103 MPIR_ERR_CHKANDJUMP(new_node_map == NULL, mpi_errno, MPI_ERR_NO_MEM, "**nomem");
104 MPIDI_global.node_map[avtid] = new_node_map;
105
106 fn_exit:
107 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_ALLOC_GLOBALS_FOR_AVTID);
108 return mpi_errno;
109 fn_fail:
110 goto fn_exit;
111 }
112
free_globals_for_avtid(int avtid)113 static int free_globals_for_avtid(int avtid)
114 {
115 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_FREE_GLOBALS_FOR_AVTID);
116 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_FREE_GLOBALS_FOR_AVTID);
117 if (avtid > 0) {
118 MPL_free(MPIDI_global.node_map[avtid]);
119 }
120 MPIDI_global.node_map[avtid] = NULL;
121 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_FREE_GLOBALS_FOR_AVTID);
122 return MPI_SUCCESS;
123 }
124
get_next_avtid(int * avtid)125 static int get_next_avtid(int *avtid)
126 {
127 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_GET_NEXT_AVTID);
128 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_GET_NEXT_AVTID);
129
130 if (MPIDI_global.avt_mgr.next_avtid == -1) { /* out of free avtids */
131 int old_max, new_max, i;
132 old_max = MPIDI_global.avt_mgr.max_n_avts;
133 new_max = old_max + 1;
134 MPIDI_global.avt_mgr.free_avtid =
135 (int *) MPL_realloc(MPIDI_global.avt_mgr.free_avtid, new_max * sizeof(int),
136 MPL_MEM_ADDRESS);
137 for (i = old_max; i < new_max - 1; i++) {
138 MPIDI_global.avt_mgr.free_avtid[i] = i + 1;
139 }
140 MPIDI_global.avt_mgr.free_avtid[new_max - 1] = -1;
141 MPIDI_global.avt_mgr.max_n_avts = new_max;
142 MPIDI_global.avt_mgr.next_avtid = old_max;
143 }
144
145 *avtid = MPIDI_global.avt_mgr.next_avtid;
146 MPIDI_global.avt_mgr.next_avtid = MPIDI_global.avt_mgr.free_avtid[*avtid];
147 MPIDI_global.avt_mgr.free_avtid[*avtid] = -1;
148 MPIDI_global.avt_mgr.n_avts++;
149 MPIR_Assert(MPIDI_global.avt_mgr.n_avts <= MPIDI_global.avt_mgr.max_n_avts);
150
151 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE, (MPL_DBG_FDEST, " new_avtid=%d", *avtid));
152
153 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_GET_NEXT_AVTID);
154 return *avtid;
155 }
156
free_avtid(int avtid)157 static int free_avtid(int avtid)
158 {
159 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_FREE_AVTID);
160 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_FREE_AVTID);
161
162 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE, (MPL_DBG_FDEST, " free_avtid=%d", avtid));
163 MPIR_Assert(MPIDI_global.avt_mgr.n_avts > 0);
164 MPIDI_global.avt_mgr.free_avtid[avtid] = MPIDI_global.avt_mgr.next_avtid;
165 MPIDI_global.avt_mgr.next_avtid = avtid;
166 MPIDI_global.avt_mgr.n_avts--;
167 /*
168 * TODO:
169 * If the allowed number of process groups are very large
170 * We need to return unsed pages back to OS.
171 * madvise(addr, len, MADV_DONTNEED);
172 * We need tracking wich page can be returned
173 */
174 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_FREE_AVTID);
175 return 0;
176 }
177
MPIDIU_new_avt(int size,int * avtid)178 int MPIDIU_new_avt(int size, int *avtid)
179 {
180 int mpi_errno = MPI_SUCCESS;
181 MPIDI_av_table_t *new_av_table;
182
183 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_NEW_AVT);
184 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_NEW_AVT);
185 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE, (MPL_DBG_FDEST, " new_avt: size=%d", size));
186
187 get_next_avtid(avtid);
188
189 new_av_table = (MPIDI_av_table_t *) MPL_malloc(size * sizeof(MPIDI_av_entry_t)
190 + sizeof(MPIDI_av_table_t), MPL_MEM_ADDRESS);
191 new_av_table->size = size;
192 MPIDI_av_table[*avtid] = new_av_table;
193
194 MPIR_Object_set_ref(MPIDI_av_table[*avtid], 0);
195
196 alloc_globals_for_avtid(*avtid);
197
198 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_NEW_AVT);
199 return mpi_errno;
200 }
201
MPIDIU_free_avt(int avtid)202 int MPIDIU_free_avt(int avtid)
203 {
204 int mpi_errno = MPI_SUCCESS;
205
206 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_FREE_AVT);
207 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_FREE_AVT);
208 MPID_THREAD_CS_ENTER(VCI, MPIDIU_THREAD_DYNPROC_MUTEX);
209
210 free_globals_for_avtid(avtid);
211 MPL_free(MPIDI_av_table[avtid]);
212 MPIDI_av_table[avtid] = NULL;
213 free_avtid(avtid);
214
215 MPID_THREAD_CS_EXIT(VCI, MPIDIU_THREAD_DYNPROC_MUTEX);
216 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_FREE_AVT);
217 return mpi_errno;
218 }
219
MPIDIU_avt_add_ref(int avtid)220 int MPIDIU_avt_add_ref(int avtid)
221 {
222 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_AVT_ADD_REF);
223 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_AVT_ADD_REF);
224
225 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE, (MPL_DBG_FDEST, " incr avtid=%d", avtid));
226 MPIR_Object_add_ref(MPIDI_av_table[avtid]);
227
228 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_AVT_ADD_REF);
229 return MPI_SUCCESS;
230 }
231
MPIDIU_avt_release_ref(int avtid)232 int MPIDIU_avt_release_ref(int avtid)
233 {
234 int in_use;
235
236 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_AVT_RELEASE_REF);
237 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_AVT_RELEASE_REF);
238
239 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE, (MPL_DBG_FDEST, " decr avtid=%d", avtid));
240 MPIR_Object_release_ref(MPIDIU_get_av_table(avtid), &in_use);
241 if (!in_use) {
242 MPIDIU_free_avt(avtid);
243 }
244
245 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_AVT_RELEASE_REF);
246 return MPI_SUCCESS;
247 }
248
249 #define AVT_SIZE (8 * 4 * 1024) /* FIXME: what is this size? */
250
MPIDIU_avt_init(void)251 int MPIDIU_avt_init(void)
252 {
253 int i, mpi_errno = MPI_SUCCESS;
254
255 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_AVT_INIT);
256 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_AVT_INIT);
257
258 MPIDI_global.avt_mgr.max_n_avts = 1;
259 MPIDI_global.avt_mgr.next_avtid = 0;
260 MPIDI_global.avt_mgr.n_avts = 0;
261
262 MPIDI_av_table = (MPIDI_av_table_t **)
263 MPL_malloc(AVT_SIZE, MPL_MEM_ADDRESS);
264 MPIR_ERR_CHKANDSTMT(MPIDI_av_table == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem");
265
266 MPIDI_global.node_map = (int **)
267 MPL_malloc(AVT_SIZE, MPL_MEM_ADDRESS);
268 MPIR_ERR_CHKANDSTMT(MPIDI_global.node_map == NULL, mpi_errno,
269 MPI_ERR_NO_MEM, goto fn_fail, "**nomem");
270
271 MPIDI_global.avt_mgr.free_avtid =
272 (int *) MPL_malloc(MPIDI_global.avt_mgr.max_n_avts * sizeof(int), MPL_MEM_ADDRESS);
273 MPIR_ERR_CHKANDSTMT(MPIDI_global.avt_mgr.free_avtid == NULL, mpi_errno,
274 MPI_ERR_NO_MEM, goto fn_fail, "**nomem");
275
276 for (i = 0; i < MPIDI_global.avt_mgr.max_n_avts - 1; i++) {
277 MPIDI_global.avt_mgr.free_avtid[i] = i + 1;
278 }
279 MPIDI_global.avt_mgr.free_avtid[MPIDI_global.avt_mgr.max_n_avts - 1] = -1;
280
281 int first_avtid;
282 get_next_avtid(&first_avtid);
283 MPIR_Assert(first_avtid == 0);
284
285 fn_exit:
286 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_AVT_INIT);
287 return mpi_errno;
288 fn_fail:
289 goto fn_exit;
290 }
291
MPIDIU_avt_destroy(void)292 int MPIDIU_avt_destroy(void)
293 {
294 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_AVT_DESTROY);
295 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_AVT_DESTROY);
296
297 MPL_free(MPIDI_global.node_map);
298 MPL_free(MPIDI_av_table);
299 MPL_free(MPIDI_global.avt_mgr.free_avtid);
300
301 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_AVT_DESTROY);
302 return MPI_SUCCESS;
303 }
304
MPIDIU_build_nodemap_avtid(int myrank,MPIR_Comm * comm,int sz,int avtid)305 int MPIDIU_build_nodemap_avtid(int myrank, MPIR_Comm * comm, int sz, int avtid)
306 {
307 int ret;
308
309 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_BUILD_NODEMAP_AVTID);
310 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_BUILD_NODEMAP_AVTID);
311 ret = MPIDIU_build_nodemap(myrank, comm, sz, MPIDI_global.node_map[avtid],
312 &MPIDI_global.max_node_id);
313
314 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_BUILD_NODEMAP_AVTID);
315 return ret;
316 }
317
MPIDIU_upids_to_lupids(int size,size_t * remote_upid_size,char * remote_upids,int ** remote_lupids,int * remote_node_ids)318 int MPIDIU_upids_to_lupids(int size, size_t * remote_upid_size, char *remote_upids,
319 int **remote_lupids, int *remote_node_ids)
320 {
321 int mpi_errno = MPI_SUCCESS, i;
322 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_UPIDS_TO_LUPIDS);
323 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_UPIDS_TO_LUPIDS);
324
325 MPID_THREAD_CS_ENTER(VCI, MPIDIU_THREAD_DYNPROC_MUTEX);
326 mpi_errno = MPIDI_NM_upids_to_lupids(size, remote_upid_size, remote_upids, remote_lupids);
327 MPIR_ERR_CHECK(mpi_errno);
328
329 /* update node_map */
330 for (i = 0; i < size; i++) {
331 int _avtid = 0, _lpid = 0;
332 /* if this is a new process, update node_map and locality */
333 if (MPIDIU_LUPID_IS_NEW_AVT((*remote_lupids)[i])) {
334 MPIDIU_LUPID_CLEAR_NEW_AVT_MARK((*remote_lupids)[i]);
335 _avtid = MPIDIU_LUPID_GET_AVTID((*remote_lupids)[i]);
336 _lpid = MPIDIU_LUPID_GET_LPID((*remote_lupids)[i]);
337 if (_avtid != 0) {
338 /*
339 * new process groups are always assumed to be remote,
340 * so CH4 don't care what node they are on
341 */
342 MPIDI_global.node_map[_avtid][_lpid] = remote_node_ids[i];
343 if (remote_node_ids[i] > MPIDI_global.max_node_id) {
344 MPIDI_global.max_node_id = remote_node_ids[i];
345 }
346 #ifdef MPIDI_BUILD_CH4_LOCALITY_INFO
347 MPIDI_av_table[_avtid]->table[_lpid].is_local = 0;
348 #endif
349 }
350 }
351 }
352
353 MPIR_ERR_CHECK(mpi_errno);
354
355 fn_exit:
356 MPID_THREAD_CS_EXIT(VCI, MPIDIU_THREAD_DYNPROC_MUTEX);
357 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_UPIDS_TO_LUPIDS);
358 return mpi_errno;
359 fn_fail:
360 goto fn_exit;
361 }
362
MPIDIU_Intercomm_map_bcast_intra(MPIR_Comm * local_comm,int local_leader,int * remote_size,int * is_low_group,int pure_intracomm,size_t * remote_upid_size,char * remote_upids,int ** remote_lupids,int * remote_node_ids)363 int MPIDIU_Intercomm_map_bcast_intra(MPIR_Comm * local_comm, int local_leader, int *remote_size,
364 int *is_low_group, int pure_intracomm,
365 size_t * remote_upid_size, char *remote_upids,
366 int **remote_lupids, int *remote_node_ids)
367 {
368 int mpi_errno = MPI_SUCCESS;
369 int i;
370 int upid_recv_size = 0;
371 int map_info[4];
372 MPIR_Errflag_t errflag = MPIR_ERR_NONE;
373 size_t *_remote_upid_size = NULL;
374 char *_remote_upids = NULL;
375 int *_remote_node_ids = NULL;
376
377 MPIR_CHKPMEM_DECL(1);
378 MPIR_CHKLMEM_DECL(3);
379
380 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_INTERCOMM_MAP_BCAST_INTRA);
381 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_INTERCOMM_MAP_BCAST_INTRA);
382
383 if (local_comm->rank == local_leader) {
384 if (!pure_intracomm) {
385 for (i = 0; i < (*remote_size); i++) {
386 upid_recv_size += remote_upid_size[i];
387 }
388 }
389 map_info[0] = *remote_size;
390 map_info[1] = upid_recv_size;
391 map_info[2] = *is_low_group;
392 map_info[3] = pure_intracomm;
393 mpi_errno =
394 MPIR_Bcast_allcomm_auto(map_info, 4, MPI_INT, local_leader, local_comm, &errflag);
395 MPIR_ERR_CHECK(mpi_errno);
396
397 if (!pure_intracomm) {
398 mpi_errno = MPIR_Bcast_allcomm_auto(remote_upid_size, *remote_size, MPI_UNSIGNED_LONG,
399 local_leader, local_comm, &errflag);
400 MPIR_ERR_CHECK(mpi_errno);
401 mpi_errno = MPIR_Bcast_allcomm_auto(remote_upids, upid_recv_size, MPI_BYTE,
402 local_leader, local_comm, &errflag);
403 MPIR_ERR_CHECK(mpi_errno);
404 mpi_errno =
405 MPIR_Bcast_allcomm_auto(remote_node_ids, (*remote_size) * sizeof(int), MPI_BYTE,
406 local_leader, local_comm, &errflag);
407 MPIR_ERR_CHECK(mpi_errno);
408 } else {
409 mpi_errno = MPIR_Bcast_allcomm_auto(*remote_lupids, *remote_size, MPI_INT,
410 local_leader, local_comm, &errflag);
411 }
412 } else {
413 mpi_errno =
414 MPIR_Bcast_allcomm_auto(map_info, 4, MPI_INT, local_leader, local_comm, &errflag);
415 MPIR_ERR_CHECK(mpi_errno);
416 *remote_size = map_info[0];
417 upid_recv_size = map_info[1];
418 *is_low_group = map_info[2];
419 pure_intracomm = map_info[3];
420
421 MPIR_CHKPMEM_MALLOC((*remote_lupids), int *, (*remote_size) * sizeof(int),
422 mpi_errno, "remote_lupids", MPL_MEM_COMM);
423 if (!pure_intracomm) {
424 MPIR_CHKLMEM_MALLOC(_remote_upid_size, size_t *, (*remote_size) * sizeof(size_t),
425 mpi_errno, "_remote_upid_size", MPL_MEM_COMM);
426 mpi_errno = MPIR_Bcast_allcomm_auto(_remote_upid_size, *remote_size, MPI_UNSIGNED_LONG,
427 local_leader, local_comm, &errflag);
428 MPIR_ERR_CHECK(mpi_errno);
429 MPIR_CHKLMEM_MALLOC(_remote_upids, char *, upid_recv_size * sizeof(char),
430 mpi_errno, "_remote_upids", MPL_MEM_COMM);
431 mpi_errno = MPIR_Bcast_allcomm_auto(_remote_upids, upid_recv_size, MPI_BYTE,
432 local_leader, local_comm, &errflag);
433 MPIR_ERR_CHECK(mpi_errno);
434 MPIR_CHKLMEM_MALLOC(_remote_node_ids, int *,
435 (*remote_size) * sizeof(int),
436 mpi_errno, "_remote_node_ids", MPL_MEM_COMM);
437 mpi_errno =
438 MPIR_Bcast_allcomm_auto(_remote_node_ids, (*remote_size) * sizeof(int), MPI_BYTE,
439 local_leader, local_comm, &errflag);
440 MPIR_ERR_CHECK(mpi_errno);
441
442 MPIDIU_upids_to_lupids(*remote_size, _remote_upid_size, _remote_upids,
443 remote_lupids, _remote_node_ids);
444 } else {
445 mpi_errno = MPIR_Bcast_allcomm_auto(*remote_lupids, *remote_size, MPI_INT,
446 local_leader, local_comm, &errflag);
447 }
448 }
449
450 MPIR_CHKPMEM_COMMIT();
451 fn_exit:
452 MPIR_CHKLMEM_FREEALL();
453 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_INTERCOMM_MAP_BCAST_INTRA);
454 return mpi_errno;
455 fn_fail:
456 MPIR_CHKPMEM_REAP();
457 goto fn_exit;
458 }
459
MPIDIU_alloc_lut(MPIDI_rank_map_lut_t ** lut,int size)460 int MPIDIU_alloc_lut(MPIDI_rank_map_lut_t ** lut, int size)
461 {
462 int mpi_errno = MPI_SUCCESS;
463 MPIDI_rank_map_lut_t *new_lut = NULL;
464
465 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_ALLOC_LUT);
466 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_ALLOC_LUT);
467
468 new_lut = (MPIDI_rank_map_lut_t *) MPL_malloc(sizeof(MPIDI_rank_map_lut_t)
469 + size * sizeof(MPIDI_lpid_t), MPL_MEM_ADDRESS);
470 if (new_lut == NULL) {
471 *lut = NULL;
472 MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomem");
473 }
474
475 MPIR_Object_set_ref(new_lut, 1);
476 *lut = new_lut;
477
478 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE,
479 (MPL_DBG_FDEST, "alloc lut %p, size %lu, refcount=%d",
480 new_lut, size * sizeof(MPIDI_lpid_t), MPIR_Object_get_ref(new_lut)));
481 fn_exit:
482 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_ALLOC_LUT);
483 return mpi_errno;
484 fn_fail:
485 goto fn_exit;
486 }
487
MPIDIU_release_lut(MPIDI_rank_map_lut_t * lut)488 int MPIDIU_release_lut(MPIDI_rank_map_lut_t * lut)
489 {
490 int mpi_errno = MPI_SUCCESS;
491 int in_use = 0;
492
493 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_RELEASE_LUT);
494 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_RELEASE_LUT);
495
496 MPIR_Object_release_ref(lut, &in_use);
497 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE, (MPL_DBG_FDEST, "dec ref to lut %p", lut));
498 if (!in_use) {
499 MPL_free(lut);
500 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE, (MPL_DBG_FDEST, "free lut %p", lut));
501 }
502 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_RELEASE_LUT);
503 return mpi_errno;
504 }
505
MPIDIU_alloc_mlut(MPIDI_rank_map_mlut_t ** mlut,int size)506 int MPIDIU_alloc_mlut(MPIDI_rank_map_mlut_t ** mlut, int size)
507 {
508 int mpi_errno = MPI_SUCCESS;
509 MPIDI_rank_map_mlut_t *new_mlut = NULL;
510
511 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_ALLOC_MLUT);
512 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_ALLOC_MLUT);
513
514 new_mlut = (MPIDI_rank_map_mlut_t *) MPL_malloc(sizeof(MPIDI_rank_map_mlut_t)
515 + size * sizeof(MPIDI_gpid_t), MPL_MEM_ADDRESS);
516 if (new_mlut == NULL) {
517 *mlut = NULL;
518 MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**nomem");
519 }
520
521 MPIR_Object_set_ref(new_mlut, 1);
522 *mlut = new_mlut;
523
524 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE,
525 (MPL_DBG_FDEST, "alloc mlut %p, size %lu, refcount=%d",
526 new_mlut, size * sizeof(MPIDI_gpid_t), MPIR_Object_get_ref(new_mlut)));
527 fn_exit:
528 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_ALLOC_MLUT);
529 return mpi_errno;
530 fn_fail:
531 goto fn_exit;
532 }
533
MPIDIU_release_mlut(MPIDI_rank_map_mlut_t * mlut)534 int MPIDIU_release_mlut(MPIDI_rank_map_mlut_t * mlut)
535 {
536 int mpi_errno = MPI_SUCCESS;
537 int in_use = 0;
538
539 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIU_RELEASE_MLUT);
540 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIU_RELEASE_MLUT);
541
542 MPIR_Object_release_ref(mlut, &in_use);
543 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE, (MPL_DBG_FDEST, "dec ref to mlut %p", mlut));
544 if (!in_use) {
545 MPL_free(mlut);
546 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MEMORY, VERBOSE, (MPL_DBG_FDEST, "free mlut %p", mlut));
547 }
548
549 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIU_RELEASE_MLUT);
550 return mpi_errno;
551 }
552