1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #include "mpidimpl.h"
7 #include "mpidch4r.h"
8 #include "datatype.h"
9 #include "mpidu_init_shm.h"
10
11 #ifdef HAVE_SIGNAL_H
12 #include <signal.h>
13 #endif
14
15 /*
16 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
17
18 categories:
19 - name : CH4
20 description : cvars that control behavior of the CH4 device
21
22 cvars:
23 - name : MPIR_CVAR_CH4_NETMOD
24 category : CH4
25 type : string
26 default : ""
27 class : none
28 verbosity : MPI_T_VERBOSITY_USER_BASIC
29 scope : MPI_T_SCOPE_ALL_EQ
30 description : >-
31 If non-empty, this cvar specifies which network module to use
32
33 - name : MPIR_CVAR_CH4_SHM
34 category : CH4
35 type : string
36 default : ""
37 class : none
38 verbosity : MPI_T_VERBOSITY_USER_BASIC
39 scope : MPI_T_SCOPE_ALL_EQ
40 description : >-
41 If non-empty, this cvar specifies which shm module to use
42
43 - name : MPIR_CVAR_CH4_ROOTS_ONLY_PMI
44 category : CH4
45 type : boolean
46 default : false
47 class : none
48 verbosity : MPI_T_VERBOSITY_USER_BASIC
49 scope : MPI_T_SCOPE_LOCAL
50 description : >-
51 Enables an optimized business card exchange over PMI for node root processes only.
52
53 - name : MPIR_CVAR_CH4_RUNTIME_CONF_DEBUG
54 category : CH4
55 type : boolean
56 default : false
57 class : none
58 verbosity : MPI_T_VERBOSITY_USER_BASIC
59 scope : MPI_T_SCOPE_ALL_EQ
60 description : >-
61 If enabled, CH4-level runtime configurations are printed out
62
63 - name : MPIR_CVAR_CH4_MT_MODEL
64 category : CH4
65 type : string
66 default : ""
67 class : none
68 verbosity : MPI_T_VERBOSITY_USER_BASIC
69 scope : MPI_T_SCOPE_ALL_EQ
70 description : >-
71 Specifies the CH4 multi-threading model. Possible values are:
72 direct (default)
73 handoff
74
75 - name : MPIR_CVAR_CH4_NUM_VCIS
76 category : CH4
77 type : int
78 default : 1
79 class : none
80 verbosity : MPI_T_VERBOSITY_USER_BASIC
81 scope : MPI_T_SCOPE_LOCAL
82 description : >-
83 Sets the number of VCIs that user needs (should be a subset of MPIDI_CH4_MAX_VCIS).
84
85 - name : MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE
86 category : COLLECTIVE
87 type : string
88 default : ""
89 class : none
90 verbosity : MPI_T_VERBOSITY_USER_BASIC
91 scope : MPI_T_SCOPE_ALL_EQ
92 description : >-
93 Defines the location of tuning file.
94
95 - name : MPIR_CVAR_CH4_IOV_DENSITY_MIN
96 category : CH4
97 type : int
98 default : 16384
99 class : none
100 verbosity : MPI_T_VERBOSITY_USER_BASIC
101 scope : MPI_T_SCOPE_LOCAL
102 description : >-
103 Defines the threshold of high-density datatype. The
104 density is calculated by (datatype_size / datatype_num_contig_blocks).
105 === END_MPI_T_CVAR_INFO_BLOCK ===
106 */
107
create_container(struct json_object * obj)108 static void *create_container(struct json_object *obj)
109 {
110 MPIDI_Csel_container_s *cnt = MPL_malloc(sizeof(MPIDI_Csel_container_s), MPL_MEM_COLL);
111
112 json_object_object_foreach(obj, key, val) {
113 char *ckey = MPL_strdup_no_spaces(key);
114
115 if (!strcmp(ckey, "composition=MPIDI_Barrier_intra_composition_alpha"))
116 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_alpha;
117 else if (!strcmp(ckey, "composition=MPIDI_Barrier_intra_composition_beta"))
118 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_beta;
119 else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_alpha"))
120 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_alpha;
121 else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_beta"))
122 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_beta;
123 else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_gamma"))
124 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_gamma;
125 else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_alpha"))
126 cnt->id =
127 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_alpha;
128 else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_beta"))
129 cnt->id =
130 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_beta;
131 else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_gamma"))
132 cnt->id =
133 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_gamma;
134 else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_alpha"))
135 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_alpha;
136 else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_beta"))
137 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_beta;
138 else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_gamma"))
139 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_gamma;
140 else if (!strcmp(ckey, "composition=MPIDI_Alltoall_intra_composition_alpha"))
141 cnt->id =
142 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha;
143 else if (!strcmp(ckey, "composition=MPIDI_Alltoallv_intra_composition_alpha"))
144 cnt->id =
145 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallv_intra_composition_alpha;
146 else if (!strcmp(ckey, "composition=MPIDI_Alltoallw_intra_composition_alpha"))
147 cnt->id =
148 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallw_intra_composition_alpha;
149 else if (!strcmp(ckey, "composition=MPIDI_Allgather_intra_composition_alpha"))
150 cnt->id =
151 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha;
152 else if (!strcmp(ckey, "composition=MPIDI_Allgatherv_intra_composition_alpha"))
153 cnt->id =
154 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgatherv_intra_composition_alpha;
155 else if (!strcmp(ckey, "composition=MPIDI_Gather_intra_composition_alpha"))
156 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gather_intra_composition_alpha;
157 else if (!strcmp(ckey, "composition=MPIDI_Gatherv_intra_composition_alpha"))
158 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gatherv_intra_composition_alpha;
159 else if (!strcmp(ckey, "composition=MPIDI_Scatter_intra_composition_alpha"))
160 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatter_intra_composition_alpha;
161 else if (!strcmp(ckey, "composition=MPIDI_Scatterv_intra_composition_alpha"))
162 cnt->id =
163 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatterv_intra_composition_alpha;
164 else if (!strcmp(ckey, "composition=MPIDI_Reduce_scatter_intra_composition_alpha"))
165 cnt->id =
166 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_intra_composition_alpha;
167 else if (!strcmp(ckey, "composition=MPIDI_Reduce_scatter_block_intra_composition_alpha"))
168 cnt->id =
169 MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_block_intra_composition_alpha;
170 else if (!strcmp(ckey, "composition=MPIDI_Scan_intra_composition_alpha"))
171 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_alpha;
172 else if (!strcmp(ckey, "composition=MPIDI_Scan_intra_composition_beta"))
173 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_beta;
174 else if (!strcmp(ckey, "composition=MPIDI_Exscan_intra_composition_alpha"))
175 cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Exscan_intra_composition_alpha;
176 else {
177 fprintf(stderr, "unrecognized key %s\n", ckey);
178 MPIR_Assert(0);
179 }
180
181 MPL_free(ckey);
182 }
183
184 return (void *) cnt;
185 }
186
187 static int choose_netmod(void);
188 static int create_init_comm(MPIR_Comm **);
189 static void destroy_init_comm(MPIR_Comm **);
190 static int init_builtin_comms(void);
191 static void finalize_builtin_comms(void);
192 static void init_av_table(void);
193 static void finalize_av_table(void);
194
choose_netmod(void)195 static int choose_netmod(void)
196 {
197 int i, mpi_errno = MPI_SUCCESS;
198 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_CHOOSE_NETMOD);
199 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_CHOOSE_NETMOD);
200
201 MPIR_Assert(MPIR_CVAR_CH4_NETMOD != NULL);
202
203 if (strcmp(MPIR_CVAR_CH4_NETMOD, "") == 0) {
204 /* netmod not specified, using the default */
205 MPIDI_NM_func = MPIDI_NM_funcs[0];
206 MPIDI_NM_native_func = MPIDI_NM_native_funcs[0];
207 goto fn_exit;
208 }
209
210 for (i = 0; i < MPIDI_num_netmods; ++i) {
211 /* use MPL variant of strncasecmp if we get one */
212 if (!strncasecmp(MPIR_CVAR_CH4_NETMOD, MPIDI_NM_strings[i], MPIDI_MAX_NETMOD_STRING_LEN)) {
213 MPIDI_NM_func = MPIDI_NM_funcs[i];
214 MPIDI_NM_native_func = MPIDI_NM_native_funcs[i];
215 goto fn_exit;
216 }
217 }
218
219 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_netmod", "**invalid_netmod %s",
220 MPIR_CVAR_CH4_NETMOD);
221 fn_exit:
222 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_CHOOSE_NETMOD);
223 return mpi_errno;
224 fn_fail:
225
226 goto fn_exit;
227 }
228
229 #ifdef MPIDI_CH4_USE_WORK_QUEUES
230 /* NOTE: MPIDI_CH4_USE_MT_RUNTIME currently are not supported. The key part is
231 * guarded by "#if 0".
232 */
233 /* TODO: move them to ch4i_workq_init.c. */
234
235 static const char *get_mt_model_name(int mt);
236 static void print_runtime_configurations(void);
237 static int parse_mt_model(const char *name);
238 static int set_runtime_configurations(void);
239
240 static const char *mt_model_names[MPIDI_CH4_NUM_MT_MODELS] = {
241 "direct",
242 "handoff",
243 };
244
get_mt_model_name(int mt)245 static const char *get_mt_model_name(int mt)
246 {
247 if (mt < 0 || mt >= MPIDI_CH4_NUM_MT_MODELS)
248 return "(invalid)";
249
250 return mt_model_names[mt];
251 }
252
print_runtime_configurations(void)253 static void print_runtime_configurations(void)
254 {
255 printf("==== CH4 runtime configurations ====\n");
256 printf("MPIDI_CH4_MT_MODEL: %d (%s)\n",
257 MPIDI_CH4_MT_MODEL, get_mt_model_name(MPIDI_CH4_MT_MODEL));
258 printf("================================\n");
259 }
260
parse_mt_model(const char * name)261 static int parse_mt_model(const char *name)
262 {
263 int i;
264
265 if (!strcmp("", name))
266 return 0; /* default */
267
268 for (i = 0; i < MPIDI_CH4_NUM_MT_MODELS; i++) {
269 if (!strcasecmp(name, mt_model_names[i]))
270 return i;
271 }
272 return -1;
273 }
274
set_runtime_configurations(void)275 static int set_runtime_configurations(void)
276 {
277 int mpi_errno = MPI_SUCCESS;
278
279 #if 0 /* defined(MPIDI_CH4_USE_MT_RUNTIME) */
280 int mt = parse_mt_model(MPIR_CVAR_CH4_MT_MODEL);
281 if (mt < 0)
282 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
283 "**ch4|invalid_mt_model", "**ch4|invalid_mt_model %s",
284 MPIR_CVAR_CH4_MT_MODEL);
285 MPIDI_global.settings.mt_model = mt;
286 #else
287 /* Static configuration - no runtime selection */
288 if (strcmp(MPIR_CVAR_CH4_MT_MODEL, "") != 0)
289 printf("Warning: MPIR_CVAR_CH4_MT_MODEL will be ignored "
290 "unless --enable-ch4-mt=runtime is given at the configure time.\n");
291 #endif /* MPIDI_CH4_USE_MT_RUNTIME */
292
293 #if 0 /* defined(MPIDI_CH4_USE_MT_RUNTIME) */
294 fn_fail:
295 #endif
296 return mpi_errno;
297 }
298
299 #endif /* MPIDI_CH4_USE_WORK_QUEUES */
300
create_init_comm(MPIR_Comm ** comm)301 static int create_init_comm(MPIR_Comm ** comm)
302 {
303 int i, mpi_errno = MPI_SUCCESS;
304 int world_rank = MPIR_Process.rank;
305 int node_root_rank = MPIR_Process.node_root_map[MPIR_Process.node_map[world_rank]];
306
307 /* if the process is not a node root, exit */
308 if (node_root_rank == world_rank) {
309 int node_roots_comm_size = MPIR_Process.num_nodes;
310 int node_roots_comm_rank = MPIR_Process.node_map[world_rank];
311 MPIR_Comm *init_comm = NULL;
312 MPIDI_rank_map_lut_t *lut = NULL;
313 MPIR_Comm_create(&init_comm);
314 init_comm->context_id = 0 << MPIR_CONTEXT_PREFIX_SHIFT;
315 init_comm->recvcontext_id = 0 << MPIR_CONTEXT_PREFIX_SHIFT;
316 init_comm->comm_kind = MPIR_COMM_KIND__INTRACOMM;
317 init_comm->rank = node_roots_comm_rank;
318 init_comm->remote_size = node_roots_comm_size;
319 init_comm->local_size = node_roots_comm_size;
320 init_comm->coll.pof2 = MPL_pof2(node_roots_comm_size);
321 MPIDI_COMM(init_comm, map).mode = MPIDI_RANK_MAP_LUT_INTRA;
322 mpi_errno = MPIDIU_alloc_lut(&lut, node_roots_comm_size);
323 MPIR_ERR_CHECK(mpi_errno);
324 MPIDI_COMM(init_comm, map).size = node_roots_comm_size;
325 MPIDI_COMM(init_comm, map).avtid = 0;
326 MPIDI_COMM(init_comm, map).irreg.lut.t = lut;
327 MPIDI_COMM(init_comm, map).irreg.lut.lpid = lut->lpid;
328 MPIDI_COMM(init_comm, local_map).mode = MPIDI_RANK_MAP_NONE;
329 for (i = 0; i < node_roots_comm_size; ++i) {
330 lut->lpid[i] = MPIR_Process.node_root_map[i];
331 }
332 mpi_errno = MPIDIG_init_comm(init_comm);
333 MPIR_ERR_CHECK(mpi_errno);
334
335 *comm = init_comm;
336 }
337 fn_exit:
338 return mpi_errno;
339 fn_fail:
340 goto fn_exit;
341 }
342
destroy_init_comm(MPIR_Comm ** comm_ptr)343 static void destroy_init_comm(MPIR_Comm ** comm_ptr)
344 {
345 int in_use;
346 MPIR_Comm *comm = NULL;
347 if (*comm_ptr != NULL) {
348 comm = *comm_ptr;
349 MPIDIU_release_lut(MPIDI_COMM(comm, map).irreg.lut.t);
350 MPIDIG_destroy_comm(comm);
351 MPIR_Object_release_ref(comm, &in_use);
352 MPIR_Assert(MPIR_Object_get_ref(comm) == 0);
353 MPII_COMML_FORGET(comm);
354 MPIR_Handle_obj_free(&MPIR_Comm_mem, comm);
355 *comm_ptr = NULL;
356 }
357 }
358
init_builtin_comms(void)359 static int init_builtin_comms(void)
360 {
361 int mpi_errno = MPI_SUCCESS;
362
363 /* ---------------------------------- */
364 /* Initialize MPI_COMM_SELF */
365 /* ---------------------------------- */
366 MPIR_Process.comm_self->rank = 0;
367 MPIR_Process.comm_self->remote_size = 1;
368 MPIR_Process.comm_self->local_size = 1;
369
370 /* ---------------------------------- */
371 /* Initialize MPI_COMM_WORLD */
372 /* ---------------------------------- */
373 MPIR_Process.comm_world->rank = MPIR_Process.rank;
374 MPIR_Process.comm_world->remote_size = MPIR_Process.size;
375 MPIR_Process.comm_world->local_size = MPIR_Process.size;
376
377 /* initialize rank_map */
378 MPIDI_COMM(MPIR_Process.comm_world, map).mode = MPIDI_RANK_MAP_DIRECT_INTRA;
379 MPIDI_COMM(MPIR_Process.comm_world, map).avtid = 0;
380 MPIDI_COMM(MPIR_Process.comm_world, map).size = MPIR_Process.size;
381 MPIDI_COMM(MPIR_Process.comm_world, local_map).mode = MPIDI_RANK_MAP_NONE;
382 MPIDIU_avt_add_ref(0);
383
384 MPIDI_COMM(MPIR_Process.comm_self, map).mode = MPIDI_RANK_MAP_OFFSET_INTRA;
385 MPIDI_COMM(MPIR_Process.comm_self, map).avtid = 0;
386 MPIDI_COMM(MPIR_Process.comm_self, map).size = 1;
387 MPIDI_COMM(MPIR_Process.comm_self, map).reg.offset = MPIR_Process.rank;
388 MPIDI_COMM(MPIR_Process.comm_self, local_map).mode = MPIDI_RANK_MAP_NONE;
389 MPIDIU_avt_add_ref(0);
390
391 mpi_errno = MPIR_Comm_commit(MPIR_Process.comm_self);
392 MPIR_ERR_CHECK(mpi_errno);
393 mpi_errno = MPIR_Comm_commit(MPIR_Process.comm_world);
394 MPIR_ERR_CHECK(mpi_errno);
395
396 fn_exit:
397 return mpi_errno;
398 fn_fail:
399 goto fn_exit;
400 }
401
init_av_table(void)402 static void init_av_table(void)
403 {
404 int i;
405 int size = MPIR_Process.size;
406 int rank = MPIR_Process.rank;
407
408 MPIDIU_avt_init();
409
410 MPIDI_av_table[0] = (MPIDI_av_table_t *)
411 MPL_malloc(size * sizeof(MPIDI_av_entry_t)
412 + sizeof(MPIDI_av_table_t), MPL_MEM_ADDRESS);
413
414 MPIDI_av_table[0]->size = size;
415 MPIR_Object_set_ref(MPIDI_av_table[0], 1);
416
417 MPIDI_global.node_map[0] = MPIR_Process.node_map;
418
419 MPIDI_av_table0 = MPIDI_av_table[0];
420
421 #ifdef MPIDI_BUILD_CH4_LOCALITY_INFO
422 MPIDI_global.max_node_id = MPIR_Process.num_nodes - 1;
423
424 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE,
425 (MPL_DBG_FDEST, "MPIDI_global.max_node_id = %d", MPIDI_global.max_node_id));
426
427 for (i = 0; i < size; i++) {
428 MPIDI_av_table0->table[i].is_local =
429 (MPIDI_global.node_map[0][i] == MPIDI_global.node_map[0][rank]) ? 1 : 0;
430 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE,
431 (MPL_DBG_FDEST, "WORLD RANK %d %s local", i,
432 MPIDI_av_table0->table[i].is_local ? "is" : "is not"));
433 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_GENERAL, VERBOSE,
434 (MPL_DBG_FDEST, "Node id (i) (me) %d %d", MPIDI_global.node_map[0][i],
435 MPIDI_global.node_map[0][rank]));
436 }
437 #endif
438 }
439
440 /* This local function is temporary until we decide where the
441 * following init code belongs */
generic_init(void)442 static int generic_init(void)
443 {
444 int mpi_errno = MPI_SUCCESS;
445 MPIDI_global.is_ch4u_initialized = 0;
446 MPIDIG_am_init();
447 MPIDIU_map_create((void **) &(MPIDI_global.win_map), MPL_MEM_RMA);
448 MPIDI_global.csel_root = NULL;
449 MPIDI_global.is_ch4u_initialized = 1;
450 return mpi_errno;
451 }
452
453 #if (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__POBJ)
454 #define MAX_THREAD_MODE MPI_THREAD_MULTIPLE
455 #elif (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__VCI)
456 #define MAX_THREAD_MODE MPI_THREAD_MULTIPLE
457 #elif (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__GLOBAL)
458 #define MAX_THREAD_MODE MPI_THREAD_MULTIPLE
459 #elif (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__SINGLE)
460 #define MAX_THREAD_MODE MPI_THREAD_SERIALIZED
461 #elif (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__LOCKFREE)
462 #define MAX_THREAD_MODE MPI_THREAD_SERIALIZED
463 #else
464 #error "Thread Granularity: Invalid"
465 #endif
466
MPID_Init(int requested,int * provided)467 int MPID_Init(int requested, int *provided)
468 {
469 int mpi_errno = MPI_SUCCESS, rank, size, appnum;
470 MPIR_Comm *init_comm = NULL;
471 char strerrbuf[MPIR_STRERROR_BUF_SIZE];
472
473 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_INIT);
474 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_INIT);
475
476 switch (requested) {
477 case MPI_THREAD_SINGLE:
478 case MPI_THREAD_SERIALIZED:
479 case MPI_THREAD_FUNNELED:
480 *provided = requested;
481 break;
482 case MPI_THREAD_MULTIPLE:
483 *provided = MAX_THREAD_MODE;
484 break;
485 }
486
487 #ifdef MPL_USE_DBG_LOGGING
488 MPIDI_CH4_DBG_GENERAL = MPL_dbg_class_alloc("CH4", "ch4");
489 MPIDI_CH4_DBG_MAP = MPL_dbg_class_alloc("CH4_MAP", "ch4_map");
490 MPIDI_CH4_DBG_COMM = MPL_dbg_class_alloc("CH4_COMM", "ch4_comm");
491 MPIDI_CH4_DBG_MEMORY = MPL_dbg_class_alloc("CH4_MEMORY", "ch4_memory");
492 #endif
493
494 #ifdef HAVE_SIGNAL
495 /* install signal handler for process failure notifications from hydra */
496 MPIDI_global.sigusr1_count = 0;
497 MPIDI_global.my_sigusr1_count = 0;
498 MPIDI_global.prev_sighandler = signal(SIGUSR1, MPIDI_sigusr1_handler);
499 MPIR_ERR_CHKANDJUMP1(MPIDI_global.prev_sighandler == SIG_ERR, mpi_errno, MPI_ERR_OTHER,
500 "**signal", "**signal %s",
501 MPIR_Strerror(errno, strerrbuf, MPIR_STRERROR_BUF_SIZE));
502 if (MPIDI_global.prev_sighandler == SIG_IGN || MPIDI_global.prev_sighandler == SIG_DFL)
503 MPIDI_global.prev_sighandler = NULL;
504 #endif
505
506 choose_netmod();
507
508 mpi_errno = MPIR_pmi_init();
509 MPIR_ERR_CHECK(mpi_errno);
510
511 rank = MPIR_Process.rank;
512 size = MPIR_Process.size;
513 appnum = MPIR_Process.appnum;
514
515 int err;
516 MPID_Thread_mutex_create(&MPIDIU_THREAD_PROGRESS_MUTEX, &err);
517 MPIR_Assert(err == 0);
518
519 MPID_Thread_mutex_create(&MPIDIU_THREAD_UTIL_MUTEX, &err);
520 MPIR_Assert(err == 0);
521
522 MPID_Thread_mutex_create(&MPIDIU_THREAD_MPIDIG_GLOBAL_MUTEX, &err);
523 MPIR_Assert(err == 0);
524
525 MPID_Thread_mutex_create(&MPIDIU_THREAD_SCHED_LIST_MUTEX, &err);
526 MPIR_Assert(err == 0);
527
528 MPID_Thread_mutex_create(&MPIDIU_THREAD_TSP_QUEUE_MUTEX, &err);
529 MPIR_Assert(err == 0);
530
531 #ifdef HAVE_LIBHCOLL
532 MPID_Thread_mutex_create(&MPIDIU_THREAD_HCOLL_MUTEX, &err);
533 MPIR_Assert(err == 0);
534 #endif
535
536 MPID_Thread_mutex_create(&MPIDIU_THREAD_DYNPROC_MUTEX, &err);
537 MPIR_Assert(err == 0);
538
539 #ifdef MPIDI_CH4_USE_WORK_QUEUES
540 mpi_errno = set_runtime_configurations();
541 MPIR_ERR_CHECK(mpi_errno);
542
543 MPIDI_workq_init(&MPIDI_global.workqueue);
544
545 if (MPIR_CVAR_CH4_RUNTIME_CONF_DEBUG && rank == 0)
546 print_runtime_configurations();
547 #endif
548
549 init_av_table();
550
551 mpi_errno = generic_init();
552 MPIR_ERR_CHECK(mpi_errno);
553
554 /* setup receive queue statistics */
555 mpi_errno = MPIDIG_recvq_init();
556 MPIR_ERR_CHECK(mpi_errno);
557
558 mpi_errno = create_init_comm(&init_comm);
559 MPIR_ERR_CHECK(mpi_errno);
560
561 mpi_errno = MPIDU_Init_shm_init();
562 MPIR_ERR_CHECK(mpi_errno);
563
564 /* Initialize multiple VCIs */
565 /* TODO: add checks to ensure MPIDI_vci_t is padded or aligned to MPL_CACHELINE_SIZE */
566 MPIDI_global.n_vcis = 1;
567 if (MPIR_CVAR_CH4_NUM_VCIS > 1) {
568 MPIDI_global.n_vcis = MPIR_CVAR_CH4_NUM_VCIS;
569 /* There are configured maxes that we need observe. */
570 /* TODO: check them at configure time to avoid discrepancy */
571 MPIR_Assert(MPIDI_global.n_vcis <= MPIDI_CH4_MAX_VCIS);
572 MPIR_Assert(MPIDI_global.n_vcis <= MPIR_REQUEST_NUM_POOLS);
573 }
574
575 for (int i = 0; i < MPIDI_global.n_vcis; i++) {
576 MPID_Thread_mutex_create(&MPIDI_VCI(i).lock, &err);
577 MPIR_Assert(err == 0);
578
579 /* NOTE: 1-1 vci-pool mapping */
580 MPIR_Request_register_pool_lock(i, &MPIDI_VCI(i).lock);
581
582 /* TODO: workq */
583 }
584
585 {
586 int shm_tag_bits = MPIR_TAG_BITS_DEFAULT, nm_tag_bits = MPIR_TAG_BITS_DEFAULT;
587 #ifndef MPIDI_CH4_DIRECT_NETMOD
588 mpi_errno = MPIDI_SHM_mpi_init_hook(rank, size, &shm_tag_bits);
589
590 if (mpi_errno != MPI_SUCCESS) {
591 MPIR_ERR_POPFATAL(mpi_errno);
592 }
593 #endif
594
595 mpi_errno = MPIDI_NM_mpi_init_hook(rank, size, appnum, &nm_tag_bits, init_comm);
596 if (mpi_errno != MPI_SUCCESS) {
597 MPIR_ERR_POPFATAL(mpi_errno);
598 }
599
600 /* Use the minimum tag_bits from the netmod and shmod */
601 MPIR_Process.tag_bits = MPL_MIN(shm_tag_bits, nm_tag_bits);
602 }
603
604 MPIDIG_am_check_init();
605
606 /* Initialize collective selection */
607 if (!strcmp(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE, "")) {
608 mpi_errno = MPIR_Csel_create_from_buf(MPIDI_coll_generic_json,
609 create_container, &MPIDI_global.csel_root);
610 } else {
611 mpi_errno = MPIR_Csel_create_from_file(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE,
612 create_container, &MPIDI_global.csel_root);
613 }
614 MPIR_ERR_CHECK(mpi_errno);
615
616 /* Override split_type */
617 MPIDI_global.MPIR_Comm_fns_store.split_type = MPIDI_Comm_split_type;
618 MPIR_Comm_fns = &MPIDI_global.MPIR_Comm_fns_store;
619
620 MPIR_Process.attrs.appnum = appnum;
621 MPIR_Process.attrs.io = MPI_ANY_SOURCE;
622
623 destroy_init_comm(&init_comm);
624 mpi_errno = init_builtin_comms();
625 MPIR_ERR_CHECK(mpi_errno);
626
627 MPIDI_global.is_initialized = 0;
628
629 fn_exit:
630 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_INIT);
631 return mpi_errno;
632 fn_fail:
633 goto fn_exit;
634 }
635
MPID_InitCompleted(void)636 int MPID_InitCompleted(void)
637 {
638 int mpi_errno = MPI_SUCCESS;
639 char parent_port[MPI_MAX_PORT_NAME];
640
641 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_INITCOMPLETED);
642 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_INITCOMPLETED);
643
644 mpi_errno = MPIDI_NM_post_init();
645 MPIR_ERR_CHECK(mpi_errno);
646
647 if (MPIR_Process.has_parent) {
648 mpi_errno = MPIR_pmi_kvs_get(-1, MPIDI_PARENT_PORT_KVSKEY, parent_port, MPI_MAX_PORT_NAME);
649 MPIR_ERR_CHECK(mpi_errno);
650 MPID_Comm_connect(parent_port, NULL, 0, MPIR_Process.comm_world, &MPIR_Process.comm_parent);
651 MPIR_Assert(MPIR_Process.comm_parent != NULL);
652 MPL_strncpy(MPIR_Process.comm_parent->name, "MPI_COMM_PARENT", MPI_MAX_OBJECT_NAME);
653 }
654
655 MPIDI_global.is_initialized = 1;
656
657 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_INITCOMPLETED);
658
659 fn_exit:
660 return mpi_errno;
661
662 fn_fail:
663 goto fn_exit;
664 }
665
finalize_builtin_comms(void)666 static void finalize_builtin_comms(void)
667 {
668 /* Release builtin comms */
669 MPIR_Comm_release_always(MPIR_Process.comm_world);
670 MPIR_Comm_release_always(MPIR_Process.comm_self);
671 }
672
finalize_av_table(void)673 static void finalize_av_table(void)
674 {
675 int i;
676 int max_n_avts;
677 max_n_avts = MPIDIU_get_max_n_avts();
678 for (i = 0; i < max_n_avts; i++) {
679 if (MPIDI_av_table[i] != NULL) {
680 MPIDIU_avt_release_ref(i);
681 }
682 }
683
684 MPIDIU_avt_destroy();
685 }
686
687 /* This local function is temporary until we decide where the
688 * following finalize code belongs */
generic_finalize(void)689 static void generic_finalize(void)
690 {
691 MPIDIG_am_finalize();
692 MPIDI_global.is_ch4u_initialized = 0;
693 }
694
MPID_Finalize(void)695 int MPID_Finalize(void)
696 {
697 int mpi_errno;
698 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_FINALIZE);
699 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_FINALIZE);
700
701 mpi_errno = MPIDI_NM_mpi_finalize_hook();
702 MPIR_ERR_CHECK(mpi_errno);
703 #ifndef MPIDI_CH4_DIRECT_NETMOD
704 mpi_errno = MPIDI_SHM_mpi_finalize_hook();
705 MPIR_ERR_CHECK(mpi_errno);
706 #endif
707
708 if (MPIDI_global.csel_root) {
709 mpi_errno = MPIR_Csel_free(MPIDI_global.csel_root);
710 MPIR_ERR_CHECK(mpi_errno);
711 }
712
713 finalize_builtin_comms();
714 generic_finalize();
715
716 finalize_av_table();
717
718 mpi_errno = MPIDU_Init_shm_finalize();
719 MPIR_ERR_CHECK(mpi_errno);
720
721 MPIR_pmi_finalize();
722
723 int err;
724 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_PROGRESS_MUTEX, &err);
725 MPIR_Assert(err == 0);
726
727 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_UTIL_MUTEX, &err);
728 MPIR_Assert(err == 0);
729
730 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_MPIDIG_GLOBAL_MUTEX, &err);
731 MPIR_Assert(err == 0);
732
733 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_SCHED_LIST_MUTEX, &err);
734 MPIR_Assert(err == 0);
735
736 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_TSP_QUEUE_MUTEX, &err);
737 MPIR_Assert(err == 0);
738
739 #ifdef HAVE_LIBHCOLL
740 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_HCOLL_MUTEX, &err);
741 MPIR_Assert(err == 0);
742 #endif
743
744 MPID_Thread_mutex_destroy(&MPIDIU_THREAD_DYNPROC_MUTEX, &err);
745 MPIR_Assert(err == 0);
746
747 for (int i = 0; i < MPIDI_global.n_vcis; i++) {
748 MPID_Thread_mutex_destroy(&MPIDI_VCI(i).lock, &err);
749 MPIR_Assert(err == 0);
750 }
751
752 fn_exit:
753 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_FINALIZE);
754 return mpi_errno;
755 fn_fail:
756 goto fn_exit;
757 }
758
MPID_Get_universe_size(int * universe_size)759 int MPID_Get_universe_size(int *universe_size)
760 {
761 int mpi_errno = MPI_SUCCESS;
762 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_GET_UNIVERSE_SIZE);
763 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_GET_UNIVERSE_SIZE);
764
765 mpi_errno = MPIR_pmi_get_universe_size(universe_size);
766
767 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_GET_UNIVERSE_SIZE);
768 return mpi_errno;
769 }
770
MPID_Get_processor_name(char * name,int namelen,int * resultlen)771 int MPID_Get_processor_name(char *name, int namelen, int *resultlen)
772 {
773 int mpi_errno = MPI_SUCCESS;
774 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_GET_PROCESSOR_NAME);
775 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_GET_PROCESSOR_NAME);
776
777 if (!MPIDI_global.pname_set) {
778 #ifdef HAVE_GETHOSTNAME
779
780 if (gethostname(MPIDI_global.pname, MPI_MAX_PROCESSOR_NAME) == 0)
781 MPIDI_global.pname_len = (int) strlen(MPIDI_global.pname);
782
783 #elif defined(HAVE_SYSINFO)
784
785 if (sysinfo(SI_HOSTNAME, MPIDI_global.pname, MPI_MAX_PROCESSOR_NAME) == 0)
786 MPIDI_global.pname_len = (int) strlen(MPIDI_global.pname);
787
788 #else
789 MPL_snprintf(MPIDI_global.pname, MPI_MAX_PROCESSOR_NAME, "%d",
790 MPIR_Process.comm_world->rank);
791 MPIDI_global.pname_len = (int) strlen(MPIDI_global.pname);
792 #endif
793 MPIDI_global.pname_set = 1;
794 }
795
796 MPIR_ERR_CHKANDJUMP(MPIDI_global.pname_len <= 0, mpi_errno, MPI_ERR_OTHER, "**procnamefailed");
797 MPL_strncpy(name, MPIDI_global.pname, namelen);
798
799 if (resultlen)
800 *resultlen = MPIDI_global.pname_len;
801
802 fn_exit:
803 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_GET_PROCESSOR_NAME);
804 return mpi_errno;
805 fn_fail:
806 goto fn_exit;
807 }
808
MPID_Alloc_mem(size_t size,MPIR_Info * info_ptr)809 void *MPID_Alloc_mem(size_t size, MPIR_Info * info_ptr)
810 {
811 void *p;
812 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_ALLOC_MEM);
813 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_ALLOC_MEM);
814
815 p = MPIDI_NM_mpi_alloc_mem(size, info_ptr);
816
817 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_ALLOC_MEM);
818 return p;
819 }
820
MPID_Free_mem(void * ptr)821 int MPID_Free_mem(void *ptr)
822 {
823 int mpi_errno;
824 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_FREE_MEM);
825 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_FREE_MEM);
826 mpi_errno = MPIDI_NM_mpi_free_mem(ptr);
827
828 MPIR_ERR_CHECK(mpi_errno);
829
830 fn_exit:
831 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_FREE_MEM);
832 return mpi_errno;
833 fn_fail:
834 goto fn_exit;
835 }
836
MPID_Comm_get_lpid(MPIR_Comm * comm_ptr,int idx,int * lpid_ptr,bool is_remote)837 int MPID_Comm_get_lpid(MPIR_Comm * comm_ptr, int idx, int *lpid_ptr, bool is_remote)
838 {
839 int mpi_errno = MPI_SUCCESS;
840 int avtid = 0, lpid = 0;
841 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_COMM_GET_LPID);
842 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_COMM_GET_LPID);
843
844 if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM)
845 MPIDIU_comm_rank_to_pid(comm_ptr, idx, &lpid, &avtid);
846 else if (is_remote)
847 MPIDIU_comm_rank_to_pid(comm_ptr, idx, &lpid, &avtid);
848 else {
849 MPIDIU_comm_rank_to_pid_local(comm_ptr, idx, &lpid, &avtid);
850 }
851
852 *lpid_ptr = MPIDIU_LUPID_CREATE(avtid, lpid);
853
854 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_COMM_GET_LPID);
855 return mpi_errno;
856 }
857
MPID_Get_node_id(MPIR_Comm * comm,int rank,int * id_p)858 int MPID_Get_node_id(MPIR_Comm * comm, int rank, int *id_p)
859 {
860 int mpi_errno = MPI_SUCCESS;
861 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_GET_NODE_ID);
862 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_GET_NODE_ID);
863
864 MPIDIU_get_node_id(comm, rank, id_p);
865
866 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_GET_NODE_ID);
867 return mpi_errno;
868 }
869
MPID_Get_max_node_id(MPIR_Comm * comm,int * max_id_p)870 int MPID_Get_max_node_id(MPIR_Comm * comm, int *max_id_p)
871 {
872 int mpi_errno = MPI_SUCCESS;
873 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_GET_MAX_NODE_ID);
874 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_GET_MAX_NODE_ID);
875
876 MPIDIU_get_max_node_id(comm, max_id_p);
877
878 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_GET_MAX_NODE_ID);
879 return mpi_errno;
880 }
881
MPID_Create_intercomm_from_lpids(MPIR_Comm * newcomm_ptr,int size,const int lpids[])882 int MPID_Create_intercomm_from_lpids(MPIR_Comm * newcomm_ptr, int size, const int lpids[])
883 {
884 int mpi_errno = MPI_SUCCESS, i;
885 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_CREATE_INTERCOMM_FROM_LPIDS);
886 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_CREATE_INTERCOMM_FROM_LPIDS);
887
888 MPIDI_rank_map_mlut_t *mlut = NULL;
889 MPIDI_COMM(newcomm_ptr, map).mode = MPIDI_RANK_MAP_MLUT;
890 MPIDI_COMM(newcomm_ptr, map).avtid = -1;
891 mpi_errno = MPIDIU_alloc_mlut(&mlut, size);
892 MPIR_ERR_CHECK(mpi_errno);
893 MPIDI_COMM(newcomm_ptr, map).size = size;
894 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.t = mlut;
895 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.gpid = mlut->gpid;
896
897 for (i = 0; i < size; i++) {
898 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.gpid[i].avtid = MPIDIU_LUPID_GET_AVTID(lpids[i]);
899 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.gpid[i].lpid = MPIDIU_LUPID_GET_LPID(lpids[i]);
900 MPL_DBG_MSG_FMT(MPIDI_CH4_DBG_MAP, VERBOSE,
901 (MPL_DBG_FDEST, " remote rank=%d, avtid=%d, lpid=%d", i,
902 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.gpid[i].avtid,
903 MPIDI_COMM(newcomm_ptr, map).irreg.mlut.gpid[i].lpid));
904 }
905
906 fn_exit:
907 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_CREATE_INTERCOMM_FROM_LPIDS);
908 return mpi_errno;
909 fn_fail:
910 goto fn_exit;
911 }
912
MPID_Aint_add(MPI_Aint base,MPI_Aint disp)913 MPI_Aint MPID_Aint_add(MPI_Aint base, MPI_Aint disp)
914 {
915 MPI_Aint result;
916 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_AINT_ADD);
917 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_AINT_ADD);
918 result = (MPI_Aint) ((char *) base + disp);
919 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_AINT_ADD);
920 return result;
921 }
922
MPID_Aint_diff(MPI_Aint addr1,MPI_Aint addr2)923 MPI_Aint MPID_Aint_diff(MPI_Aint addr1, MPI_Aint addr2)
924 {
925 MPI_Aint result;
926 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_AINT_DIFF);
927 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_AINT_DIFF);
928
929 result = (MPI_Aint) ((char *) addr1 - (char *) addr2);
930 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_AINT_DIFF);
931 return result;
932 }
933
MPID_Type_commit_hook(MPIR_Datatype * type)934 int MPID_Type_commit_hook(MPIR_Datatype * type)
935 {
936 int mpi_errno;
937
938 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_TYPE_COMMIT_HOOK);
939 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_TYPE_COMMIT_HOOK);
940
941 mpi_errno = MPIDI_NM_mpi_type_commit_hook(type);
942 MPIR_ERR_CHECK(mpi_errno);
943 #ifndef MPIDI_CH4_DIRECT_NETMOD
944 mpi_errno = MPIDI_SHM_mpi_type_commit_hook(type);
945 MPIR_ERR_CHECK(mpi_errno);
946 #endif
947
948 fn_exit:
949 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_TYPE_COMMIT_HOOK);
950 return mpi_errno;
951 fn_fail:
952 goto fn_exit;
953 }
954
MPID_Type_free_hook(MPIR_Datatype * type)955 int MPID_Type_free_hook(MPIR_Datatype * type)
956 {
957 int mpi_errno;
958
959 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_TYPE_FREE_HOOK);
960 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_TYPE_FREE_HOOK);
961
962 mpi_errno = MPIDI_NM_mpi_type_free_hook(type);
963 MPIR_ERR_CHECK(mpi_errno);
964 #ifndef MPIDI_CH4_DIRECT_NETMOD
965 mpi_errno = MPIDI_SHM_mpi_type_free_hook(type);
966 MPIR_ERR_CHECK(mpi_errno);
967 #endif
968
969 fn_exit:
970 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_TYPE_FREE_HOOK);
971 return mpi_errno;
972 fn_fail:
973 goto fn_exit;
974 }
975
MPID_Op_commit_hook(MPIR_Op * op)976 int MPID_Op_commit_hook(MPIR_Op * op)
977 {
978 int mpi_errno;
979
980 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_OP_COMMIT_HOOK);
981 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_OP_COMMIT_HOOK);
982
983 mpi_errno = MPIDI_NM_mpi_op_commit_hook(op);
984 MPIR_ERR_CHECK(mpi_errno);
985 #ifndef MPIDI_CH4_DIRECT_NETMOD
986 mpi_errno = MPIDI_SHM_mpi_op_commit_hook(op);
987 MPIR_ERR_CHECK(mpi_errno);
988 #endif
989
990 fn_exit:
991 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_OP_COMMIT_HOOK);
992 return mpi_errno;
993 fn_fail:
994 goto fn_exit;
995 }
996
MPID_Op_free_hook(MPIR_Op * op)997 int MPID_Op_free_hook(MPIR_Op * op)
998 {
999 int mpi_errno;
1000
1001 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_OP_FREE_HOOK);
1002 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_OP_FREE_HOOK);
1003
1004 mpi_errno = MPIDI_NM_mpi_op_free_hook(op);
1005 MPIR_ERR_CHECK(mpi_errno);
1006 #ifndef MPIDI_CH4_DIRECT_NETMOD
1007 mpi_errno = MPIDI_SHM_mpi_op_free_hook(op);
1008 MPIR_ERR_CHECK(mpi_errno);
1009 #endif
1010
1011 fn_exit:
1012 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_OP_FREE_HOOK);
1013 return mpi_errno;
1014 fn_fail:
1015 goto fn_exit;
1016 }
1017