1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #ifdef HAVE_UNISTD_H
7 #include <unistd.h>
8 #endif
9 #ifdef HAVE_SYS_PARAM_H
10 #include <sys/param.h>
11 #endif
12
13 #include "mpid_nem_impl.h"
14
15 void *MPIDI_CH3_packet_buffer = NULL;
16 int MPIDI_CH3I_my_rank = -1;
17 MPIDI_PG_t *MPIDI_CH3I_my_pg = NULL;
18
19 static int nemesis_initialized = 0;
20
split_type(MPIR_Comm * user_comm_ptr,int stype,int key,MPIR_Info * info_ptr,MPIR_Comm ** newcomm_ptr)21 static int split_type(MPIR_Comm * user_comm_ptr, int stype, int key,
22 MPIR_Info *info_ptr, MPIR_Comm ** newcomm_ptr)
23 {
24 MPIR_Comm *comm_ptr = NULL;
25 int mpi_errno = MPI_SUCCESS;
26
27 mpi_errno = MPIR_Comm_split_impl(user_comm_ptr, stype == MPI_UNDEFINED ? MPI_UNDEFINED : 0,
28 key, &comm_ptr);
29 MPIR_ERR_CHECK(mpi_errno);
30
31 if (stype == MPI_UNDEFINED) {
32 *newcomm_ptr = NULL;
33 goto fn_exit;
34 }
35
36 if (stype == MPI_COMM_TYPE_SHARED) {
37 if (MPIDI_CH3I_Shm_supported()) {
38 mpi_errno = MPIR_Comm_split_type_node_topo(comm_ptr, stype, key, info_ptr, newcomm_ptr);
39 } else {
40 mpi_errno = MPIR_Comm_split_type_self(comm_ptr, stype, key, newcomm_ptr);
41 }
42 } else if (stype == MPIX_COMM_TYPE_NEIGHBORHOOD) {
43 mpi_errno = MPIR_Comm_split_type_neighborhood(comm_ptr, stype, key, info_ptr, newcomm_ptr);
44 } else {
45 /* we don't know how to handle other split types; hand it back
46 * to the upper layer */
47 mpi_errno = MPIR_Comm_split_type(comm_ptr, stype, key, info_ptr, newcomm_ptr);
48 }
49
50 MPIR_ERR_CHECK(mpi_errno);
51
52 fn_exit:
53 if (comm_ptr)
54 MPIR_Comm_free_impl(comm_ptr);
55 return mpi_errno;
56
57 /* --BEGIN ERROR HANDLING-- */
58 fn_fail:
59 goto fn_exit;
60 /* --END ERROR HANDLING-- */
61 }
62
MPIDI_CH3I_Shm_supported(void)63 int MPIDI_CH3I_Shm_supported(void)
64 {
65 int mutex_err;
66 pthread_mutexattr_t attr;
67
68 /* Test for PTHREAD_PROCESS_SHARED support. Some platforms do not support
69 * this capability even though it is a part of the pthreads core API (e.g.,
70 * FreeBSD does not support this as of version 9.1) */
71 pthread_mutexattr_init(&attr);
72 mutex_err = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
73 pthread_mutexattr_destroy(&attr);
74
75 return !mutex_err;
76 }
77
78 static MPIR_Commops comm_fns = {
79 split_type
80 };
81
82 /* MPIDI_CH3_Init(): Initialize the nemesis channel */
MPIDI_CH3_Init(int has_parent,MPIDI_PG_t * pg_p,int pg_rank)83 int MPIDI_CH3_Init(int has_parent, MPIDI_PG_t *pg_p, int pg_rank)
84 {
85 int mpi_errno = MPI_SUCCESS;
86 int i;
87 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_INIT);
88
89 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_INIT);
90
91 /* Override split_type */
92 MPIR_Comm_fns = &comm_fns;
93
94 mpi_errno = MPID_nem_init (pg_rank, pg_p, has_parent);
95 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
96
97 nemesis_initialized = 1;
98
99 MPIDI_CH3I_my_rank = pg_rank;
100 MPIDI_CH3I_my_pg = pg_p;
101
102 /*
103 * Initialize Progress Engine
104 */
105 mpi_errno = MPIDI_CH3I_Progress_init();
106 if (mpi_errno) MPIR_ERR_SETFATALANDJUMP (mpi_errno, MPI_ERR_OTHER, "**init_progress");
107
108 for (i = 0; i < pg_p->size; i++)
109 {
110 mpi_errno = MPIDI_CH3_VC_Init(&pg_p->vct[i]);
111 MPIR_ERR_CHECK(mpi_errno);
112 }
113
114 fn_exit:
115 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_INIT);
116 return mpi_errno;
117 fn_fail:
118 goto fn_exit;
119 }
120
121 /* This function simply tells the CH3 device to use the defaults for the
122 MPI Port functions */
MPIDI_CH3_PortFnsInit(MPIDI_PortFns * portFns)123 int MPIDI_CH3_PortFnsInit( MPIDI_PortFns *portFns )
124 {
125 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_PORTFNSINIT);
126
127 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_PORTFNSINIT);
128
129 MPL_UNREFERENCED_ARG(portFns);
130
131 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_PORTFNSINIT);
132 return 0;
133 }
134
MPIDI_CH3_Get_business_card(int myRank,char * value,int length)135 int MPIDI_CH3_Get_business_card(int myRank, char *value, int length)
136 {
137 int mpi_errno = MPI_SUCCESS;
138 MPIR_FUNC_VERBOSE_STATE_DECL(MPIDI_STATE_MPIDI_CH3_GET_BUSINESS_CARD);
139
140 MPIR_FUNC_VERBOSE_ENTER(MPIDI_STATE_MPIDI_CH3_GET_BUSINESS_CARD);
141
142 mpi_errno = MPID_nem_get_business_card(myRank, value, length);
143 MPIR_ERR_CHECK(mpi_errno);
144
145 fn_exit:
146 MPIR_FUNC_VERBOSE_EXIT(MPIDI_STATE_MPIDI_CH3_GET_BUSINESS_CARD);
147 return mpi_errno;
148 fn_fail:
149 goto fn_exit;
150 }
151
152 /* Perform the channel-specific vc initialization */
MPIDI_CH3_VC_Init(MPIDI_VC_t * vc)153 int MPIDI_CH3_VC_Init( MPIDI_VC_t *vc )
154 {
155 int mpi_errno = MPI_SUCCESS;
156 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_VC_INIT);
157
158 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_VC_INIT);
159
160 /* FIXME: Circular dependency. Before calling MPIDI_CH3_Init,
161 MPID_Init calls InitPG which calls MPIDI_PG_Create which calls
162 MPIDI_CH3_VC_Init. But MPIDI_CH3_VC_Init needs nemesis to be
163 initialized. We can't call MPIDI_CH3_Init before initializing
164 the PG, because it needs the PG.
165
166 There is a hook called MPIDI_CH3_PG_Init which is called from
167 MPIDI_PG_Create before MPIDI_CH3_VC_Init, but we don't have
168 the pg_rank in that function.
169
170 Maybe this shouldn't really be a FIXME, since I believe that
171 this issue will be moot once nemesis is a device.
172
173 So what we do now, is do nothing if MPIDI_CH3_VC_Init is
174 called before MPIDI_CH3_Init, and call MPIDI_CH3_VC_Init from
175 inside MPIDI_CH3_Init after initializing nemesis
176 */
177 if (!nemesis_initialized)
178 goto fn_exit;
179
180 /* no need to initialize vc to self */
181 if (vc->pg == MPIDI_CH3I_my_pg && vc->pg_rank == MPIDI_CH3I_my_rank)
182 goto fn_exit;
183
184 vc->ch.recv_active = NULL;
185
186 mpi_errno = MPID_nem_vc_init (vc);
187 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
188
189 fn_exit:
190 fn_fail:
191 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_VC_INIT);
192 return mpi_errno;
193 }
194
MPIDI_CH3_VC_Destroy(MPIDI_VC_t * vc)195 int MPIDI_CH3_VC_Destroy(MPIDI_VC_t *vc )
196 {
197 int mpi_errno = MPI_SUCCESS;
198 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_VC_DESTROY);
199
200 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_VC_DESTROY);
201
202 /* no need to destroy vc to self, this corresponds to the optimization above
203 * in MPIDI_CH3_VC_Init */
204 if (vc->pg == MPIDI_CH3I_my_pg && vc->pg_rank == MPIDI_CH3I_my_rank) {
205 MPL_DBG_MSG_P(MPIDI_CH3_DBG_VC, VERBOSE, "skipping self vc=%p", vc);
206 goto fn_exit;
207 }
208
209 mpi_errno = MPID_nem_vc_destroy(vc);
210
211 fn_exit:
212 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_VC_DESTROY);
213 return mpi_errno;
214 }
215
216 /* MPIDI_CH3_Connect_to_root() create a new vc, and connect it to the process listening on port_name */
MPIDI_CH3_Connect_to_root(const char * port_name,MPIDI_VC_t ** new_vc)217 int MPIDI_CH3_Connect_to_root (const char *port_name, MPIDI_VC_t **new_vc)
218 {
219 int mpi_errno = MPI_SUCCESS;
220 MPIDI_VC_t * vc;
221 MPIR_CHKPMEM_DECL(1);
222 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
223
224 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
225
226 *new_vc = NULL; /* so that the err handling knows to cleanup */
227
228 MPIR_CHKPMEM_MALLOC (vc, MPIDI_VC_t *, sizeof(MPIDI_VC_t), mpi_errno, "vc", MPL_MEM_ADDRESS);
229 /* FIXME - where does this vc get freed?
230 ANSWER (goodell@) - ch3u_port.c FreeNewVC
231 (but the VC_Destroy is in this file) */
232
233 /* init ch3 portion of vc */
234 MPIDI_VC_Init (vc, NULL, 0);
235
236 /* init channel portion of vc */
237 MPIR_ERR_CHKINTERNAL(!nemesis_initialized, mpi_errno, "Nemesis not initialized");
238 vc->ch.recv_active = NULL;
239 MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
240
241 *new_vc = vc; /* we now have a valid, disconnected, temp VC */
242
243 mpi_errno = MPID_nem_connect_to_root (port_name, vc);
244 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
245
246 MPIR_CHKPMEM_COMMIT();
247 fn_exit:
248 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
249 return mpi_errno;
250 fn_fail:
251 /* freeing without giving the lower layer a chance to cleanup can lead to
252 leaks on error */
253 if (*new_vc)
254 MPIDI_CH3_VC_Destroy(*new_vc);
255 MPIR_CHKPMEM_REAP();
256 goto fn_exit;
257 }
258
259 #ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS
260 #ifdef MPL_USE_DBG_LOGGING
MPIDI_CH3_VC_GetStateString(struct MPIDI_VC * vc)261 const char * MPIDI_CH3_VC_GetStateString( struct MPIDI_VC *vc )
262 {
263 /* Nemesis doesn't have connection state associated with the VC */
264 return "N/A";
265 }
266 #endif
267 #endif
268
269 /* We don't initialize before calling MPIDI_CH3_VC_Init */
MPIDI_CH3_PG_Init(MPIDI_PG_t * pg_p)270 int MPIDI_CH3_PG_Init(MPIDI_PG_t *pg_p)
271 {
272 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_PG_INIT);
273
274 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_PG_INIT);
275 MPL_UNREFERENCED_ARG(pg_p);
276
277 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_PG_INIT);
278 return MPI_SUCCESS;
279 }
280
MPIDI_CH3_PG_Destroy(MPIDI_PG_t * pg_p)281 int MPIDI_CH3_PG_Destroy(MPIDI_PG_t *pg_p)
282 {
283 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_PG_DESTROY);
284
285 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_PG_DESTROY);
286 MPL_UNREFERENCED_ARG(pg_p);
287
288 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_PG_DESTROY);
289 return MPI_SUCCESS;
290 }
291
292
293 typedef struct initcomp_cb
294 {
295 int (* callback)(void);
296 struct initcomp_cb *next;
297 } initcomp_cb_t;
298
299 static struct {initcomp_cb_t *top;} initcomp_cb_stack;
300
301 #define INITCOMP_S_TOP() GENERIC_S_TOP(initcomp_cb_stack)
302 #define INITCOMP_S_PUSH(ep) GENERIC_S_PUSH(&initcomp_cb_stack, ep, next)
303
304 /* register a function to be called when all initialization is finished */
MPID_nem_register_initcomp_cb(int (* callback)(void))305 int MPID_nem_register_initcomp_cb(int (* callback)(void))
306 {
307 int mpi_errno = MPI_SUCCESS;
308 initcomp_cb_t *ep;
309 MPIR_CHKPMEM_DECL(1);
310 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_REGISTER_INITCOMP_CB);
311
312 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_REGISTER_INITCOMP_CB);
313 MPIR_CHKPMEM_MALLOC(ep, initcomp_cb_t *, sizeof(*ep), mpi_errno, "initcomp callback element", MPL_MEM_OTHER);
314
315 ep->callback = callback;
316 INITCOMP_S_PUSH(ep);
317
318 MPIR_CHKPMEM_COMMIT();
319 fn_exit:
320 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_REGISTER_INITCOMP_CB);
321 return mpi_errno;
322 fn_fail:
323 MPIR_CHKPMEM_REAP();
324 goto fn_exit;
325 }
326
MPIDI_CH3_InitCompleted(void)327 int MPIDI_CH3_InitCompleted(void)
328 {
329 int mpi_errno = MPI_SUCCESS;
330 initcomp_cb_t *ep;
331 initcomp_cb_t *ep_tmp;
332 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_INITCOMPLETED);
333
334 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_INITCOMPLETED);
335 ep = INITCOMP_S_TOP();
336 while (ep)
337 {
338 mpi_errno = ep->callback();
339 MPIR_ERR_CHECK(mpi_errno);
340 ep_tmp = ep;
341 ep = ep->next;
342 MPL_free(ep_tmp);
343 }
344
345 fn_exit:
346 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_INITCOMPLETED);
347 return mpi_errno;
348 fn_fail:
349 goto fn_exit;
350 }
351