1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #include "mpid_nem_impl.h"
7
8 /*
9 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
10
11 categories:
12 - name : FT
13 description : cvars that control behavior of fault tolerance
14
15 cvars:
16 - name : MPIR_CVAR_ENABLE_FT
17 category : FT
18 type : boolean
19 default : false
20 class : none
21 verbosity : MPI_T_VERBOSITY_USER_BASIC
22 scope : MPI_T_SCOPE_ALL_EQ
23 description : >-
24 Enable fault tolerance functions
25
26 === END_MPI_T_CVAR_INFO_BLOCK ===
27 */
28
29 #define set_request_info(rreq_, pkt_, msg_type_) \
30 { \
31 (rreq_)->status.MPI_SOURCE = (pkt_)->match.parts.rank; \
32 (rreq_)->status.MPI_TAG = (pkt_)->match.parts.tag; \
33 MPIR_STATUS_SET_COUNT((rreq_)->status, (pkt_)->data_sz); \
34 (rreq_)->dev.sender_req_id = (pkt_)->sender_req_id; \
35 (rreq_)->dev.recv_data_sz = (pkt_)->data_sz; \
36 MPIDI_Request_set_seqnum((rreq_), (pkt_)->seqnum); \
37 MPIDI_Request_set_msg_type((rreq_), (msg_type_)); \
38 }
39
40 /* request completion actions */
41 static int do_cts(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete);
42 static int do_send(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete);
43 static int do_cookie(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete);
44
45 /* packet handlers */
46 static int pkt_RTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp);
47 static int pkt_CTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp);
48 static int pkt_DONE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp);
49 static int pkt_COOKIE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp);
50
MPID_nem_lmt_pkthandler_init(MPIDI_CH3_PktHandler_Fcn * pktArray[],int arraySize)51 int MPID_nem_lmt_pkthandler_init(MPIDI_CH3_PktHandler_Fcn *pktArray[], int arraySize)
52 {
53 int mpi_errno = MPI_SUCCESS;
54 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LMT_PKTHANDLER_INIT);
55
56 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LMT_PKTHANDLER_INIT);
57
58 /* Check that the array is large enough */
59 if (arraySize <= MPIDI_CH3_PKT_END_ALL) {
60 MPIR_ERR_SETFATALANDJUMP(mpi_errno,MPI_ERR_INTERN, "**ch3|pktarraytoosmall");
61 }
62
63 pktArray[MPIDI_NEM_PKT_LMT_RTS] = pkt_RTS_handler;
64 pktArray[MPIDI_NEM_PKT_LMT_CTS] = pkt_CTS_handler;
65 pktArray[MPIDI_NEM_PKT_LMT_DONE] = pkt_DONE_handler;
66 pktArray[MPIDI_NEM_PKT_LMT_COOKIE] = pkt_COOKIE_handler;
67
68 fn_exit:
69 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LMT_PKTHANDLER_INIT);
70 return mpi_errno;
71 fn_fail:
72 goto fn_exit;
73 }
74
75 /* MPID_nem_lmt_RndvSend - Send a request to perform a rendezvous send */
MPID_nem_lmt_RndvSend(MPIR_Request ** sreq_p,const void * buf,MPI_Aint count,MPI_Datatype datatype,int dt_contig ATTRIBUTE ((unused)),intptr_t data_sz,MPI_Aint dt_true_lb ATTRIBUTE ((unused)),int rank,int tag,MPIR_Comm * comm,int context_offset)76 int MPID_nem_lmt_RndvSend(MPIR_Request **sreq_p, const void * buf, MPI_Aint count,
77 MPI_Datatype datatype, int dt_contig ATTRIBUTE((unused)),
78 intptr_t data_sz, MPI_Aint dt_true_lb ATTRIBUTE((unused)),
79 int rank, int tag, MPIR_Comm * comm, int context_offset)
80 {
81 int mpi_errno = MPI_SUCCESS;
82 MPID_PKT_DECL_CAST(upkt, MPID_nem_pkt_lmt_rts_t, rts_pkt);
83 MPIDI_VC_t *vc;
84 MPIR_Request *sreq =*sreq_p;
85 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LMT_RNDVSEND);
86
87 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LMT_RNDVSEND);
88
89 MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
90
91 /* if the lmt functions are not set, fall back to the default rendezvous code */
92 if (vc->ch.lmt_initiate_lmt == NULL)
93 {
94 mpi_errno = MPIDI_CH3_RndvSend(sreq_p, buf, count, datatype, dt_contig, data_sz, dt_true_lb, rank, tag, comm, context_offset);
95 MPIR_ERR_CHECK(mpi_errno);
96 goto fn_exit;
97 }
98
99 MPL_DBG_MSG_D(MPIDI_CH3_DBG_OTHER,VERBOSE,
100 "sending lmt RTS, data_sz=%" PRIdPTR, data_sz);
101 sreq->dev.partner_request = NULL;
102 sreq->ch.lmt_tmp_cookie.iov_len = 0;
103
104 MPIDI_Pkt_init(rts_pkt, MPIDI_NEM_PKT_LMT_RTS);
105 rts_pkt->match.parts.rank = comm->rank;
106 rts_pkt->match.parts.tag = tag;
107 rts_pkt->match.parts.context_id = comm->context_id + context_offset;
108 rts_pkt->sender_req_id = sreq->handle;
109 rts_pkt->data_sz = data_sz;
110
111 MPIDI_VC_FAI_send_seqnum(vc, seqnum);
112 MPIDI_Pkt_set_seqnum(rts_pkt, seqnum);
113 MPIDI_Request_set_seqnum(sreq, seqnum);
114 sreq->ch.vc = vc;
115
116 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
117 mpi_errno = vc->ch.lmt_initiate_lmt(vc, &upkt.p, sreq);
118 if (MPIR_CVAR_ENABLE_FT) {
119 if (MPI_SUCCESS == mpi_errno)
120 MPID_nem_lmt_rtsq_enqueue(&vc->ch.lmt_rts_queue, sreq);
121 }
122 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
123 MPIR_ERR_CHECK(mpi_errno);
124
125 fn_exit:
126 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LMT_RNDVSEND);
127 return mpi_errno;
128 fn_fail:
129 goto fn_exit;
130 }
131
132 /*
133 * This routine processes a rendezvous message once the message is matched.
134 * It is used in mpid_recv and mpid_irecv.
135 */
MPID_nem_lmt_RndvRecv(MPIDI_VC_t * vc,MPIR_Request * rreq)136 int MPID_nem_lmt_RndvRecv(MPIDI_VC_t *vc, MPIR_Request *rreq)
137 {
138 int mpi_errno = MPI_SUCCESS;
139 int complete = 0;
140 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LMT_RNDVRECV);
141
142 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LMT_RNDVRECV);
143
144 /* if the lmt functions are not set, fall back to the default rendezvous code */
145 if (vc->ch.lmt_initiate_lmt == NULL)
146 {
147 mpi_errno = MPIDI_CH3_RecvRndv(vc, rreq);
148 MPIR_ERR_CHECK(mpi_errno);
149 goto fn_exit;
150 }
151
152 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE, "lmt RTS in the request");
153
154 mpi_errno = do_cts(vc, rreq, &complete);
155 MPIR_ERR_CHECK(mpi_errno);
156
157 MPIR_Assert(complete);
158
159 fn_exit:
160 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LMT_RNDVRECV);
161 return mpi_errno;
162 fn_fail:
163 goto fn_exit;
164 }
165
pkt_RTS_handler(MPIDI_VC_t * vc,MPIDI_CH3_Pkt_t * pkt,void * data,intptr_t * buflen,MPIR_Request ** rreqp)166 static int pkt_RTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp)
167 {
168 int mpi_errno = MPI_SUCCESS;
169 MPIR_Request * rreq;
170 int found;
171 MPID_nem_pkt_lmt_rts_t * const rts_pkt = (MPID_nem_pkt_lmt_rts_t *)pkt;
172 intptr_t data_len;
173 MPIR_CHKPMEM_DECL(1);
174 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PKT_RTS_HANDLER);
175
176 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PKT_RTS_HANDLER);
177
178 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
179
180 MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST, "received LMT RTS pkt, sreq=0x%08x, rank=%d, tag=%d, context=%d, data_sz=%" PRIdPTR,
181 rts_pkt->sender_req_id, rts_pkt->match.parts.rank, rts_pkt->match.parts.tag, rts_pkt->match.parts.context_id,
182 rts_pkt->data_sz));
183
184 rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&rts_pkt->match, &found);
185 MPIR_ERR_CHKANDJUMP1(!rreq, mpi_errno,MPI_ERR_OTHER, "**nomemreq", "**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
186
187 /* If the completion counter is 0, that means that the communicator to
188 * which this message is being sent has been revoked and we shouldn't
189 * bother finishing this. */
190 if (!found && MPIR_cc_get(rreq->cc) == 0) {
191 *rreqp = NULL;
192 goto fn_exit;
193 }
194
195 set_request_info(rreq, rts_pkt, MPIDI_REQUEST_RNDV_MSG);
196
197 rreq->ch.lmt_req_id = rts_pkt->sender_req_id;
198 rreq->ch.lmt_data_sz = rts_pkt->data_sz;
199
200 data_len = *buflen;
201
202
203
204 if (data_len < rts_pkt->cookie_len)
205 {
206 /* set for the cookie to be received into the tmp_cookie in the request */
207 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"haven't received entire cookie");
208 MPIR_CHKPMEM_MALLOC(rreq->ch.lmt_tmp_cookie.iov_base, char *, rts_pkt->cookie_len, mpi_errno, "tmp cookie buf", MPL_MEM_BUFFER);
209 rreq->ch.lmt_tmp_cookie.iov_len = rts_pkt->cookie_len;
210
211 rreq->dev.iov[0] = rreq->ch.lmt_tmp_cookie;
212 rreq->dev.iov_count = 1;
213 *rreqp = rreq;
214 *buflen = 0;
215
216 if (found)
217 {
218 /* set do_cts() to be called once we've received the entire cookie */
219 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"posted request found");
220 rreq->dev.OnDataAvail = do_cts;
221 }
222 else
223 {
224 /* receive the rest of the cookie and wait for a match */
225 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
226 rreq->dev.OnDataAvail = 0;
227 MPIDI_CH3_Progress_signal_completion();
228 }
229 }
230 else
231 {
232 if (rts_pkt->cookie_len == 0)
233 {
234 /* there's no cookie to receive */
235 rreq->ch.lmt_tmp_cookie.iov_len = 0;
236 rreq->dev.iov_count = 0;
237 *buflen = 0;
238 *rreqp = NULL;
239 }
240 else
241 {
242 /* receive cookie into tmp_cookie in the request */
243 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"received entire cookie");
244 MPIR_CHKPMEM_MALLOC(rreq->ch.lmt_tmp_cookie.iov_base, char *, rts_pkt->cookie_len, mpi_errno, "tmp cookie buf", MPL_MEM_BUFFER);
245 rreq->ch.lmt_tmp_cookie.iov_len = rts_pkt->cookie_len;
246
247 MPIR_Memcpy(rreq->ch.lmt_tmp_cookie.iov_base, data, rts_pkt->cookie_len);
248 *buflen = rts_pkt->cookie_len;
249 *rreqp = NULL;
250 }
251
252 if (found)
253 {
254 /* have a matching request and the entire cookie (if any), call do_cts() */
255 int complete;
256 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"posted request found");
257 mpi_errno = do_cts(vc, rreq, &complete);
258 MPIR_ERR_CHECK(mpi_errno);
259 MPIR_Assert(complete);
260 }
261 else
262 {
263 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
264 rreq->dev.OnDataAvail = 0;
265 MPIDI_CH3_Progress_signal_completion();
266 }
267
268 }
269
270
271 MPIR_CHKPMEM_COMMIT();
272 fn_exit:
273 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
274 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PKT_RTS_HANDLER);
275 return mpi_errno;
276 fn_fail:
277 MPIR_CHKPMEM_REAP();
278 goto fn_exit;
279 }
280
pkt_CTS_handler(MPIDI_VC_t * vc,MPIDI_CH3_Pkt_t * pkt,void * data,intptr_t * buflen,MPIR_Request ** rreqp)281 static int pkt_CTS_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp)
282 {
283 MPID_nem_pkt_lmt_cts_t * const cts_pkt = (MPID_nem_pkt_lmt_cts_t *)pkt;
284 MPIR_Request *sreq;
285 MPIR_Request *rts_sreq;
286 intptr_t data_len;
287 int mpi_errno = MPI_SUCCESS;
288 MPIR_CHKPMEM_DECL(1);
289 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PKT_CTS_HANDLER);
290
291 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PKT_CTS_HANDLER);
292
293 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"received rndv CTS pkt");
294
295 data_len = *buflen;
296
297 MPIR_Request_get_ptr(cts_pkt->sender_req_id, sreq);
298
299 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
300 if (MPIR_CVAR_ENABLE_FT) {
301 /* Remove the request from the VC RTS queue. */
302 MPID_nem_lmt_rtsq_search_remove(&vc->ch.lmt_rts_queue, cts_pkt->sender_req_id, &rts_sreq);
303 }
304 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
305
306 sreq->ch.lmt_req_id = cts_pkt->receiver_req_id;
307 sreq->ch.lmt_data_sz = cts_pkt->data_sz;
308
309 /* Release the RTS request if one exists.
310 MPIR_Request_fetch_and_clear_rts_sreq() needs to be atomic to
311 prevent cancel send from cancelling the wrong (future) request.
312 If MPIR_Request_fetch_and_clear_rts_sreq() returns a NULL
313 rts_sreq, then MPID_Cancel_send() is responsible for releasing
314 the RTS request object. */
315 MPIDI_Request_fetch_and_clear_rts_sreq(sreq, &rts_sreq);
316 if (rts_sreq != NULL)
317 MPIR_Request_free(rts_sreq);
318
319 if (cts_pkt->cookie_len != 0)
320 {
321 if (data_len >= cts_pkt->cookie_len)
322 {
323 /* if whole cookie has been received, start the send */
324 sreq->ch.lmt_tmp_cookie.iov_base = data;
325 sreq->ch.lmt_tmp_cookie.iov_len = cts_pkt->cookie_len;
326 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
327 mpi_errno = vc->ch.lmt_start_send(vc, sreq, sreq->ch.lmt_tmp_cookie);
328 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
329 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
330 sreq->ch.lmt_tmp_cookie.iov_len = 0;
331 *buflen = cts_pkt->cookie_len;
332 *rreqp = NULL;
333 }
334 else
335 {
336 /* create a recv req and set up to receive the cookie into the sreq's tmp_cookie */
337 MPIR_Request *rreq;
338
339 MPIR_CHKPMEM_MALLOC(sreq->ch.lmt_tmp_cookie.iov_base, char *, cts_pkt->cookie_len, mpi_errno, "tmp cookie buf", MPL_MEM_BUFFER);
340 sreq->ch.lmt_tmp_cookie.iov_len = cts_pkt->cookie_len;
341
342 MPIDI_Request_create_rreq(rreq, mpi_errno, goto fn_fail);
343 /* FIXME: where does this request get freed? */
344
345 rreq->dev.iov[0] = sreq->ch.lmt_tmp_cookie;
346 rreq->dev.iov_count = 1;
347 rreq->ch.lmt_req = sreq;
348 rreq->dev.OnDataAvail = do_send;
349 *buflen = 0;
350 *rreqp = rreq;
351 }
352 }
353 else
354 {
355 struct iovec cookie = {0,0};
356 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
357 mpi_errno = vc->ch.lmt_start_send(vc, sreq, cookie);
358 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
359 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
360 *buflen = 0;
361 *rreqp = NULL;
362 }
363
364 fn_exit:
365 MPIR_CHKPMEM_COMMIT();
366 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PKT_CTS_HANDLER);
367 return mpi_errno;
368 fn_fail:
369 MPIR_CHKPMEM_REAP();
370 goto fn_exit;
371 }
372
pkt_DONE_handler(MPIDI_VC_t * vc,MPIDI_CH3_Pkt_t * pkt,void * data ATTRIBUTE ((unused)),intptr_t * buflen,MPIR_Request ** rreqp)373 static int pkt_DONE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data ATTRIBUTE((unused)),
374 intptr_t *buflen, MPIR_Request **rreqp)
375 {
376 int mpi_errno = MPI_SUCCESS;
377 MPID_nem_pkt_lmt_done_t * const done_pkt = (MPID_nem_pkt_lmt_done_t *)pkt;
378 MPIR_Request *req;
379 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PKT_DONE_HANDLER);
380
381 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PKT_DONE_HANDLER);
382
383 *buflen = 0;
384 MPIR_Request_get_ptr(done_pkt->req_id, req);
385
386 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
387
388 switch (MPIDI_Request_get_type(req))
389 {
390 case MPIDI_REQUEST_TYPE_RECV:
391 mpi_errno = vc->ch.lmt_done_recv(vc, req);
392 MPIR_ERR_CHECK(mpi_errno);
393 break;
394 case MPIDI_REQUEST_TYPE_SEND:
395 case MPIDI_REQUEST_TYPE_RSEND:
396 case MPIDI_REQUEST_TYPE_SSEND:
397 case MPIDI_REQUEST_TYPE_BSEND:
398 mpi_errno = vc->ch.lmt_done_send(vc, req);
399 MPIR_ERR_CHECK(mpi_errno);
400 break;
401 default:
402 MPIR_ERR_INTERNALANDJUMP(mpi_errno, "unexpected request type");
403 break;
404 }
405
406 *rreqp = NULL;
407
408 fn_exit:
409 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
410 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PKT_DONE_HANDLER);
411 return mpi_errno;
412 fn_fail:
413 goto fn_exit;
414 }
415
pkt_COOKIE_handler(MPIDI_VC_t * vc,MPIDI_CH3_Pkt_t * pkt,void * data,intptr_t * buflen,MPIR_Request ** rreqp)416 static int pkt_COOKIE_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, void *data, intptr_t *buflen, MPIR_Request **rreqp)
417 {
418 int mpi_errno = MPI_SUCCESS;
419 MPID_nem_pkt_lmt_cookie_t * const cookie_pkt = (MPID_nem_pkt_lmt_cookie_t *)pkt;
420 MPIR_Request *req;
421 intptr_t data_len;
422 MPIR_CHKPMEM_DECL(1);
423 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PKT_COOKIE_HANDLER);
424
425 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PKT_COOKIE_HANDLER);
426
427 data_len = *buflen;
428
429 if (cookie_pkt->from_sender) {
430 MPIR_Request_get_ptr(cookie_pkt->receiver_req_id, req);
431 MPIR_Assert(req != NULL);
432 req->ch.lmt_req_id = cookie_pkt->sender_req_id;
433 }
434 else {
435 MPIR_Request_get_ptr(cookie_pkt->sender_req_id, req);
436 MPIR_Assert(req != NULL);
437 req->ch.lmt_req_id = cookie_pkt->receiver_req_id;
438 }
439
440 if (cookie_pkt->cookie_len != 0)
441 {
442 if (data_len >= cookie_pkt->cookie_len)
443 {
444 /* call handle cookie with cookie data in receive buffer */
445 struct iovec cookie;
446
447 cookie.iov_base = data;
448 cookie.iov_len = cookie_pkt->cookie_len;
449 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
450 mpi_errno = vc->ch.lmt_handle_cookie(vc, req, cookie);
451 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
452 MPIR_ERR_CHECK(mpi_errno);
453
454 *rreqp = NULL;
455 *buflen = cookie_pkt->cookie_len;
456 }
457 else
458 {
459 /* create a recv req and set up to receive the cookie into the rreq's tmp_cookie */
460 MPIR_Request *rreq;
461
462 MPIDI_Request_create_rreq(rreq, mpi_errno, goto fn_fail);
463 MPIR_CHKPMEM_MALLOC(rreq->ch.lmt_tmp_cookie.iov_base, char *, cookie_pkt->cookie_len, mpi_errno, "tmp cookie buf", MPL_MEM_BUFFER);
464 /* FIXME: where does this request get freed? */
465 rreq->ch.lmt_tmp_cookie.iov_len = cookie_pkt->cookie_len;
466
467 rreq->dev.iov[0] = rreq->ch.lmt_tmp_cookie;
468 rreq->dev.iov_count = 1;
469 rreq->ch.lmt_req = req;
470 rreq->dev.OnDataAvail = do_cookie;
471 *rreqp = rreq;
472 *buflen = 0;
473 }
474 }
475 else
476 {
477 struct iovec cookie = {0,0};
478
479 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
480 mpi_errno = vc->ch.lmt_handle_cookie(vc, req, cookie);
481 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
482 MPIR_ERR_CHECK(mpi_errno);
483 *buflen = 0;
484 *rreqp = NULL;
485 }
486
487 fn_exit:
488 MPIR_CHKPMEM_COMMIT();
489 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PKT_COOKIE_HANDLER);
490 return mpi_errno;
491 fn_fail:
492 MPIR_CHKPMEM_REAP();
493 goto fn_exit;
494 }
495
496
do_cts(MPIDI_VC_t * vc,MPIR_Request * rreq,int * complete)497 static int do_cts(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete)
498 {
499 int mpi_errno = MPI_SUCCESS;
500 intptr_t data_sz;
501 int dt_contig ATTRIBUTE((unused));
502 MPI_Aint dt_true_lb ATTRIBUTE((unused));
503 MPIR_Datatype* dt_ptr;
504 struct iovec s_cookie;
505 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_DO_CTS);
506
507 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_DO_CTS);
508
509 MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"posted request found");
510
511 /* determine amount of data to be transfered and check for truncation */
512 MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
513 if (rreq->ch.lmt_data_sz > data_sz)
514 {
515 MPIR_ERR_SET2(rreq->status.MPI_ERROR, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", rreq->ch.lmt_data_sz, data_sz);
516 rreq->ch.lmt_data_sz = data_sz;
517 }
518
519 s_cookie = rreq->ch.lmt_tmp_cookie;
520
521 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
522 mpi_errno = vc->ch.lmt_start_recv(vc, rreq, s_cookie);
523 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
524 MPIR_ERR_CHECK(mpi_errno);
525
526 /* free cookie buffer allocated in RTS handler */
527 if (rreq->ch.lmt_tmp_cookie.iov_len)
528 {
529 MPL_free(rreq->ch.lmt_tmp_cookie.iov_base);
530 rreq->ch.lmt_tmp_cookie.iov_len = 0;
531 }
532
533 *complete = TRUE;
534
535 fn_exit:
536 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_DO_CTS);
537 return mpi_errno;
538 fn_fail:
539 goto fn_exit;
540 }
541
do_send(MPIDI_VC_t * vc,MPIR_Request * rreq,int * complete)542 static int do_send(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete)
543 {
544 int mpi_errno = MPI_SUCCESS;
545 struct iovec r_cookie;
546 MPIR_Request * const sreq = rreq->ch.lmt_req;
547 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_DO_SEND);
548
549 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_DO_SEND);
550
551 r_cookie = sreq->ch.lmt_tmp_cookie;
552
553 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
554 mpi_errno = vc->ch.lmt_start_send(vc, sreq, r_cookie);
555 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
556 MPIR_ERR_CHECK(mpi_errno);
557
558 /* free cookie buffer allocated in CTS handler */
559 MPL_free(sreq->ch.lmt_tmp_cookie.iov_base);
560 sreq->ch.lmt_tmp_cookie.iov_len = 0;
561
562 *complete = TRUE;
563
564 fn_exit:
565 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_DO_SEND);
566 return mpi_errno;
567 fn_fail:
568 goto fn_exit;
569 }
570
do_cookie(MPIDI_VC_t * vc,MPIR_Request * rreq,int * complete)571 static int do_cookie(MPIDI_VC_t *vc, MPIR_Request *rreq, int *complete)
572 {
573 int mpi_errno = MPI_SUCCESS;
574 struct iovec cookie;
575 MPIR_Request *req = rreq->ch.lmt_req;
576 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_DO_COOKIE);
577
578 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_DO_COOKIE);
579
580 cookie = req->ch.lmt_tmp_cookie;
581
582 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
583 mpi_errno = vc->ch.lmt_handle_cookie(vc, req, cookie);
584 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
585 if (mpi_errno) MPIR_ERR_POP (mpi_errno);
586
587 /* free cookie buffer allocated in COOKIE handler */
588 MPL_free(req->ch.lmt_tmp_cookie.iov_base);
589 req->ch.lmt_tmp_cookie.iov_len = 0;
590
591 *complete = TRUE;
592
593 fn_exit:
594 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_DO_COOKIE);
595 return mpi_errno;
596 fn_fail:
597 goto fn_exit;
598 }
599