1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #include "mpidimpl.h"
7 #include "mpidch4r.h"
8 #include "ch4r_rma_target_callbacks.h"
9
10 static int ack_put(MPIR_Request * rreq);
11 static int ack_cswap(MPIR_Request * rreq);
12 static int ack_acc(MPIR_Request * rreq);
13 static int ack_get_acc(MPIR_Request * rreq);
14 static int win_lock_advance(MPIR_Win * win);
15 static void win_lock_req_proc(int handler_id, const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win);
16 static void win_lock_ack_proc(int handler_id, const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win);
17 static void win_unlock_proc(const MPIDIG_win_cntrl_msg_t * info, int is_local, MPIR_Win * win);
18 static void win_complete_proc(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win);
19 static void win_post_proc(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win);
20 static void win_unlock_done(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win);
21 static int handle_acc_cmpl(MPIR_Request * rreq);
22 static int handle_get_acc_cmpl(MPIR_Request * rreq);
23 static void handle_acc_data(MPI_Aint in_data_sz, MPIR_Request * rreq);
24 static int get_target_cmpl_cb(MPIR_Request * req);
25 static int put_target_cmpl_cb(MPIR_Request * rreq);
26 static int put_dt_target_cmpl_cb(MPIR_Request * rreq);
27 static int acc_dt_target_cmpl_cb(MPIR_Request * rreq);
28 static int get_acc_dt_target_cmpl_cb(MPIR_Request * rreq);
29 static int cswap_target_cmpl_cb(MPIR_Request * rreq);
30 static int acc_target_cmpl_cb(MPIR_Request * rreq);
31 static int get_acc_target_cmpl_cb(MPIR_Request * rreq);
32 static int get_ack_target_cmpl_cb(MPIR_Request * rreq);
33 static int get_acc_ack_target_cmpl_cb(MPIR_Request * rreq);
34 static int cswap_ack_target_cmpl_cb(MPIR_Request * rreq);
35
MPIDIG_RMA_Init_targetcb_pvars(void)36 int MPIDIG_RMA_Init_targetcb_pvars(void)
37 {
38 int mpi_errno = MPI_SUCCESS;
39 /* rma_targetcb_put */
40 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
41 MPI_DOUBLE,
42 rma_targetcb_put,
43 MPI_T_VERBOSITY_MPIDEV_DETAIL,
44 MPI_T_BIND_NO_OBJECT,
45 MPIR_T_PVAR_FLAG_READONLY,
46 "RMA", "RMA:TARGETCB for Put (in seconds)");
47
48 /* rma_targetcb_put_ack */
49 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
50 MPI_DOUBLE,
51 rma_targetcb_put_ack,
52 MPI_T_VERBOSITY_MPIDEV_DETAIL,
53 MPI_T_BIND_NO_OBJECT,
54 MPIR_T_PVAR_FLAG_READONLY,
55 "RMA", "RMA:TARGETCB for Put ACK (in seconds)");
56
57 /* rma_targetcb_get */
58 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
59 MPI_DOUBLE,
60 rma_targetcb_get,
61 MPI_T_VERBOSITY_MPIDEV_DETAIL,
62 MPI_T_BIND_NO_OBJECT,
63 MPIR_T_PVAR_FLAG_READONLY,
64 "RMA", "RMA:TARGETCB for Get (in seconds)");
65
66 /* rma_targetcb_get_ack */
67 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
68 MPI_DOUBLE,
69 rma_targetcb_get_ack,
70 MPI_T_VERBOSITY_MPIDEV_DETAIL,
71 MPI_T_BIND_NO_OBJECT,
72 MPIR_T_PVAR_FLAG_READONLY,
73 "RMA", "RMA:TARGETCB for Get ACK (in seconds)");
74
75 /* rma_targetcb_cas */
76 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
77 MPI_DOUBLE,
78 rma_targetcb_cas,
79 MPI_T_VERBOSITY_MPIDEV_DETAIL,
80 MPI_T_BIND_NO_OBJECT,
81 MPIR_T_PVAR_FLAG_READONLY,
82 "RMA", "RMA:TARGETCB for Compare-and-swap (in seconds)");
83
84 /* rma_targetcb_cas_ack */
85 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
86 MPI_DOUBLE,
87 rma_targetcb_cas_ack,
88 MPI_T_VERBOSITY_MPIDEV_DETAIL,
89 MPI_T_BIND_NO_OBJECT,
90 MPIR_T_PVAR_FLAG_READONLY,
91 "RMA", "RMA:TARGETCB for Compare-and-swap ACK (in seconds)");
92
93 /* rma_targetcb_acc */
94 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
95 MPI_DOUBLE,
96 rma_targetcb_acc,
97 MPI_T_VERBOSITY_MPIDEV_DETAIL,
98 MPI_T_BIND_NO_OBJECT,
99 MPIR_T_PVAR_FLAG_READONLY,
100 "RMA", "RMA:TARGETCB for Accumulate (in seconds)");
101
102 /* rma_targetcb_get_acc */
103 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
104 MPI_DOUBLE,
105 rma_targetcb_get_acc,
106 MPI_T_VERBOSITY_MPIDEV_DETAIL,
107 MPI_T_BIND_NO_OBJECT,
108 MPIR_T_PVAR_FLAG_READONLY,
109 "RMA", "RMA:TARGETCB for Get-Accumulate (in seconds)");
110
111 /* rma_targetcb_acc_ack */
112 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
113 MPI_DOUBLE,
114 rma_targetcb_acc_ack,
115 MPI_T_VERBOSITY_MPIDEV_DETAIL,
116 MPI_T_BIND_NO_OBJECT,
117 MPIR_T_PVAR_FLAG_READONLY,
118 "RMA", "RMA:TARGETCB for Accumulate ACK (in seconds)");
119
120 /* rma_targetcb_get_acc_ack */
121 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
122 MPI_DOUBLE,
123 rma_targetcb_get_acc_ack,
124 MPI_T_VERBOSITY_MPIDEV_DETAIL,
125 MPI_T_BIND_NO_OBJECT,
126 MPIR_T_PVAR_FLAG_READONLY,
127 "RMA", "RMA:TARGETCB for Get-Accumulate ACK (in seconds)");
128
129 /* rma_targetcb_win_ctrl */
130 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
131 MPI_DOUBLE,
132 rma_targetcb_win_ctrl,
133 MPI_T_VERBOSITY_MPIDEV_DETAIL,
134 MPI_T_BIND_NO_OBJECT,
135 MPIR_T_PVAR_FLAG_READONLY,
136 "RMA", "RMA:TARGETCB for WIN CTRL (in seconds)");
137
138 /* rma_targetcb_put_dt */
139 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
140 MPI_DOUBLE,
141 rma_targetcb_put_dt,
142 MPI_T_VERBOSITY_MPIDEV_DETAIL,
143 MPI_T_BIND_NO_OBJECT,
144 MPIR_T_PVAR_FLAG_READONLY,
145 "RMA", "RMA:TARGETCB for PUT IOV (in seconds)");
146
147 /* rma_targetcb_put_dt_ack */
148 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
149 MPI_DOUBLE,
150 rma_targetcb_put_dt_ack,
151 MPI_T_VERBOSITY_MPIDEV_DETAIL,
152 MPI_T_BIND_NO_OBJECT,
153 MPIR_T_PVAR_FLAG_READONLY,
154 "RMA", "RMA:TARGETCB for PUT IOV ACK (in seconds)");
155
156 /* rma_targetcb_put_data */
157 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
158 MPI_DOUBLE,
159 rma_targetcb_put_data,
160 MPI_T_VERBOSITY_MPIDEV_DETAIL,
161 MPI_T_BIND_NO_OBJECT,
162 MPIR_T_PVAR_FLAG_READONLY,
163 "RMA", "RMA:TARGETCB for PUT DATA (in seconds)");
164
165 /* rma_targetcb_acc_dt */
166 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
167 MPI_DOUBLE,
168 rma_targetcb_acc_dt,
169 MPI_T_VERBOSITY_MPIDEV_DETAIL,
170 MPI_T_BIND_NO_OBJECT,
171 MPIR_T_PVAR_FLAG_READONLY,
172 "RMA", "RMA:TARGETCB for ACC IOV (in seconds)");
173
174 /* rma_targetcb_get_acc_dt */
175 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
176 MPI_DOUBLE,
177 rma_targetcb_get_acc_dt,
178 MPI_T_VERBOSITY_MPIDEV_DETAIL,
179 MPI_T_BIND_NO_OBJECT,
180 MPIR_T_PVAR_FLAG_READONLY,
181 "RMA", "RMA:TARGETCB for GET ACC IOV (in seconds)");
182
183 /* rma_targetcb_acc_dt_ack */
184 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
185 MPI_DOUBLE,
186 rma_targetcb_acc_dt_ack,
187 MPI_T_VERBOSITY_MPIDEV_DETAIL,
188 MPI_T_BIND_NO_OBJECT,
189 MPIR_T_PVAR_FLAG_READONLY,
190 "RMA", "RMA:TARGETCB for ACC IOV ACK (in seconds)");
191
192 /* rma_targetcb_get_acc_dt_ack */
193 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
194 MPI_DOUBLE,
195 rma_targetcb_get_acc_dt_ack,
196 MPI_T_VERBOSITY_MPIDEV_DETAIL,
197 MPI_T_BIND_NO_OBJECT,
198 MPIR_T_PVAR_FLAG_READONLY,
199 "RMA", "RMA:TARGETCB for GET ACC IOV ACK (in seconds)");
200
201 /* rma_targetcb_acc_data */
202 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
203 MPI_DOUBLE,
204 rma_targetcb_acc_data,
205 MPI_T_VERBOSITY_MPIDEV_DETAIL,
206 MPI_T_BIND_NO_OBJECT,
207 MPIR_T_PVAR_FLAG_READONLY,
208 "RMA", "RMA:TARGETCB for ACC DATA (in seconds)");
209
210 /* rma_targetcb_get_acc_data */
211 MPIR_T_PVAR_TIMER_REGISTER_STATIC(RMA,
212 MPI_DOUBLE,
213 rma_targetcb_get_acc_data,
214 MPI_T_VERBOSITY_MPIDEV_DETAIL,
215 MPI_T_BIND_NO_OBJECT,
216 MPIR_T_PVAR_FLAG_READONLY,
217 "RMA", "RMA:TARGETCB for GET ACC DATA (in seconds)");
218
219 return mpi_errno;
220 }
221
ack_put(MPIR_Request * rreq)222 static int ack_put(MPIR_Request * rreq)
223 {
224 int mpi_errno = MPI_SUCCESS;
225 MPIDIG_put_ack_msg_t ack_msg;
226
227 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACK_PUT);
228 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACK_PUT);
229
230 ack_msg.preq_ptr = MPIDIG_REQUEST(rreq, req->preq.preq_ptr);
231 #ifndef MPIDI_CH4_DIRECT_NETMOD
232 if (MPIDI_REQUEST(rreq, is_local))
233 mpi_errno =
234 MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context
235 (MPIDIG_REQUEST(rreq, req->preq.win_ptr)),
236 MPIDIG_REQUEST(rreq, rank), MPIDIG_PUT_ACK,
237 &ack_msg, sizeof(ack_msg));
238 else
239 #endif
240 {
241 mpi_errno =
242 MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context
243 (MPIDIG_REQUEST(rreq, req->preq.win_ptr)),
244 MPIDIG_REQUEST(rreq, rank), MPIDIG_PUT_ACK,
245 &ack_msg, sizeof(ack_msg));
246 }
247
248 MPIR_ERR_CHECK(mpi_errno);
249 fn_exit:
250 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACK_PUT);
251 return mpi_errno;
252 fn_fail:
253 goto fn_exit;
254 }
255
ack_cswap(MPIR_Request * rreq)256 static int ack_cswap(MPIR_Request * rreq)
257 {
258 int mpi_errno = MPI_SUCCESS, c;
259 MPIDIG_cswap_ack_msg_t ack_msg;
260 void *result_addr;
261 size_t data_sz;
262
263 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACK_CSWAP);
264 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACK_CSWAP);
265
266 MPIDI_Datatype_check_size(MPIDIG_REQUEST(rreq, req->creq.datatype), 1, data_sz);
267 result_addr = ((char *) MPIDIG_REQUEST(rreq, req->creq.data)) + data_sz;
268
269 MPIR_cc_incr(rreq->cc_ptr, &c);
270 ack_msg.req_ptr = MPIDIG_REQUEST(rreq, req->creq.creq_ptr);
271
272 #ifndef MPIDI_CH4_DIRECT_NETMOD
273 if (MPIDI_REQUEST(rreq, is_local))
274 mpi_errno =
275 MPIDI_SHM_am_isend_reply(MPIDIG_win_to_context
276 (MPIDIG_REQUEST(rreq, req->creq.win_ptr)),
277 MPIDIG_REQUEST(rreq, rank), MPIDIG_CSWAP_ACK, &ack_msg,
278 sizeof(ack_msg), result_addr, 1,
279 MPIDIG_REQUEST(rreq, req->creq.datatype), rreq);
280 else
281 #endif
282 {
283 mpi_errno =
284 MPIDI_NM_am_isend_reply(MPIDIG_win_to_context
285 (MPIDIG_REQUEST(rreq, req->creq.win_ptr)),
286 MPIDIG_REQUEST(rreq, rank), MPIDIG_CSWAP_ACK, &ack_msg,
287 sizeof(ack_msg), result_addr, 1,
288 MPIDIG_REQUEST(rreq, req->creq.datatype), rreq);
289 }
290
291 MPIR_ERR_CHECK(mpi_errno);
292 fn_exit:
293 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACK_CSWAP);
294 return mpi_errno;
295 fn_fail:
296 goto fn_exit;
297 }
298
ack_acc(MPIR_Request * rreq)299 static int ack_acc(MPIR_Request * rreq)
300 {
301 int mpi_errno = MPI_SUCCESS;
302 MPIDIG_acc_ack_msg_t ack_msg;
303
304 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACK_ACC);
305 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACK_ACC);
306
307 ack_msg.req_ptr = MPIDIG_REQUEST(rreq, req->areq.req_ptr);
308 #ifndef MPIDI_CH4_DIRECT_NETMOD
309 if (MPIDI_REQUEST(rreq, is_local))
310 mpi_errno =
311 MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context
312 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
313 MPIDIG_REQUEST(rreq, rank), MPIDIG_ACC_ACK,
314 &ack_msg, sizeof(ack_msg));
315 else
316 #endif
317 {
318 mpi_errno =
319 MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context
320 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
321 MPIDIG_REQUEST(rreq, rank), MPIDIG_ACC_ACK,
322 &ack_msg, sizeof(ack_msg));
323 }
324
325 MPIR_ERR_CHECK(mpi_errno);
326 fn_exit:
327 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACK_ACC);
328 return mpi_errno;
329 fn_fail:
330 goto fn_exit;
331 }
332
ack_get_acc(MPIR_Request * rreq)333 static int ack_get_acc(MPIR_Request * rreq)
334 {
335 int mpi_errno = MPI_SUCCESS, c;
336 MPIDIG_acc_ack_msg_t ack_msg;
337
338 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACK_GET_ACC);
339 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACK_GET_ACC);
340
341 MPIR_cc_incr(rreq->cc_ptr, &c);
342 ack_msg.req_ptr = MPIDIG_REQUEST(rreq, req->areq.req_ptr);
343
344 #ifndef MPIDI_CH4_DIRECT_NETMOD
345 if (MPIDI_REQUEST(rreq, is_local))
346 mpi_errno =
347 MPIDI_SHM_am_isend_reply(MPIDIG_win_to_context
348 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
349 MPIDIG_REQUEST(rreq, rank), MPIDIG_GET_ACC_ACK,
350 &ack_msg, sizeof(ack_msg),
351 MPIDIG_REQUEST(rreq, req->areq.data),
352 MPIDIG_REQUEST(rreq, req->areq.data_sz), MPI_BYTE, rreq);
353 else
354 #endif
355 {
356 mpi_errno =
357 MPIDI_NM_am_isend_reply(MPIDIG_win_to_context
358 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
359 MPIDIG_REQUEST(rreq, rank), MPIDIG_GET_ACC_ACK,
360 &ack_msg, sizeof(ack_msg),
361 MPIDIG_REQUEST(rreq, req->areq.data),
362 MPIDIG_REQUEST(rreq, req->areq.data_sz), MPI_BYTE, rreq);
363 }
364
365 MPIR_ERR_CHECK(mpi_errno);
366 fn_exit:
367 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACK_GET_ACC);
368 return mpi_errno;
369 fn_fail:
370 goto fn_exit;
371 }
372
373
win_lock_advance(MPIR_Win * win)374 static int win_lock_advance(MPIR_Win * win)
375 {
376 int mpi_errno = MPI_SUCCESS;
377 MPIDIG_win_lock_recvd_t *lock_recvd_q = &MPIDIG_WIN(win, sync).lock_recvd;
378
379 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_LOCK_ADVANCE);
380 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_LOCK_ADVANCE);
381
382 if ((lock_recvd_q->head != NULL) && ((lock_recvd_q->count == 0) ||
383 ((lock_recvd_q->type == MPI_LOCK_SHARED) &&
384 (lock_recvd_q->head->type == MPI_LOCK_SHARED)))) {
385 struct MPIDIG_win_lock *lock = lock_recvd_q->head;
386 lock_recvd_q->head = lock->next;
387
388 if (lock_recvd_q->head == NULL)
389 lock_recvd_q->tail = NULL;
390
391 ++lock_recvd_q->count;
392 lock_recvd_q->type = lock->type;
393
394 MPIDIG_win_cntrl_msg_t msg;
395 int handler_id;
396 msg.win_id = MPIDIG_WIN(win, win_id);
397 msg.origin_rank = win->comm_ptr->rank;
398
399 if (lock->mtype == MPIDIG_WIN_LOCK)
400 handler_id = MPIDIG_WIN_LOCK_ACK;
401 else if (lock->mtype == MPIDIG_WIN_LOCKALL)
402 handler_id = MPIDIG_WIN_LOCKALL_ACK;
403 else
404 MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**rmasync");
405
406 #ifndef MPIDI_CH4_DIRECT_NETMOD
407 if (MPIDI_rank_is_local(lock->rank, win->comm_ptr))
408 mpi_errno = MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context(win),
409 lock->rank, handler_id, &msg, sizeof(msg));
410 else
411 #endif
412 {
413 mpi_errno = MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context(win),
414 lock->rank, handler_id, &msg, sizeof(msg));
415 }
416
417 MPIR_ERR_CHECK(mpi_errno);
418 MPL_free(lock);
419
420 mpi_errno = win_lock_advance(win);
421 MPIR_ERR_CHECK(mpi_errno);
422 }
423
424 fn_exit:
425 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_LOCK_ADVANCE);
426 return mpi_errno;
427 fn_fail:
428 goto fn_exit;
429 }
430
win_lock_req_proc(int handler_id,const MPIDIG_win_cntrl_msg_t * info,MPIR_Win * win)431 static void win_lock_req_proc(int handler_id, const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win)
432 {
433 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_LOCK_REQ_PROC);
434 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_LOCK_REQ_PROC);
435
436 MPIR_T_PVAR_TIMER_START(RMA, rma_winlock_getlocallock);
437 struct MPIDIG_win_lock *lock = (struct MPIDIG_win_lock *)
438 MPL_calloc(1, sizeof(struct MPIDIG_win_lock), MPL_MEM_RMA);
439
440 lock->mtype = handler_id;
441 lock->rank = info->origin_rank;
442 lock->type = info->lock_type;
443 MPIDIG_win_lock_recvd_t *lock_recvd_q = &MPIDIG_WIN(win, sync).lock_recvd;
444 MPIR_Assert((lock_recvd_q->head != NULL) ^ (lock_recvd_q->tail == NULL));
445
446 if (lock_recvd_q->tail == NULL)
447 lock_recvd_q->head = lock;
448 else
449 lock_recvd_q->tail->next = lock;
450
451 lock_recvd_q->tail = lock;
452
453 win_lock_advance(win);
454 MPIR_T_PVAR_TIMER_END(RMA, rma_winlock_getlocallock);
455 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_LOCK_REQ_PROC);
456 return;
457 }
458
win_lock_ack_proc(int handler_id,const MPIDIG_win_cntrl_msg_t * info,MPIR_Win * win)459 static void win_lock_ack_proc(int handler_id, const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win)
460 {
461 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_LOCK_ACK_PROC);
462 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_LOCK_ACK_PROC);
463
464 if (handler_id == MPIDIG_WIN_LOCK_ACK) {
465 MPIDIG_win_target_t *target_ptr = MPIDIG_win_target_find(win, info->origin_rank);
466 MPIR_Assert(target_ptr);
467
468 MPIR_Assert((int) target_ptr->sync.lock.locked == 0);
469 target_ptr->sync.lock.locked = 1;
470 } else if (handler_id == MPIDIG_WIN_LOCKALL_ACK) {
471 MPIDIG_WIN(win, sync).lockall.allLocked += 1;
472 } else {
473 MPIR_Assert(0);
474 }
475
476 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_LOCK_ACK_PROC);
477 }
478
win_unlock_proc(const MPIDIG_win_cntrl_msg_t * info,int is_local,MPIR_Win * win)479 static void win_unlock_proc(const MPIDIG_win_cntrl_msg_t * info, int is_local, MPIR_Win * win)
480 {
481
482 int mpi_errno = MPI_SUCCESS;
483
484 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_UNLOCK_PROC);
485 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_UNLOCK_PROC);
486
487 /* NOTE: origin blocking waits in lock or lockall call till lock granted. */
488 --MPIDIG_WIN(win, sync).lock_recvd.count;
489 MPIR_Assert((int) MPIDIG_WIN(win, sync).lock_recvd.count >= 0);
490 win_lock_advance(win);
491
492 MPIDIG_win_cntrl_msg_t msg;
493 msg.win_id = MPIDIG_WIN(win, win_id);
494 msg.origin_rank = win->comm_ptr->rank;
495
496 #ifndef MPIDI_CH4_DIRECT_NETMOD
497 if (is_local)
498 mpi_errno = MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context(win),
499 info->origin_rank,
500 MPIDIG_WIN_UNLOCK_ACK, &msg, sizeof(msg));
501 else
502 #endif
503 {
504 mpi_errno = MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context(win),
505 info->origin_rank,
506 MPIDIG_WIN_UNLOCK_ACK, &msg, sizeof(msg));
507 }
508
509 MPIR_ERR_CHECK(mpi_errno);
510 fn_exit:
511 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_UNLOCK_PROC);
512 return;
513 fn_fail:
514 goto fn_exit;
515 }
516
win_complete_proc(const MPIDIG_win_cntrl_msg_t * info,MPIR_Win * win)517 static void win_complete_proc(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win)
518 {
519 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_COMPLETE_PROC);
520 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_COMPLETE_PROC);
521
522 ++MPIDIG_WIN(win, sync).sc.count;
523
524 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_COMPLETE_PROC);
525 }
526
win_post_proc(const MPIDIG_win_cntrl_msg_t * info,MPIR_Win * win)527 static void win_post_proc(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win)
528 {
529 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_POST_PROC);
530 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_POST_PROC);
531
532 ++MPIDIG_WIN(win, sync).pw.count;
533
534 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_POST_PROC);
535 }
536
537
win_unlock_done(const MPIDIG_win_cntrl_msg_t * info,MPIR_Win * win)538 static void win_unlock_done(const MPIDIG_win_cntrl_msg_t * info, MPIR_Win * win)
539 {
540 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_WIN_UNLOCK_DONE);
541 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_WIN_UNLOCK_DONE);
542
543 if (MPIDIG_WIN(win, sync).access_epoch_type == MPIDIG_EPOTYPE_LOCK) {
544 MPIDIG_win_target_t *target_ptr = MPIDIG_win_target_find(win, info->origin_rank);
545 MPIR_Assert(target_ptr);
546
547 MPIR_Assert((int) target_ptr->sync.lock.locked == 1);
548 target_ptr->sync.lock.locked = 0;
549 } else if (MPIDIG_WIN(win, sync).access_epoch_type == MPIDIG_EPOTYPE_LOCK_ALL) {
550 MPIR_Assert((int) MPIDIG_WIN(win, sync).lockall.allLocked > 0);
551 MPIDIG_WIN(win, sync).lockall.allLocked -= 1;
552 } else {
553 MPIR_Assert(0);
554 }
555
556 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_WIN_UNLOCK_DONE);
557 }
558
handle_acc_cmpl(MPIR_Request * rreq)559 static int handle_acc_cmpl(MPIR_Request * rreq)
560 {
561 int mpi_errno = MPI_SUCCESS;
562 int shm_locked ATTRIBUTE((unused)) = 0;
563 MPIR_Win *win ATTRIBUTE((unused)) = MPIDIG_REQUEST(rreq, req->areq.win_ptr);
564
565 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLE_ACC_CMPL);
566 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLE_ACC_CMPL);
567
568 /* MPIDI_CS_ENTER(); */
569
570 if (MPIDIG_REQUEST(rreq, req->areq.op) == MPI_NO_OP) {
571 MPIDIG_REQUEST(rreq, req->areq.origin_count) = MPIDIG_REQUEST(rreq, req->areq.target_count);
572 }
573 #ifndef MPIDI_CH4_DIRECT_NETMOD
574 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
575 mpi_errno = MPIDI_SHM_rma_op_cs_enter_hook(win);
576 MPIR_ERR_CHECK(mpi_errno);
577
578 shm_locked = 1;
579 }
580 #endif
581
582 mpi_errno = MPIDIG_compute_acc_op(MPIDIG_REQUEST(rreq, req->areq.data),
583 MPIDIG_REQUEST(rreq, req->areq.origin_count),
584 MPIDIG_REQUEST(rreq, req->areq.origin_datatype),
585 MPIDIG_REQUEST(rreq, req->areq.target_addr),
586 MPIDIG_REQUEST(rreq, req->areq.target_count),
587 MPIDIG_REQUEST(rreq, req->areq.target_datatype),
588 MPIDIG_REQUEST(rreq, req->areq.op), MPIDIG_ACC_SRCBUF_PACKED);
589 MPIR_ERR_CHECK(mpi_errno);
590 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->areq.target_datatype));
591 MPL_free(MPIDIG_REQUEST(rreq, req->areq.flattened_dt));
592
593 #ifndef MPIDI_CH4_DIRECT_NETMOD
594 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
595 mpi_errno = MPIDI_SHM_rma_op_cs_exit_hook(win);
596 MPIR_ERR_CHECK(mpi_errno);
597 }
598 #endif
599
600 /* MPIDI_CS_EXIT(); */
601 MPL_free(MPIDIG_REQUEST(rreq, req->areq.data));
602
603 MPIDIG_REQUEST(rreq, req->areq.data) = NULL;
604 mpi_errno = ack_acc(rreq);
605 MPIR_ERR_CHECK(mpi_errno);
606
607 MPID_Request_complete(rreq);
608 fn_exit:
609 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLE_ACC_CMPL);
610 return mpi_errno;
611 fn_fail:
612 #ifndef MPIDI_CH4_DIRECT_NETMOD
613 /* release lock if error is reported inside CS. */
614 if (shm_locked)
615 MPIDI_SHM_rma_op_cs_exit_hook(win);
616 #endif
617 goto fn_exit;
618 }
619
handle_get_acc_cmpl(MPIR_Request * rreq)620 static int handle_get_acc_cmpl(MPIR_Request * rreq)
621 {
622 int mpi_errno = MPI_SUCCESS;
623 char *original = NULL;
624 size_t data_sz;
625 int shm_locked ATTRIBUTE((unused)) = 0;
626 MPIR_Win *win ATTRIBUTE((unused)) = MPIDIG_REQUEST(rreq, req->areq.win_ptr);
627
628 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLE_GET_ACC_CMPL);
629 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLE_GET_ACC_CMPL);
630
631 data_sz = MPIDIG_REQUEST(rreq, req->areq.data_sz);
632
633 /* MPIDI_CS_ENTER(); */
634
635 original = (char *) MPL_malloc(data_sz, MPL_MEM_RMA);
636 MPIR_Assert(original);
637
638 if (MPIDIG_REQUEST(rreq, req->areq.op) == MPI_NO_OP) {
639 MPIDIG_REQUEST(rreq, req->areq.origin_count) = MPIDIG_REQUEST(rreq, req->areq.target_count);
640 MPIDIG_REQUEST(rreq, req->areq.data_sz) = data_sz;
641 }
642 #ifndef MPIDI_CH4_DIRECT_NETMOD
643 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
644 mpi_errno = MPIDI_SHM_rma_op_cs_enter_hook(win);
645 MPIR_ERR_CHECK(mpi_errno);
646 shm_locked = 1;
647 }
648 #endif
649
650 MPI_Aint actual_pack_bytes;
651 MPIR_Typerep_pack(MPIDIG_REQUEST(rreq, req->areq.target_addr),
652 MPIDIG_REQUEST(rreq, req->areq.target_count), MPIDIG_REQUEST(rreq,
653 req->
654 areq.target_datatype),
655 0, original, data_sz, &actual_pack_bytes);
656
657 mpi_errno = MPIDIG_compute_acc_op(MPIDIG_REQUEST(rreq, req->areq.data),
658 MPIDIG_REQUEST(rreq, req->areq.origin_count),
659 MPIDIG_REQUEST(rreq, req->areq.origin_datatype),
660 MPIDIG_REQUEST(rreq, req->areq.target_addr),
661 MPIDIG_REQUEST(rreq, req->areq.target_count),
662 MPIDIG_REQUEST(rreq, req->areq.target_datatype),
663 MPIDIG_REQUEST(rreq, req->areq.op), MPIDIG_ACC_SRCBUF_PACKED);
664 MPIR_ERR_CHECK(mpi_errno);
665 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->areq.target_datatype));
666 MPL_free(MPIDIG_REQUEST(rreq, req->areq.flattened_dt));
667
668 #ifndef MPIDI_CH4_DIRECT_NETMOD
669 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
670 mpi_errno = MPIDI_SHM_rma_op_cs_exit_hook(win);
671 MPIR_ERR_CHECK(mpi_errno);
672 }
673 #endif
674
675 /* MPIDI_CS_EXIT(); */
676 MPL_free(MPIDIG_REQUEST(rreq, req->areq.data));
677
678 MPIDIG_REQUEST(rreq, req->areq.data) = original;
679 mpi_errno = ack_get_acc(rreq);
680 MPIR_ERR_CHECK(mpi_errno);
681
682 MPID_Request_complete(rreq);
683
684 fn_exit:
685 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLE_GET_ACC_CMPL);
686 return mpi_errno;
687 fn_fail:
688 #ifndef MPIDI_CH4_DIRECT_NETMOD
689 /* release lock if error is reported inside CS. */
690 if (shm_locked)
691 MPIDI_SHM_rma_op_cs_exit_hook(win);
692 #endif
693 goto fn_exit;
694 }
695
handle_acc_data(MPI_Aint in_data_sz,MPIR_Request * rreq)696 static void handle_acc_data(MPI_Aint in_data_sz, MPIR_Request * rreq)
697 {
698 void *p_data = NULL;
699 size_t data_sz;
700
701 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_HANDLE_ACC_DATA);
702 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_HANDLE_ACC_DATA);
703
704 MPIDI_Datatype_check_size(MPIDIG_REQUEST(rreq, req->areq.origin_datatype),
705 MPIDIG_REQUEST(rreq, req->areq.origin_count), data_sz);
706
707 /* The origin_data can be NULL only with no-op.
708 * TODO: when no-op is set, we do not need send origin_data at all. */
709 if (data_sz) {
710 p_data = MPL_malloc(data_sz, MPL_MEM_RMA);
711 MPIR_Assert(p_data);
712 } else {
713 MPIR_Assert(MPIDIG_REQUEST(rreq, req->areq.op) == MPI_NO_OP);
714 }
715
716 MPIDIG_REQUEST(rreq, req->areq.data) = p_data;
717
718 if (MPIDIG_REQUEST(rreq, req->areq.flattened_dt)) {
719 /* FIXME: MPIR_Typerep_unflatten should allocate the new object */
720 MPIR_Datatype *dt = (MPIR_Datatype *) MPIR_Handle_obj_alloc(&MPIR_Datatype_mem);
721 MPIR_Assert(dt);
722 MPIR_Object_set_ref(dt, 1);
723 MPIR_Typerep_unflatten(dt, MPIDIG_REQUEST(rreq, req->areq.flattened_dt));
724 MPIDIG_REQUEST(rreq, req->areq.target_datatype) = dt->handle;
725 }
726
727 MPIDIG_recv_init(1, in_data_sz, p_data, data_sz, rreq);
728
729 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_HANDLE_ACC_DATA);
730 }
731
get_target_cmpl_cb(MPIR_Request * rreq)732 static int get_target_cmpl_cb(MPIR_Request * rreq)
733 {
734 int mpi_errno = MPI_SUCCESS, c;
735 MPIDIG_get_ack_msg_t get_ack;
736 MPIR_Win *win;
737 MPIR_Context_id_t context_id;
738
739 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_TARGET_CMPL_CB);
740 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_TARGET_CMPL_CB);
741
742 MPIR_cc_incr(rreq->cc_ptr, &c);
743 get_ack.greq_ptr = MPIDIG_REQUEST(rreq, req->greq.greq_ptr);
744 win = MPIDIG_REQUEST(rreq, req->greq.win_ptr);
745 context_id = MPIDIG_win_to_context(win);
746
747 if (MPIDIG_REQUEST(rreq, req->greq.flattened_dt) == NULL) {
748 #ifndef MPIDI_CH4_DIRECT_NETMOD
749 if (MPIDI_REQUEST(rreq, is_local))
750 mpi_errno = MPIDI_SHM_am_isend_reply(context_id, MPIDIG_REQUEST(rreq, rank),
751 MPIDIG_GET_ACK, &get_ack, sizeof(get_ack),
752 (void *) MPIDIG_REQUEST(rreq, req->greq.addr),
753 MPIDIG_REQUEST(rreq, req->greq.count),
754 MPIDIG_REQUEST(rreq, req->greq.datatype), rreq);
755 else
756 #endif
757 {
758 mpi_errno = MPIDI_NM_am_isend_reply(context_id, MPIDIG_REQUEST(rreq, rank),
759 MPIDIG_GET_ACK, &get_ack, sizeof(get_ack),
760 (void *) MPIDIG_REQUEST(rreq, req->greq.addr),
761 MPIDIG_REQUEST(rreq, req->greq.count),
762 MPIDIG_REQUEST(rreq, req->greq.datatype), rreq);
763 }
764
765 MPID_Request_complete(rreq);
766 MPIR_ERR_CHECK(mpi_errno);
767 goto fn_exit;
768 }
769
770 /* FIXME: MPIR_Typerep_unflatten should allocate the new object */
771 MPIR_Datatype *dt = (MPIR_Datatype *) MPIR_Handle_obj_alloc(&MPIR_Datatype_mem);
772 if (!dt) {
773 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
774 "MPIR_Datatype_mem");
775 }
776 MPIR_Object_set_ref(dt, 1);
777 MPIR_Typerep_unflatten(dt, MPIDIG_REQUEST(rreq, req->greq.flattened_dt));
778 MPIDIG_REQUEST(rreq, req->greq.dt) = dt;
779 MPIDIG_REQUEST(rreq, req->greq.count) /= dt->size;
780
781 #ifndef MPIDI_CH4_DIRECT_NETMOD
782 if (MPIDI_REQUEST(rreq, is_local))
783 mpi_errno = MPIDI_SHM_am_isend_reply(context_id, MPIDIG_REQUEST(rreq, rank),
784 MPIDIG_GET_ACK, &get_ack, sizeof(get_ack),
785 MPIDIG_REQUEST(rreq, req->greq.addr),
786 MPIDIG_REQUEST(rreq, req->greq.count), dt->handle,
787 rreq);
788 else
789 #endif
790 {
791 mpi_errno = MPIDI_NM_am_isend_reply(context_id, MPIDIG_REQUEST(rreq, rank),
792 MPIDIG_GET_ACK, &get_ack, sizeof(get_ack),
793 MPIDIG_REQUEST(rreq, req->greq.addr),
794 MPIDIG_REQUEST(rreq, req->greq.count), dt->handle,
795 rreq);
796 }
797
798 MPID_Request_complete(rreq);
799 MPIR_ERR_CHECK(mpi_errno);
800 fn_exit:
801 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_TARGET_CMPL_CB);
802 return mpi_errno;
803 fn_fail:
804 goto fn_exit;
805 }
806
put_target_cmpl_cb(MPIR_Request * rreq)807 static int put_target_cmpl_cb(MPIR_Request * rreq)
808 {
809 int mpi_errno = MPI_SUCCESS;
810
811 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PUT_TARGET_CMPL_CB);
812 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PUT_TARGET_CMPL_CB);
813
814 MPIDIG_recv_finish(rreq);
815
816 MPL_free(MPIDIG_REQUEST(rreq, req->preq.flattened_dt));
817 if (MPIDIG_REQUEST(rreq, req->preq.dt))
818 MPIR_Datatype_ptr_release(MPIDIG_REQUEST(rreq, req->preq.dt));
819
820 mpi_errno = ack_put(rreq);
821 MPIR_ERR_CHECK(mpi_errno);
822
823 MPID_Request_complete(rreq);
824 fn_exit:
825 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PUT_TARGET_CMPL_CB);
826 return mpi_errno;
827 fn_fail:
828 goto fn_exit;
829 }
830
put_dt_target_cmpl_cb(MPIR_Request * rreq)831 static int put_dt_target_cmpl_cb(MPIR_Request * rreq)
832 {
833 int mpi_errno = MPI_SUCCESS;
834 MPIDIG_put_dt_ack_msg_t ack_msg;
835
836 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_PUT_DT_TARGET_CMPL_CB);
837 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_PUT_DT_TARGET_CMPL_CB);
838
839 ack_msg.src_rank = MPIDIG_REQUEST(rreq, rank);
840 ack_msg.origin_preq_ptr = MPIDIG_REQUEST(rreq, req->preq.preq_ptr);
841 ack_msg.target_preq_ptr = rreq;
842
843 #ifndef MPIDI_CH4_DIRECT_NETMOD
844 if (MPIDI_REQUEST(rreq, is_local))
845 mpi_errno =
846 MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context
847 (MPIDIG_REQUEST(rreq, req->preq.win_ptr)),
848 MPIDIG_REQUEST(rreq, rank), MPIDIG_PUT_DT_ACK,
849 &ack_msg, sizeof(ack_msg));
850 else
851 #endif
852 {
853 mpi_errno =
854 MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context
855 (MPIDIG_REQUEST(rreq, req->preq.win_ptr)),
856 MPIDIG_REQUEST(rreq, rank), MPIDIG_PUT_DT_ACK,
857 &ack_msg, sizeof(ack_msg));
858 }
859
860 MPIR_ERR_CHECK(mpi_errno);
861
862 fn_exit:
863 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_PUT_DT_TARGET_CMPL_CB);
864 return mpi_errno;
865 fn_fail:
866 goto fn_exit;
867 }
868
acc_dt_target_cmpl_cb(MPIR_Request * rreq)869 static int acc_dt_target_cmpl_cb(MPIR_Request * rreq)
870 {
871 int mpi_errno = MPI_SUCCESS;
872 MPIDIG_acc_dt_ack_msg_t ack_msg;
873
874 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACC_DT_TARGET_CMPL_CB);
875 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACC_DT_TARGET_CMPL_CB);
876
877 ack_msg.origin_preq_ptr = MPIDIG_REQUEST(rreq, req->areq.req_ptr);
878 ack_msg.target_preq_ptr = rreq;
879
880 #ifndef MPIDI_CH4_DIRECT_NETMOD
881 if (MPIDI_REQUEST(rreq, is_local))
882 mpi_errno =
883 MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context
884 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
885 MPIDIG_REQUEST(rreq, rank), MPIDIG_ACC_DT_ACK,
886 &ack_msg, sizeof(ack_msg));
887 else
888 #endif
889 {
890 mpi_errno =
891 MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context
892 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
893 MPIDIG_REQUEST(rreq, rank), MPIDIG_ACC_DT_ACK,
894 &ack_msg, sizeof(ack_msg));
895 }
896
897 MPIR_ERR_CHECK(mpi_errno);
898
899 fn_exit:
900 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACC_DT_TARGET_CMPL_CB);
901 return mpi_errno;
902 fn_fail:
903 goto fn_exit;
904 }
905
get_acc_dt_target_cmpl_cb(MPIR_Request * rreq)906 static int get_acc_dt_target_cmpl_cb(MPIR_Request * rreq)
907 {
908 int mpi_errno = MPI_SUCCESS;
909 MPIDIG_get_acc_dt_ack_msg_t ack_msg;
910
911 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_ACC_DT_TARGET_CMPL_CB);
912 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_ACC_DT_TARGET_CMPL_CB);
913
914 ack_msg.origin_preq_ptr = MPIDIG_REQUEST(rreq, req->areq.req_ptr);
915 ack_msg.target_preq_ptr = rreq;
916
917 #ifndef MPIDI_CH4_DIRECT_NETMOD
918 if (MPIDI_REQUEST(rreq, is_local))
919 mpi_errno =
920 MPIDI_SHM_am_send_hdr_reply(MPIDIG_win_to_context
921 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
922 MPIDIG_REQUEST(rreq, rank), MPIDIG_GET_ACC_DT_ACK,
923 &ack_msg, sizeof(ack_msg));
924 else
925 #endif
926 {
927 mpi_errno =
928 MPIDI_NM_am_send_hdr_reply(MPIDIG_win_to_context
929 (MPIDIG_REQUEST(rreq, req->areq.win_ptr)),
930 MPIDIG_REQUEST(rreq, rank), MPIDIG_GET_ACC_DT_ACK,
931 &ack_msg, sizeof(ack_msg));
932 }
933
934 MPIR_ERR_CHECK(mpi_errno);
935
936 fn_exit:
937 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_ACC_DT_TARGET_CMPL_CB);
938 return mpi_errno;
939 fn_fail:
940 goto fn_exit;
941 }
942
cswap_target_cmpl_cb(MPIR_Request * rreq)943 static int cswap_target_cmpl_cb(MPIR_Request * rreq)
944 {
945 int mpi_errno = MPI_SUCCESS;
946 void *compare_addr;
947 void *origin_addr;
948 size_t data_sz;
949 MPIR_Win *win ATTRIBUTE((unused)) = NULL;
950
951 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_CSWAP_TARGET_CMPL_CB);
952 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_CSWAP_TARGET_CMPL_CB);
953
954 if (!MPIDIG_check_cmpl_order(rreq))
955 return mpi_errno;
956
957 MPIDI_Datatype_check_size(MPIDIG_REQUEST(rreq, req->creq.datatype), 1, data_sz);
958 origin_addr = MPIDIG_REQUEST(rreq, req->creq.data);
959 compare_addr = ((char *) MPIDIG_REQUEST(rreq, req->creq.data)) + data_sz;
960
961 /* MPIDI_CS_ENTER(); */
962 #ifndef MPIDI_CH4_DIRECT_NETMOD
963 win = MPIDIG_REQUEST(rreq, req->creq.win_ptr);
964 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
965 mpi_errno = MPIDI_SHM_rma_op_cs_enter_hook(win);
966 MPIR_ERR_CHECK(mpi_errno);
967 }
968 #endif
969
970 if (MPIR_Compare_equal((void *) MPIDIG_REQUEST(rreq, req->creq.addr), compare_addr,
971 MPIDIG_REQUEST(rreq, req->creq.datatype))) {
972 MPIR_Typerep_copy(compare_addr, (void *) MPIDIG_REQUEST(rreq, req->creq.addr), data_sz);
973 MPIR_Typerep_copy((void *) MPIDIG_REQUEST(rreq, req->creq.addr), origin_addr, data_sz);
974 } else {
975 MPIR_Typerep_copy(compare_addr, (void *) MPIDIG_REQUEST(rreq, req->creq.addr), data_sz);
976 }
977
978 #ifndef MPIDI_CH4_DIRECT_NETMOD
979 if (MPIDI_WIN(win, winattr) & MPIDI_WINATTR_SHM_ALLOCATED) {
980 mpi_errno = MPIDI_SHM_rma_op_cs_exit_hook(win);
981 MPIR_ERR_CHECK(mpi_errno);
982 }
983 #endif
984 /* MPIDI_CS_EXIT(); */
985
986 mpi_errno = ack_cswap(rreq);
987 MPIR_ERR_CHECK(mpi_errno);
988 MPID_Request_complete(rreq);
989 MPIDIG_progress_compl_list();
990 fn_exit:
991 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_CSWAP_TARGET_CMPL_CB);
992 return mpi_errno;
993 fn_fail:
994 goto fn_exit;
995 }
996
997
998
acc_target_cmpl_cb(MPIR_Request * rreq)999 static int acc_target_cmpl_cb(MPIR_Request * rreq)
1000 {
1001 int mpi_errno = MPI_SUCCESS;
1002 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_ACC_TARGET_CMPL_CB);
1003 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_ACC_TARGET_CMPL_CB);
1004
1005 if (!MPIDIG_check_cmpl_order(rreq))
1006 return mpi_errno;
1007
1008 mpi_errno = handle_acc_cmpl(rreq);
1009 MPIR_ERR_CHECK(mpi_errno);
1010
1011 MPIDIG_progress_compl_list();
1012 fn_exit:
1013 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_ACC_TARGET_CMPL_CB);
1014 return mpi_errno;
1015 fn_fail:
1016 goto fn_exit;
1017 }
1018
get_acc_target_cmpl_cb(MPIR_Request * rreq)1019 static int get_acc_target_cmpl_cb(MPIR_Request * rreq)
1020 {
1021 int mpi_errno = MPI_SUCCESS;
1022 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_ACC_TARGET_CMPL_CB);
1023 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_ACC_TARGET_CMPL_CB);
1024
1025 if (!MPIDIG_check_cmpl_order(rreq))
1026 return mpi_errno;
1027
1028 mpi_errno = handle_get_acc_cmpl(rreq);
1029 MPIR_ERR_CHECK(mpi_errno);
1030
1031 MPIDIG_progress_compl_list();
1032 fn_exit:
1033 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_ACC_TARGET_CMPL_CB);
1034 return mpi_errno;
1035 fn_fail:
1036 goto fn_exit;
1037 }
1038
get_ack_target_cmpl_cb(MPIR_Request * rreq)1039 static int get_ack_target_cmpl_cb(MPIR_Request * rreq)
1040 {
1041 int mpi_errno = MPI_SUCCESS;
1042 MPIR_Win *win;
1043
1044 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_ACK_TARGET_CMPL_CB);
1045 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_ACK_TARGET_CMPL_CB);
1046
1047 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->greq.target_datatype));
1048
1049 win = MPIDIG_REQUEST(rreq, req->greq.win_ptr);
1050 MPIDIG_win_remote_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1051
1052 MPIDIG_recv_finish(rreq);
1053
1054 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->greq.datatype));
1055 MPID_Request_complete(rreq);
1056 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_ACK_TARGET_CMPL_CB);
1057 return mpi_errno;
1058 }
1059
1060
get_acc_ack_target_cmpl_cb(MPIR_Request * rreq)1061 static int get_acc_ack_target_cmpl_cb(MPIR_Request * rreq)
1062 {
1063 int mpi_errno = MPI_SUCCESS;
1064 MPIR_Win *win;
1065
1066 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_GET_ACC_ACK_TARGET_CMPL_CB);
1067 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_GET_ACC_ACK_TARGET_CMPL_CB);
1068
1069 MPIDIG_recv_finish(rreq);
1070
1071 win = MPIDIG_REQUEST(rreq, req->areq.win_ptr);
1072 MPIDIG_win_remote_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1073 MPIDIG_win_remote_acc_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1074
1075 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->areq.result_datatype));
1076 MPID_Request_complete(rreq);
1077
1078 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_GET_ACC_ACK_TARGET_CMPL_CB);
1079 return mpi_errno;
1080 }
1081
cswap_ack_target_cmpl_cb(MPIR_Request * rreq)1082 static int cswap_ack_target_cmpl_cb(MPIR_Request * rreq)
1083 {
1084 int mpi_errno = MPI_SUCCESS;
1085 MPIR_Win *win;
1086
1087 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_CSWAP_ACK_TARGET_CMPL_CB);
1088 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_CSWAP_ACK_TARGET_CMPL_CB);
1089
1090 win = MPIDIG_REQUEST(rreq, req->creq.win_ptr);
1091 MPIDIG_win_remote_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1092 MPIDIG_win_remote_acc_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1093
1094 MPL_free(MPIDIG_REQUEST(rreq, req->creq.data));
1095 MPID_Request_complete(rreq);
1096
1097 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_CSWAP_ACK_TARGET_CMPL_CB);
1098 return mpi_errno;
1099 }
1100
MPIDIG_put_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1101 int MPIDIG_put_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1102 int is_local, int is_async, MPIR_Request ** req)
1103 {
1104 int mpi_errno = MPI_SUCCESS;
1105 MPIDIG_put_ack_msg_t *msg_hdr = (MPIDIG_put_ack_msg_t *) am_hdr;
1106 MPIR_Win *win;
1107 MPIR_Request *preq;
1108
1109 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PUT_ACK_TARGET_MSG_CB);
1110 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PUT_ACK_TARGET_MSG_CB);
1111 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_put_ack);
1112
1113 preq = (MPIR_Request *) msg_hdr->preq_ptr;
1114 win = MPIDIG_REQUEST(preq, req->preq.win_ptr);
1115
1116 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(preq, req->preq.target_datatype));
1117
1118 MPIDIG_win_remote_cmpl_cnt_decr(win, MPIDIG_REQUEST(preq, rank));
1119
1120 MPID_Request_complete(preq);
1121
1122 if (is_async)
1123 *req = NULL;
1124
1125 MPIDIG_rma_set_am_flag();
1126 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_put_ack);
1127 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PUT_ACK_TARGET_MSG_CB);
1128 return mpi_errno;
1129 }
1130
MPIDIG_acc_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1131 int MPIDIG_acc_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1132 int is_local, int is_async, MPIR_Request ** req)
1133 {
1134 int mpi_errno = MPI_SUCCESS;
1135 MPIDIG_acc_ack_msg_t *msg_hdr = (MPIDIG_acc_ack_msg_t *) am_hdr;
1136 MPIR_Win *win;
1137 MPIR_Request *rreq;
1138
1139 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_ACC_ACK_TARGET_MSG_CB);
1140 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_ACC_ACK_TARGET_MSG_CB);
1141 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_acc_ack);
1142
1143 rreq = (MPIR_Request *) msg_hdr->req_ptr;
1144 win = MPIDIG_REQUEST(rreq, req->areq.win_ptr);
1145
1146 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->areq.target_datatype));
1147
1148 MPIDIG_win_remote_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1149 MPIDIG_win_remote_acc_cmpl_cnt_decr(win, MPIDIG_REQUEST(rreq, rank));
1150
1151 MPID_Request_complete(rreq);
1152
1153 if (is_async)
1154 *req = NULL;
1155
1156 MPIDIG_rma_set_am_flag();
1157 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_acc_ack);
1158 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_ACC_ACK_TARGET_MSG_CB);
1159 return mpi_errno;
1160 }
1161
MPIDIG_get_acc_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1162 int MPIDIG_get_acc_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1163 int is_local, int is_async, MPIR_Request ** req)
1164 {
1165 int mpi_errno = MPI_SUCCESS;
1166 MPIDIG_get_acc_ack_msg_t *msg_hdr = (MPIDIG_get_acc_ack_msg_t *) am_hdr;
1167 MPIR_Request *rreq;
1168
1169 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACC_ACK_TARGET_MSG_CB);
1170 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACC_ACK_TARGET_MSG_CB);
1171 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_acc_ack);
1172
1173 rreq = (MPIR_Request *) msg_hdr->req_ptr;
1174
1175 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(rreq, req->areq.target_datatype));
1176
1177 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_acc_ack_target_cmpl_cb;
1178
1179 MPIDIG_REQUEST(rreq, buffer) = MPIDIG_REQUEST(rreq, req->areq.result_addr);
1180 MPIDIG_REQUEST(rreq, count) = MPIDIG_REQUEST(rreq, req->areq.result_count);
1181 MPIDIG_REQUEST(rreq, datatype) = MPIDIG_REQUEST(rreq, req->areq.result_datatype);
1182 MPIDIG_recv_type_init(in_data_sz, rreq);
1183
1184 if (is_async) {
1185 *req = rreq;
1186 } else {
1187 MPIDIG_recv_copy(data, rreq);
1188 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1189 }
1190
1191 MPIDIG_rma_set_am_flag();
1192 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_acc_ack);
1193 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACC_ACK_TARGET_MSG_CB);
1194 return mpi_errno;
1195 }
1196
MPIDIG_cswap_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1197 int MPIDIG_cswap_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1198 int is_local, int is_async, MPIR_Request ** req)
1199 {
1200 int mpi_errno = MPI_SUCCESS;
1201 MPIDIG_cswap_ack_msg_t *msg_hdr = (MPIDIG_cswap_ack_msg_t *) am_hdr;
1202 MPIR_Request *rreq;
1203 uint64_t data_sz;
1204
1205 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_CSWAP_ACK_TARGET_MSG_CB);
1206 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_CSWAP_ACK_TARGET_MSG_CB);
1207 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_cas_ack);
1208
1209 rreq = (MPIR_Request *) msg_hdr->req_ptr;
1210 MPIDI_Datatype_check_size(MPIDIG_REQUEST(rreq, req->creq.datatype), 1, data_sz);
1211 void *result_addr = MPIDIG_REQUEST(rreq, req->creq.result_addr);
1212
1213 MPIDIG_recv_init(1, in_data_sz, result_addr, data_sz, rreq);
1214
1215 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = cswap_ack_target_cmpl_cb;
1216
1217 if (is_async) {
1218 *req = rreq;
1219 } else {
1220 MPIDIG_recv_copy(data, rreq);
1221 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1222 }
1223
1224 MPIDIG_rma_set_am_flag();
1225 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_cas_ack);
1226 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_CSWAP_ACK_TARGET_MSG_CB);
1227 return mpi_errno;
1228 }
1229
1230
MPIDIG_win_ctrl_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1231 int MPIDIG_win_ctrl_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1232 int is_local, int is_async, MPIR_Request ** req)
1233 {
1234 int mpi_errno = MPI_SUCCESS;
1235 MPIDIG_win_cntrl_msg_t *msg_hdr = (MPIDIG_win_cntrl_msg_t *) am_hdr;
1236 MPIR_Win *win;
1237
1238 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_WIN_CTRL_TARGET_MSG_CB);
1239 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_WIN_CTRL_TARGET_MSG_CB);
1240 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_win_ctrl);
1241
1242 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1243 /* TODO: check output win ptr */
1244
1245 switch (handler_id) {
1246 char buff[32];
1247
1248 case MPIDIG_WIN_LOCK:
1249 case MPIDIG_WIN_LOCKALL:
1250 win_lock_req_proc(handler_id, msg_hdr, win);
1251 break;
1252
1253 case MPIDIG_WIN_LOCK_ACK:
1254 case MPIDIG_WIN_LOCKALL_ACK:
1255 win_lock_ack_proc(handler_id, msg_hdr, win);
1256 break;
1257
1258 case MPIDIG_WIN_UNLOCK:
1259 case MPIDIG_WIN_UNLOCKALL:
1260 win_unlock_proc(msg_hdr, is_local, win);
1261 break;
1262
1263 case MPIDIG_WIN_UNLOCK_ACK:
1264 case MPIDIG_WIN_UNLOCKALL_ACK:
1265 win_unlock_done(msg_hdr, win);
1266 break;
1267
1268 case MPIDIG_WIN_COMPLETE:
1269 win_complete_proc(msg_hdr, win);
1270 break;
1271
1272 case MPIDIG_WIN_POST:
1273 win_post_proc(msg_hdr, win);
1274 break;
1275
1276 default:
1277 MPL_snprintf(buff, sizeof(buff), "Invalid message type: %d\n", handler_id);
1278 MPID_Abort(NULL, MPI_ERR_INTERN, 1, buff);
1279 }
1280
1281 if (is_async)
1282 *req = NULL;
1283
1284 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_win_ctrl);
1285 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_WIN_CTRL_TARGET_MSG_CB);
1286 return mpi_errno;
1287 }
1288
MPIDIG_put_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1289 int MPIDIG_put_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1290 int is_local, int is_async, MPIR_Request ** req)
1291 {
1292 int mpi_errno = MPI_SUCCESS;
1293 MPIR_Request *rreq = NULL;
1294 uintptr_t base; /* Base address of the window */
1295 size_t offset;
1296
1297 MPIR_Win *win;
1298 MPIDIG_put_msg_t *msg_hdr = (MPIDIG_put_msg_t *) am_hdr;
1299 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PUT_TARGET_MSG_CB);
1300 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PUT_TARGET_MSG_CB);
1301 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_put);
1302
1303 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1304 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1305
1306 MPIDIG_REQUEST(rreq, req->preq.preq_ptr) = msg_hdr->preq_ptr;
1307 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1308
1309 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1310 MPIR_Assert(win);
1311
1312 base = MPIDIG_win_base_at_target(win);
1313
1314 MPIDIG_REQUEST(rreq, req->preq.win_ptr) = win;
1315
1316 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = put_target_cmpl_cb;
1317 #ifndef MPIDI_CH4_DIRECT_NETMOD
1318 MPIDI_REQUEST(rreq, is_local) = is_local;
1319 #endif
1320
1321 offset = win->disp_unit * msg_hdr->target_disp;
1322 if (msg_hdr->flattened_sz) {
1323 /* FIXME: MPIR_Typerep_unflatten should allocate the new object */
1324 MPIR_Datatype *dt = (MPIR_Datatype *) MPIR_Handle_obj_alloc(&MPIR_Datatype_mem);
1325 if (!dt) {
1326 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
1327 "MPIR_Datatype_mem");
1328 }
1329 MPIR_Object_set_ref(dt, 1);
1330 MPIR_Typerep_unflatten(dt, (char *) am_hdr + sizeof(*msg_hdr));
1331 MPIDIG_REQUEST(rreq, req->preq.flattened_dt) = NULL;
1332 MPIDIG_REQUEST(rreq, req->preq.dt) = dt;
1333
1334 MPIDIG_REQUEST(rreq, buffer) = (void *) (base + offset);
1335 MPIDIG_REQUEST(rreq, datatype) = dt->handle;
1336 MPIDIG_REQUEST(rreq, count) = msg_hdr->target_count;
1337 MPIDIG_recv_type_init(in_data_sz, rreq);
1338 } else {
1339 MPIDIG_REQUEST(rreq, req->preq.flattened_dt) = NULL;
1340 MPIDIG_REQUEST(rreq, req->preq.dt) = NULL;
1341
1342 MPIDIG_REQUEST(rreq, buffer) = (void *) (base + offset + msg_hdr->target_true_lb);
1343 MPIDIG_REQUEST(rreq, count) = msg_hdr->target_count;
1344 MPIDIG_REQUEST(rreq, datatype) = msg_hdr->target_datatype;
1345 MPIDIG_recv_type_init(in_data_sz, rreq);
1346 }
1347
1348 if (is_async) {
1349 *req = rreq;
1350 } else {
1351 MPIDIG_recv_copy(data, rreq);
1352 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1353 }
1354
1355 fn_exit:
1356 MPIDIG_rma_set_am_flag();
1357 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_put);
1358 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PUT_TARGET_MSG_CB);
1359 return mpi_errno;
1360 fn_fail:
1361 goto fn_exit;
1362 }
1363
MPIDIG_put_dt_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1364 int MPIDIG_put_dt_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1365 int is_local, int is_async, MPIR_Request ** req)
1366 {
1367 int mpi_errno = MPI_SUCCESS;
1368 MPIR_Request *rreq = NULL;
1369 uintptr_t base;
1370 size_t offset;
1371
1372 MPIR_Win *win;
1373 MPIDIG_put_msg_t *msg_hdr = (MPIDIG_put_msg_t *) am_hdr;
1374 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PUT_DT_TARGET_MSG_CB);
1375 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PUT_DT_TARGET_MSG_CB);
1376 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_put_dt);
1377
1378 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1379 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1380
1381 MPIDIG_REQUEST(rreq, req->preq.preq_ptr) = msg_hdr->preq_ptr;
1382 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1383
1384 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1385 MPIR_Assert(win);
1386
1387 MPIDIG_REQUEST(rreq, req->preq.win_ptr) = win;
1388
1389 offset = win->disp_unit * msg_hdr->target_disp;
1390 base = MPIDIG_win_base_at_target(win);
1391 MPIDIG_REQUEST(rreq, buffer) = (void *) (offset + base);
1392 MPIDIG_REQUEST(rreq, count) = msg_hdr->target_count;
1393
1394 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = put_dt_target_cmpl_cb;
1395 #ifndef MPIDI_CH4_DIRECT_NETMOD
1396 MPIDI_REQUEST(rreq, is_local) = is_local;
1397 #endif
1398
1399 MPIR_Assert(msg_hdr->flattened_sz);
1400 void *flattened_dt = MPL_malloc(msg_hdr->flattened_sz, MPL_MEM_BUFFER);
1401 MPIDIG_recv_init(1, in_data_sz, flattened_dt, msg_hdr->flattened_sz, rreq);
1402 MPIDIG_REQUEST(rreq, req->preq.flattened_dt) = flattened_dt;
1403
1404 if (is_async) {
1405 *req = rreq;
1406 } else {
1407 MPIDIG_recv_copy(data, rreq);
1408 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1409 }
1410
1411 fn_exit:
1412 MPIDIG_rma_set_am_flag();
1413 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_put_dt);
1414 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PUT_DT_TARGET_MSG_CB);
1415 return mpi_errno;
1416 fn_fail:
1417 goto fn_exit;
1418 }
1419
MPIDIG_put_dt_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1420 int MPIDIG_put_dt_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1421 int is_local, int is_async, MPIR_Request ** req)
1422 {
1423 int mpi_errno = MPI_SUCCESS;
1424 MPIR_Request *rreq, *origin_req;
1425 MPIDIG_put_dt_ack_msg_t *msg_hdr = (MPIDIG_put_dt_ack_msg_t *) am_hdr;
1426 MPIDIG_put_dat_msg_t dat_msg;
1427 MPIR_Win *win;
1428
1429 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PUT_DT_ACK_TARGET_MSG_CB);
1430 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PUT_DT_ACK_TARGET_MSG_CB);
1431 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_put_dt_ack);
1432
1433 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1434 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1435
1436 origin_req = (MPIR_Request *) msg_hdr->origin_preq_ptr;
1437 dat_msg.preq_ptr = msg_hdr->target_preq_ptr;
1438 win = MPIDIG_REQUEST(origin_req, req->preq.win_ptr);
1439
1440 #ifndef MPIDI_CH4_DIRECT_NETMOD
1441 if (is_local)
1442 mpi_errno = MPIDI_SHM_am_isend_reply(MPIDIG_win_to_context(win),
1443 MPIDIG_REQUEST(origin_req, rank),
1444 MPIDIG_PUT_DAT_REQ, &dat_msg, sizeof(dat_msg),
1445 MPIDIG_REQUEST(origin_req, req->preq.origin_addr),
1446 MPIDIG_REQUEST(origin_req, req->preq.origin_count),
1447 MPIDIG_REQUEST(origin_req, req->preq.origin_datatype),
1448 rreq);
1449 else
1450 #endif
1451 {
1452 mpi_errno = MPIDI_NM_am_isend_reply(MPIDIG_win_to_context(win),
1453 MPIDIG_REQUEST(origin_req, rank),
1454 MPIDIG_PUT_DAT_REQ, &dat_msg, sizeof(dat_msg),
1455 MPIDIG_REQUEST(origin_req, req->preq.origin_addr),
1456 MPIDIG_REQUEST(origin_req, req->preq.origin_count),
1457 MPIDIG_REQUEST(origin_req, req->preq.origin_datatype),
1458 rreq);
1459 }
1460
1461 MPIR_ERR_CHECK(mpi_errno);
1462 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(origin_req, req->preq.origin_datatype));
1463
1464 if (is_async)
1465 *req = NULL;
1466
1467 fn_exit:
1468 MPIDIG_rma_set_am_flag();
1469 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_put_dt_ack);
1470 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PUT_DT_ACK_TARGET_MSG_CB);
1471 return mpi_errno;
1472 fn_fail:
1473 goto fn_exit;
1474 }
1475
MPIDIG_acc_dt_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1476 int MPIDIG_acc_dt_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1477 int is_local, int is_async, MPIR_Request ** req)
1478 {
1479 int mpi_errno = MPI_SUCCESS;
1480 MPIR_Request *rreq, *origin_req;
1481 MPIDIG_acc_dt_ack_msg_t *msg_hdr = (MPIDIG_acc_dt_ack_msg_t *) am_hdr;
1482 MPIDIG_acc_dat_msg_t dat_msg;
1483 MPIR_Win *win;
1484
1485 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_ACC_DT_ACK_TARGET_MSG_CB);
1486 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_ACC_DT_ACK_TARGET_MSG_CB);
1487 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_acc_dt_ack);
1488
1489 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1490 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1491
1492 origin_req = (MPIR_Request *) msg_hdr->origin_preq_ptr;
1493 dat_msg.preq_ptr = msg_hdr->target_preq_ptr;
1494 win = MPIDIG_REQUEST(origin_req, req->areq.win_ptr);
1495
1496 #ifndef MPIDI_CH4_DIRECT_NETMOD
1497 if (is_local)
1498 mpi_errno = MPIDI_SHM_am_isend_reply(MPIDIG_win_to_context(win),
1499 MPIDIG_REQUEST(origin_req, rank),
1500 MPIDIG_ACC_DAT_REQ,
1501 &dat_msg, sizeof(dat_msg),
1502 MPIDIG_REQUEST(origin_req, req->areq.origin_addr),
1503 MPIDIG_REQUEST(origin_req, req->areq.origin_count),
1504 MPIDIG_REQUEST(origin_req, req->areq.origin_datatype),
1505 rreq);
1506 else
1507 #endif
1508 {
1509 mpi_errno = MPIDI_NM_am_isend_reply(MPIDIG_win_to_context(win),
1510 MPIDIG_REQUEST(origin_req, rank),
1511 MPIDIG_ACC_DAT_REQ,
1512 &dat_msg, sizeof(dat_msg),
1513 MPIDIG_REQUEST(origin_req, req->areq.origin_addr),
1514 MPIDIG_REQUEST(origin_req, req->areq.origin_count),
1515 MPIDIG_REQUEST(origin_req, req->areq.origin_datatype),
1516 rreq);
1517 }
1518
1519 MPIR_ERR_CHECK(mpi_errno);
1520 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(origin_req, req->areq.origin_datatype));
1521
1522 if (is_async)
1523 *req = NULL;
1524
1525 fn_exit:
1526 MPIDIG_rma_set_am_flag();
1527 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_acc_dt_ack);
1528 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_ACC_DT_ACK_TARGET_MSG_CB);
1529 return mpi_errno;
1530 fn_fail:
1531 goto fn_exit;
1532 }
1533
MPIDIG_get_acc_dt_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1534 int MPIDIG_get_acc_dt_ack_target_msg_cb(int handler_id, void *am_hdr, void *data,
1535 MPI_Aint in_data_sz, int is_local, int is_async,
1536 MPIR_Request ** req)
1537 {
1538 int mpi_errno = MPI_SUCCESS;
1539 MPIR_Request *rreq, *origin_req;
1540 MPIDIG_get_acc_dt_ack_msg_t *msg_hdr = (MPIDIG_get_acc_dt_ack_msg_t *) am_hdr;
1541 MPIDIG_get_acc_dat_msg_t dat_msg;
1542 MPIR_Win *win;
1543
1544 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACC_DT_ACK_TARGET_MSG_CB);
1545 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACC_DT_ACK_TARGET_MSG_CB);
1546 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_acc_dt_ack);
1547
1548 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1549 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1550
1551 origin_req = (MPIR_Request *) msg_hdr->origin_preq_ptr;
1552 dat_msg.preq_ptr = msg_hdr->target_preq_ptr;
1553 win = MPIDIG_REQUEST(origin_req, req->areq.win_ptr);
1554
1555 #ifndef MPIDI_CH4_DIRECT_NETMOD
1556 if (is_local)
1557 mpi_errno = MPIDI_SHM_am_isend_reply(MPIDIG_win_to_context(win),
1558 MPIDIG_REQUEST(origin_req, rank),
1559 MPIDIG_GET_ACC_DAT_REQ,
1560 &dat_msg, sizeof(dat_msg),
1561 MPIDIG_REQUEST(origin_req, req->areq.origin_addr),
1562 MPIDIG_REQUEST(origin_req, req->areq.origin_count),
1563 MPIDIG_REQUEST(origin_req, req->areq.origin_datatype),
1564 rreq);
1565 else
1566 #endif
1567 {
1568 mpi_errno = MPIDI_NM_am_isend_reply(MPIDIG_win_to_context(win),
1569 MPIDIG_REQUEST(origin_req, rank),
1570 MPIDIG_GET_ACC_DAT_REQ,
1571 &dat_msg, sizeof(dat_msg),
1572 MPIDIG_REQUEST(origin_req, req->areq.origin_addr),
1573 MPIDIG_REQUEST(origin_req, req->areq.origin_count),
1574 MPIDIG_REQUEST(origin_req, req->areq.origin_datatype),
1575 rreq);
1576 }
1577
1578 MPIR_ERR_CHECK(mpi_errno);
1579 MPIR_Datatype_release_if_not_builtin(MPIDIG_REQUEST(origin_req, req->areq.origin_datatype));
1580
1581 if (is_async)
1582 *req = NULL;
1583
1584 MPIDIG_rma_set_am_flag();
1585
1586 fn_exit:
1587 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_acc_dt_ack);
1588 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACC_DT_ACK_TARGET_MSG_CB);
1589 return mpi_errno;
1590 fn_fail:
1591 goto fn_exit;
1592 }
1593
MPIDIG_put_data_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1594 int MPIDIG_put_data_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1595 int is_local, int is_async, MPIR_Request ** req)
1596 {
1597 int mpi_errno = MPI_SUCCESS;
1598 MPIR_Request *rreq;
1599 MPIDIG_put_dat_msg_t *msg_hdr = (MPIDIG_put_dat_msg_t *) am_hdr;
1600
1601 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PUT_DATA_TARGET_MSG_CB);
1602 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PUT_DATA_TARGET_MSG_CB);
1603 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_put_data);
1604
1605 rreq = (MPIR_Request *) msg_hdr->preq_ptr;
1606
1607 /* FIXME: MPIR_Typerep_unflatten should allocate the new object */
1608 MPIR_Datatype *dt = (MPIR_Datatype *) MPIR_Handle_obj_alloc(&MPIR_Datatype_mem);
1609 if (!dt) {
1610 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
1611 "MPIR_Datatype_mem");
1612 }
1613 /* Note: handle is filled in by MPIR_Handle_obj_alloc() */
1614 MPIR_Object_set_ref(dt, 1);
1615 MPIR_Typerep_unflatten(dt, MPIDIG_REQUEST(rreq, req->preq.flattened_dt));
1616 MPIDIG_REQUEST(rreq, req->preq.dt) = dt;
1617 MPIDIG_REQUEST(rreq, datatype) = dt->handle;
1618 MPIDIG_REQUEST(rreq, count) /= dt->size;
1619
1620 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = put_target_cmpl_cb;
1621 MPIDIG_recv_type_init(in_data_sz, rreq);
1622
1623 if (is_async) {
1624 *req = rreq;
1625 } else {
1626 MPIDIG_recv_copy(data, rreq);
1627 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1628 }
1629
1630 fn_exit:
1631 MPIDIG_rma_set_am_flag();
1632 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_put_data);
1633 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PUT_DATA_TARGET_MSG_CB);
1634 return mpi_errno;
1635 fn_fail:
1636 goto fn_exit;
1637 }
1638
MPIDIG_acc_data_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1639 int MPIDIG_acc_data_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1640 int is_local, int is_async, MPIR_Request ** req)
1641 {
1642 int mpi_errno = MPI_SUCCESS;
1643 MPIR_Request *rreq;
1644 MPIDIG_acc_dat_msg_t *msg_hdr = (MPIDIG_acc_dat_msg_t *) am_hdr;
1645
1646 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_ACC_DATA_TARGET_MSG_CB);
1647 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_ACC_DATA_TARGET_MSG_CB);
1648 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_acc_data);
1649
1650 rreq = (MPIR_Request *) msg_hdr->preq_ptr;
1651 handle_acc_data(in_data_sz, rreq);
1652
1653 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = acc_target_cmpl_cb;
1654
1655 if (is_async) {
1656 *req = rreq;
1657 } else {
1658 MPIDIG_recv_copy(data, rreq);
1659 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1660 }
1661
1662 MPIDIG_rma_set_am_flag();
1663
1664 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_acc_data);
1665 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_ACC_DATA_TARGET_MSG_CB);
1666 return mpi_errno;
1667 }
1668
MPIDIG_get_acc_data_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1669 int MPIDIG_get_acc_data_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1670 int is_local, int is_async, MPIR_Request ** req)
1671 {
1672 int mpi_errno = MPI_SUCCESS;
1673 MPIR_Request *rreq;
1674 MPIDIG_get_acc_dat_msg_t *msg_hdr = (MPIDIG_get_acc_dat_msg_t *) am_hdr;
1675
1676 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACC_DATA_TARGET_MSG_CB);
1677 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACC_DATA_TARGET_MSG_CB);
1678 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_acc_data);
1679
1680 rreq = (MPIR_Request *) msg_hdr->preq_ptr;
1681 handle_acc_data(in_data_sz, rreq);
1682
1683 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_acc_target_cmpl_cb;
1684
1685 if (is_async) {
1686 *req = rreq;
1687 } else {
1688 MPIDIG_recv_copy(data, rreq);
1689 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1690 }
1691
1692 MPIDIG_rma_set_am_flag();
1693 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_acc_data);
1694 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACC_DATA_TARGET_MSG_CB);
1695 return mpi_errno;
1696 }
1697
MPIDIG_cswap_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1698 int MPIDIG_cswap_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1699 int is_local, int is_async, MPIR_Request ** req)
1700 {
1701 int mpi_errno = MPI_SUCCESS;
1702 MPIR_Request *rreq = NULL;
1703 size_t data_sz;
1704 MPIR_Win *win;
1705
1706 int dt_contig;
1707 void *p_data;
1708
1709 MPIDIG_cswap_req_msg_t *msg_hdr = (MPIDIG_cswap_req_msg_t *) am_hdr;
1710 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_CSWAP_TARGET_MSG_CB);
1711 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_CSWAP_TARGET_MSG_CB);
1712 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_cas);
1713
1714 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1715 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1716
1717 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = cswap_target_cmpl_cb;
1718 MPIDIG_REQUEST(rreq, req->seq_no) = MPL_atomic_fetch_add_uint64(&MPIDI_global.nxt_seq_no, 1);
1719 #ifndef MPIDI_CH4_DIRECT_NETMOD
1720 MPIDI_REQUEST(rreq, is_local) = is_local;
1721 #endif
1722
1723 MPIDI_Datatype_check_contig_size(msg_hdr->datatype, 1, dt_contig, data_sz);
1724
1725 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1726 MPIR_Assert(win);
1727
1728 uintptr_t base = MPIDIG_win_base_at_target(win);
1729 size_t offset = win->disp_unit * msg_hdr->target_disp;
1730
1731 MPIDIG_REQUEST(rreq, req->creq.win_ptr) = win;
1732 MPIDIG_REQUEST(rreq, req->creq.creq_ptr) = msg_hdr->req_ptr;
1733 MPIDIG_REQUEST(rreq, req->creq.datatype) = msg_hdr->datatype;
1734 MPIDIG_REQUEST(rreq, req->creq.addr) = (char *) base + offset;
1735 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1736
1737 MPIR_Assert(dt_contig == 1);
1738 p_data = MPL_malloc(data_sz * 2, MPL_MEM_RMA);
1739 MPIR_Assert(p_data);
1740
1741 MPIDIG_REQUEST(rreq, req->creq.data) = p_data;
1742
1743 MPIDIG_recv_init(1, in_data_sz, p_data, data_sz * 2, rreq);
1744
1745 if (is_async) {
1746 *req = rreq;
1747 } else {
1748 MPIDIG_recv_copy(data, rreq);
1749 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1750 }
1751
1752 fn_exit:
1753 MPIDIG_rma_set_am_flag();
1754 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_cas);
1755 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_CSWAP_TARGET_MSG_CB);
1756 return mpi_errno;
1757 fn_fail:
1758 goto fn_exit;
1759 }
1760
MPIDIG_acc_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1761 int MPIDIG_acc_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1762 int is_local, int is_async, MPIR_Request ** req)
1763 {
1764 int mpi_errno = MPI_SUCCESS;
1765 MPIR_Request *rreq = NULL;
1766 size_t data_sz;
1767 void *p_data = NULL;
1768 MPIR_Win *win;
1769
1770 MPIDIG_acc_req_msg_t *msg_hdr = (MPIDIG_acc_req_msg_t *) am_hdr;
1771 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_ACC_TARGET_MSG_CB);
1772 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_ACC_TARGET_MSG_CB);
1773 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_acc);
1774
1775 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1776 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1777
1778 MPIDI_Datatype_check_size(msg_hdr->origin_datatype, msg_hdr->origin_count, data_sz);
1779 if (data_sz) {
1780 p_data = MPL_malloc(data_sz, MPL_MEM_RMA);
1781 MPIR_Assert(p_data);
1782 }
1783
1784 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = acc_target_cmpl_cb;
1785 MPIDIG_REQUEST(rreq, req->seq_no) = MPL_atomic_fetch_add_uint64(&MPIDI_global.nxt_seq_no, 1);
1786 #ifndef MPIDI_CH4_DIRECT_NETMOD
1787 MPIDI_REQUEST(rreq, is_local) = is_local;
1788 #endif
1789
1790 MPIDIG_recv_init(1, in_data_sz, p_data, data_sz, rreq);
1791
1792 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1793 MPIR_Assert(win);
1794
1795 uintptr_t base = MPIDIG_win_base_at_target(win);
1796 size_t offset = win->disp_unit * msg_hdr->target_disp;
1797
1798 MPIDIG_REQUEST(rreq, req->areq.win_ptr) = win;
1799 MPIDIG_REQUEST(rreq, req->areq.req_ptr) = msg_hdr->req_ptr;
1800 MPIDIG_REQUEST(rreq, req->areq.origin_datatype) = msg_hdr->origin_datatype;
1801 MPIDIG_REQUEST(rreq, req->areq.target_datatype) = msg_hdr->target_datatype;
1802 MPIDIG_REQUEST(rreq, req->areq.origin_count) = msg_hdr->origin_count;
1803 MPIDIG_REQUEST(rreq, req->areq.target_count) = msg_hdr->target_count;
1804 MPIDIG_REQUEST(rreq, req->areq.target_addr) = (char *) base + offset;
1805 MPIDIG_REQUEST(rreq, req->areq.op) = msg_hdr->op;
1806 MPIDIG_REQUEST(rreq, req->areq.data) = p_data;
1807 MPIDIG_REQUEST(rreq, req->areq.flattened_dt) = NULL;
1808 MPIDIG_REQUEST(rreq, req->areq.data_sz) = msg_hdr->result_data_sz;
1809 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1810
1811 if (msg_hdr->flattened_sz) {
1812 /* FIXME: MPIR_Typerep_unflatten should allocate the new object */
1813 MPIR_Datatype *dt = (MPIR_Datatype *) MPIR_Handle_obj_alloc(&MPIR_Datatype_mem);
1814 if (!dt) {
1815 MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s",
1816 "MPIR_Datatype_mem");
1817 }
1818 MPIR_Object_set_ref(dt, 1);
1819 MPIR_Typerep_unflatten(dt, (char *) am_hdr + sizeof(*msg_hdr));
1820 MPIDIG_REQUEST(rreq, req->areq.target_datatype) = dt->handle;
1821 }
1822
1823 if (is_async) {
1824 *req = rreq;
1825 } else {
1826 MPIDIG_recv_copy(data, rreq);
1827 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1828 }
1829
1830 fn_exit:
1831 MPIDIG_rma_set_am_flag();
1832 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_acc);
1833 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_ACC_TARGET_MSG_CB);
1834 return mpi_errno;
1835 fn_fail:
1836 goto fn_exit;
1837 }
1838
1839
MPIDIG_get_acc_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1840 int MPIDIG_get_acc_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1841 int is_local, int is_async, MPIR_Request ** req)
1842 {
1843 int mpi_errno = MPI_SUCCESS;
1844
1845 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACC_TARGET_MSG_CB);
1846 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACC_TARGET_MSG_CB);
1847 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_acc);
1848
1849 /* the same handling processing as ACC except the completion handler function. */
1850 /* set is_async to 1 so we can get rreq back */
1851 MPIR_Request *rreq;
1852 mpi_errno = MPIDIG_acc_target_msg_cb(handler_id, am_hdr, data, in_data_sz, is_local, 1, &rreq);
1853
1854 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_acc_target_cmpl_cb;
1855
1856 if (is_async) {
1857 *req = rreq;
1858 } else {
1859 MPIDIG_recv_copy(data, rreq);
1860 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1861 }
1862
1863 MPIDIG_rma_set_am_flag();
1864 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_acc);
1865 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACC_TARGET_MSG_CB);
1866 return mpi_errno;
1867 }
1868
MPIDIG_acc_dt_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1869 int MPIDIG_acc_dt_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1870 int is_local, int is_async, MPIR_Request ** req)
1871 {
1872 int mpi_errno = MPI_SUCCESS;
1873 MPIR_Request *rreq = NULL;
1874 MPIR_Win *win;
1875 uintptr_t base;
1876 size_t offset;
1877
1878 MPIDIG_acc_req_msg_t *msg_hdr = (MPIDIG_acc_req_msg_t *) am_hdr;
1879 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_ACC_DT_TARGET_MSG_CB);
1880 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_ACC_DT_TARGET_MSG_CB);
1881 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_acc_dt);
1882
1883 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1884 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1885
1886 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1887 MPIR_Assert(win);
1888
1889 base = MPIDIG_win_base_at_target(win);
1890 offset = win->disp_unit * msg_hdr->target_disp;
1891
1892 MPIDIG_REQUEST(rreq, req->areq.win_ptr) = win;
1893 MPIDIG_REQUEST(rreq, req->areq.req_ptr) = msg_hdr->req_ptr;
1894 MPIDIG_REQUEST(rreq, req->areq.origin_datatype) = msg_hdr->origin_datatype;
1895 MPIDIG_REQUEST(rreq, req->areq.target_datatype) = MPI_DATATYPE_NULL;
1896 MPIDIG_REQUEST(rreq, req->areq.origin_count) = msg_hdr->origin_count;
1897 MPIDIG_REQUEST(rreq, req->areq.target_count) = msg_hdr->target_count;
1898 MPIDIG_REQUEST(rreq, req->areq.target_addr) = (void *) (offset + base);
1899 MPIDIG_REQUEST(rreq, req->areq.op) = msg_hdr->op;
1900 MPIDIG_REQUEST(rreq, req->areq.data_sz) = msg_hdr->result_data_sz;
1901 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1902
1903 void *flattened_dt = MPL_malloc(msg_hdr->flattened_sz, MPL_MEM_RMA);
1904 MPIR_Assert(flattened_dt);
1905 MPIDIG_REQUEST(rreq, req->areq.flattened_dt) = flattened_dt;
1906 MPIDIG_recv_init(1, in_data_sz, flattened_dt, msg_hdr->flattened_sz, rreq);
1907
1908 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = acc_dt_target_cmpl_cb;
1909 MPIDIG_REQUEST(rreq, req->seq_no) = MPL_atomic_fetch_add_uint64(&MPIDI_global.nxt_seq_no, 1);
1910 #ifndef MPIDI_CH4_DIRECT_NETMOD
1911 MPIDI_REQUEST(rreq, is_local) = is_local;
1912 #endif
1913
1914 if (is_async) {
1915 *req = rreq;
1916 } else {
1917 MPIDIG_recv_copy(data, rreq);
1918 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1919 }
1920
1921 fn_exit:
1922 MPIDIG_rma_set_am_flag();
1923 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_acc_dt);
1924 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_ACC_DT_TARGET_MSG_CB);
1925 return mpi_errno;
1926 fn_fail:
1927 goto fn_exit;
1928 }
1929
MPIDIG_get_acc_dt_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1930 int MPIDIG_get_acc_dt_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1931 int is_local, int is_async, MPIR_Request ** req)
1932 {
1933 int mpi_errno = MPI_SUCCESS;
1934
1935 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACC_DT_TARGET_MSG_CB);
1936 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACC_DT_TARGET_MSG_CB);
1937 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_acc_dt);
1938
1939 /* the same handling processing as ACC except the completion handler function. */
1940 /* set is_async to 1 so we can get rreq back */
1941 MPIR_Request *rreq;
1942 mpi_errno = MPIDIG_acc_dt_target_msg_cb(handler_id, am_hdr, data,
1943 in_data_sz, is_local, 1, &rreq);
1944
1945 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_acc_dt_target_cmpl_cb;
1946
1947 if (is_async) {
1948 *req = rreq;
1949 } else {
1950 MPIDIG_recv_copy(data, rreq);
1951 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
1952 }
1953
1954 MPIDIG_rma_set_am_flag();
1955 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_acc_dt);
1956 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACC_DT_TARGET_MSG_CB);
1957 return mpi_errno;
1958 }
1959
MPIDIG_get_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)1960 int MPIDIG_get_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
1961 int is_local, int is_async, MPIR_Request ** req)
1962 {
1963 int mpi_errno = MPI_SUCCESS;
1964 MPIR_Request *rreq = NULL;
1965 MPIDIG_get_msg_t *msg_hdr = (MPIDIG_get_msg_t *) am_hdr;
1966 MPIR_Win *win;
1967
1968 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_TARGET_MSG_CB);
1969 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_TARGET_MSG_CB);
1970 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get);
1971
1972 rreq = MPIDIG_request_create(MPIR_REQUEST_KIND__RMA, 1);
1973 MPIR_ERR_CHKANDSTMT(rreq == NULL, mpi_errno, MPIX_ERR_NOREQ, goto fn_fail, "**nomemreq");
1974
1975 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_target_cmpl_cb;
1976 #ifndef MPIDI_CH4_DIRECT_NETMOD
1977 MPIDI_REQUEST(rreq, is_local) = is_local;
1978 #endif
1979
1980 win = (MPIR_Win *) MPIDIU_map_lookup(MPIDI_global.win_map, msg_hdr->win_id);
1981 MPIR_Assert(win);
1982
1983 uintptr_t base = MPIDIG_win_base_at_target(win);
1984 size_t offset = win->disp_unit * msg_hdr->target_disp;
1985
1986 MPIDIG_REQUEST(rreq, req->greq.win_ptr) = win;
1987 MPIDIG_REQUEST(rreq, req->greq.count) = msg_hdr->target_count;
1988 MPIDIG_REQUEST(rreq, req->greq.datatype) = msg_hdr->target_datatype;
1989 MPIDIG_REQUEST(rreq, req->greq.flattened_dt) = NULL;
1990 MPIDIG_REQUEST(rreq, req->greq.dt) = NULL;
1991 MPIDIG_REQUEST(rreq, req->greq.greq_ptr) = msg_hdr->greq_ptr;
1992 MPIDIG_REQUEST(rreq, rank) = msg_hdr->src_rank;
1993
1994 if (msg_hdr->flattened_sz) {
1995 void *flattened_dt = MPL_malloc(msg_hdr->flattened_sz, MPL_MEM_BUFFER);
1996 MPIDIG_recv_init(1, in_data_sz, flattened_dt, msg_hdr->flattened_sz, rreq);
1997 MPIDIG_REQUEST(rreq, req->greq.flattened_dt) = flattened_dt;
1998 MPIDIG_REQUEST(rreq, req->greq.addr) = (char *) base + offset;
1999 } else {
2000 MPIR_Assert(!in_data_sz || in_data_sz == 0);
2001 MPIDIG_recv_init(1, 0, NULL, 0, rreq);
2002 MPIDIG_REQUEST(rreq, req->greq.addr) = (char *) base + offset + msg_hdr->target_true_lb;
2003 }
2004
2005 if (is_async) {
2006 *req = rreq;
2007 } else {
2008 MPIDIG_recv_copy(data, rreq);
2009 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
2010 }
2011
2012 fn_exit:
2013 MPIDIG_rma_set_am_flag();
2014 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get);
2015 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_TARGET_MSG_CB);
2016 return mpi_errno;
2017 fn_fail:
2018 goto fn_exit;
2019 }
2020
MPIDIG_get_ack_target_msg_cb(int handler_id,void * am_hdr,void * data,MPI_Aint in_data_sz,int is_local,int is_async,MPIR_Request ** req)2021 int MPIDIG_get_ack_target_msg_cb(int handler_id, void *am_hdr, void *data, MPI_Aint in_data_sz,
2022 int is_local, int is_async, MPIR_Request ** req)
2023 {
2024 int mpi_errno = MPI_SUCCESS;
2025 MPIR_Request *rreq;
2026
2027 MPIDIG_get_ack_msg_t *msg_hdr = (MPIDIG_get_ack_msg_t *) am_hdr;
2028 MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_GET_ACK_TARGET_MSG_CB);
2029 MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_GET_ACK_TARGET_MSG_CB);
2030 MPIR_T_PVAR_TIMER_START(RMA, rma_targetcb_get_ack);
2031
2032 rreq = (MPIR_Request *) msg_hdr->greq_ptr;
2033 MPIR_Assert(rreq->kind == MPIR_REQUEST_KIND__RMA);
2034
2035 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) = get_ack_target_cmpl_cb;
2036 #ifndef MPIDI_CH4_DIRECT_NETMOD
2037 MPIDI_REQUEST(rreq, is_local) = is_local;
2038 #endif
2039
2040 MPIDIG_REQUEST(rreq, buffer) = MPIDIG_REQUEST(rreq, req->greq.addr);
2041 MPIDIG_REQUEST(rreq, count) = MPIDIG_REQUEST(rreq, req->greq.count);
2042 MPIDIG_REQUEST(rreq, datatype) = MPIDIG_REQUEST(rreq, req->greq.datatype);
2043 MPIDIG_recv_type_init(in_data_sz, rreq);
2044
2045 if (is_async) {
2046 *req = rreq;
2047 } else {
2048 MPIDIG_recv_copy(data, rreq);
2049 MPIDIG_REQUEST(rreq, req->target_cmpl_cb) (rreq);
2050 }
2051
2052 MPIDIG_rma_set_am_flag();
2053 MPIR_T_PVAR_TIMER_END(RMA, rma_targetcb_get_ack);
2054 MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_GET_ACK_TARGET_MSG_CB);
2055 return mpi_errno;
2056 }
2057