1 /*
2 * Copyright (C) by Argonne National Laboratory
3 * See COPYRIGHT in top-level directory
4 */
5
6 #ifndef MPIR_REQUEST_H_INCLUDED
7 #define MPIR_REQUEST_H_INCLUDED
8
9 #include "mpir_process.h"
10
11 /*
12 === BEGIN_MPI_T_CVAR_INFO_BLOCK ===
13
14 categories :
15 - name : REQUEST
16 description : A category for requests mangement variables
17
18 cvars:
19 - name : MPIR_CVAR_REQUEST_POLL_FREQ
20 category : REQUEST
21 type : int
22 default : 8
23 class : none
24 verbosity : MPI_T_VERBOSITY_USER_BASIC
25 scope : MPI_T_SCOPE_LOCAL
26 description : >-
27 How frequent to poll during completion calls (wait/test) in terms
28 of number of processed requests before polling.
29
30 - name : MPIR_CVAR_REQUEST_BATCH_SIZE
31 category : REQUEST
32 type : int
33 default : 64
34 class : none
35 verbosity : MPI_T_VERBOSITY_USER_BASIC
36 scope : MPI_T_SCOPE_LOCAL
37 description : >-
38 The number of requests to make completion as a batch
39 in MPI_Waitall and MPI_Testall implementation. A large number
40 is likely to cause more cache misses.
41
42 === END_MPI_T_CVAR_INFO_BLOCK ===
43 */
44
45 /* NOTE-R1: MPIR_REQUEST_KIND__MPROBE signifies that this is a request created by
46 * MPI_Mprobe or MPI_Improbe. Since we use MPI_Request objects as our
47 * MPI_Message objects, we use this separate kind in order to provide stronger
48 * error checking. Once a message (backed by a request) is promoted to a real
49 * request by calling MPI_Mrecv/MPI_Imrecv, we actually modify the kind to be
50 * MPIR_REQUEST_KIND__RECV in order to keep completion logic as simple as possible. */
51 /*E
52 MPIR_Request_kind - Kinds of MPI Requests
53
54 Module:
55 Request-DS
56
57 E*/
58 typedef enum MPIR_Request_kind_t {
59 MPIR_REQUEST_KIND__UNDEFINED,
60 MPIR_REQUEST_KIND__SEND,
61 MPIR_REQUEST_KIND__RECV,
62 MPIR_REQUEST_KIND__PREQUEST_SEND,
63 MPIR_REQUEST_KIND__PREQUEST_RECV,
64 MPIR_REQUEST_KIND__GREQUEST,
65 MPIR_REQUEST_KIND__COLL,
66 MPIR_REQUEST_KIND__MPROBE, /* see NOTE-R1 */
67 MPIR_REQUEST_KIND__RMA,
68 MPIR_REQUEST_KIND__LAST
69 #ifdef MPID_REQUEST_KIND_DECL
70 , MPID_REQUEST_KIND_DECL
71 #endif
72 } MPIR_Request_kind_t;
73
74 /* define built-in handles for pre-completed requests. These are internally used
75 * and are not exposed to the user.
76 */
77 #define MPIR_REQUEST_COMPLETE (MPI_Request)0x6c000000
78 #define MPIR_REQUEST_COMPLETE_SEND (MPI_Request)0x6c000001
79 #define MPIR_REQUEST_COMPLETE_RECV (MPI_Request)0x6c000002
80 #define MPIR_REQUEST_COMPLETE_COLL (MPI_Request)0x6c000006
81 #define MPIR_REQUEST_COMPLETE_RMA (MPI_Request)0x6c000008
82
83 #define MPIR_REQUEST_NULL_RECV (MPI_Request)0x6c000010
84
85 #define MPIR_REQUEST_BUILTIN_COUNT 0x11
86
87 /* This currently defines a single structure type for all requests.
88 Eventually, we may want a union type, as used in MPICH-1 */
89 /* Typedefs for Fortran generalized requests */
90 typedef void (MPIR_Grequest_f77_cancel_function) (void *, MPI_Fint *, MPI_Fint *);
91 typedef void (MPIR_Grequest_f77_free_function) (void *, MPI_Fint *);
92 typedef void (MPIR_Grequest_f77_query_function) (void *, MPI_Fint *, MPI_Fint *);
93
94 /* vtable-ish structure holding generalized request function pointers and other
95 * state. Saves ~48 bytes in pt2pt requests on many platforms. */
96 struct MPIR_Grequest_fns {
97 union {
98 struct {
99 MPI_Grequest_cancel_function *cancel_fn;
100 MPI_Grequest_free_function *free_fn;
101 MPI_Grequest_query_function *query_fn;
102 } C;
103 struct {
104 MPIR_Grequest_f77_cancel_function *cancel_fn;
105 MPIR_Grequest_f77_free_function *free_fn;
106 MPIR_Grequest_f77_query_function *query_fn;
107 } F;
108 } U;
109 MPIX_Grequest_poll_function *poll_fn;
110 MPIX_Grequest_wait_function *wait_fn;
111 void *grequest_extra_state;
112 MPIX_Grequest_class greq_class;
113 MPIR_Lang_t greq_lang; /* language that defined
114 * the generalize req */
115 };
116
117 typedef struct MPIR_Grequest_class {
118 MPIR_OBJECT_HEADER; /* adds handle and ref_count fields */
119 MPI_Grequest_query_function *query_fn;
120 MPI_Grequest_free_function *free_fn;
121 MPI_Grequest_cancel_function *cancel_fn;
122 MPIX_Grequest_poll_function *poll_fn;
123 MPIX_Grequest_wait_function *wait_fn;
124 struct MPIR_Grequest_class *next;
125 } MPIR_Grequest_class;
126
127 #define MPIR_Request_extract_status(request_ptr_, status_) \
128 { \
129 if ((status_) != MPI_STATUS_IGNORE) \
130 { \
131 int error__; \
132 \
133 /* According to the MPI 1.1 standard page 22 lines 9-12, \
134 * the MPI_ERROR field may not be modified except by the \
135 * functions in section 3.7.5 which return \
136 * MPI_ERR_IN_STATUSES (MPI_Wait{all,some} and \
137 * MPI_Test{all,some}). */ \
138 error__ = (status_)->MPI_ERROR; \
139 *(status_) = (request_ptr_)->status; \
140 (status_)->MPI_ERROR = error__; \
141 } \
142 }
143
144 #define MPIR_Request_is_complete(req_) (MPIR_cc_is_complete((req_)->cc_ptr))
145
146 /*S
147 MPIR_Request - Description of the Request data structure
148
149 Module:
150 Request-DS
151
152 Notes:
153 If it is necessary to remember the MPI datatype, this information is
154 saved within the device-specific fields provided by 'MPID_DEV_REQUEST_DECL'.
155
156 Requests come in many flavors, as stored in the 'kind' field. It is
157 expected that each kind of request will have its own structure type
158 (e.g., 'MPIR_Request_send_t') that extends the 'MPIR_Request'.
159
160 S*/
161 struct MPIR_Request {
162 MPIR_OBJECT_HEADER; /* adds handle and ref_count fields */
163
164 MPIR_Request_kind_t kind;
165
166 /* pointer to the completion counter. This is necessary for the
167 * case when an operation is described by a list of requests */
168 MPIR_cc_t *cc_ptr;
169 /* the actual completion counter. Ensure cc and status are in the
170 * same cache line, assuming the cache line size is a multiple of
171 * 32 bytes and 32-bit integers */
172 MPIR_cc_t cc;
173
174 /* completion notification counter: this must be decremented by
175 * the request completion routine, when the completion count hits
176 * zero. this counter allows us to keep track of the completion
177 * of multiple requests in a single place. */
178 MPIR_cc_t *completion_notification;
179
180 /* A comm is needed to find the proper error handler */
181 MPIR_Comm *comm;
182 /* Status is needed for wait/test/recv */
183 MPI_Status status;
184
185 union {
186 struct {
187 struct MPIR_Grequest_fns *greq_fns;
188 } ureq; /* kind : MPIR_REQUEST_KIND__GREQUEST */
189 struct {
190 MPIR_Errflag_t errflag;
191 MPII_Coll_req_t coll;
192 } nbc; /* kind : MPIR_REQUEST_KIND__COLL */
193 #if defined HAVE_DEBUGGER_SUPPORT
194 struct {
195 struct MPIR_Sendq *dbg_next;
196 } send; /* kind : MPID_REQUEST_SEND */
197 #endif /* HAVE_DEBUGGER_SUPPORT */
198 struct {
199 #if defined HAVE_DEBUGGER_SUPPORT
200 struct MPIR_Sendq *dbg_next;
201 #endif /* HAVE_DEBUGGER_SUPPORT */
202 /* Persistent requests have their own "real" requests */
203 struct MPIR_Request *real_request;
204 } persist; /* kind : MPID_PREQUEST_SEND or MPID_PREQUEST_RECV */
205 } u;
206
207 /* Other, device-specific information */
208 #ifdef MPID_DEV_REQUEST_DECL
209 MPID_DEV_REQUEST_DECL
210 #endif
211 };
212
213 /* Multiple Request Pools
214 * Request objects creation and freeing is in a hot path. Multiple pools allow different
215 * threads to access different pools without incurring global locks. Because only request
216 * objects benefit from this multi-pool scheme, the normal handlemem macros and functions
217 * are extended here for request objects. It is separate from the other objects, and the
218 * bit patterns for POOL and BLOCK sizes can be adjusted if necessary.
219 *
220 * MPIR_Request_create_from_pool is used to create request objects from a specific pool.
221 * MPIR_Request_create is a wrapper to create request from pool 0.
222 */
223 /* Handle Bits - 2+4+6+8+12 - Type, Kind, Pool_idx, Block_idx, Object_idx */
224 #define REQUEST_POOL_MASK 0x03f00000
225 #define REQUEST_POOL_SHIFT 20
226 #define REQUEST_POOL_MAX 64
227 #define REQUEST_BLOCK_MASK 0x000ff000
228 #define REQUEST_BLOCK_SHIFT 12
229 #define REQUEST_BLOCK_MAX 256
230 #define REQUEST_OBJECT_MASK 0x00000fff
231 #define REQUEST_OBJECT_SHIFT 0
232 #define REQUEST_OBJECT_MAX 4096
233
234 #define REQUEST_NUM_BLOCKS 256
235 #define REQUEST_NUM_INDICES 1024
236
237 #define MPIR_REQUEST_NUM_POOLS REQUEST_POOL_MAX
238 #define MPIR_REQUEST_PREALLOC 8
239
240 #define MPIR_REQUEST_POOL(req_) (((req_)->handle & REQUEST_POOL_MASK) >> REQUEST_POOL_SHIFT);
241
242 extern MPIR_Request MPIR_Request_builtins[MPIR_REQUEST_BUILTIN_COUNT];
243 extern MPIR_Object_alloc_t MPIR_Request_mem[MPIR_REQUEST_NUM_POOLS];
244 extern MPIR_Request MPIR_Request_direct[MPIR_REQUEST_PREALLOC];
245
246 #define MPIR_Request_get_ptr(a, ptr) \
247 do { \
248 int pool, blk, idx; \
249 pool = ((a) & REQUEST_POOL_MASK) >> REQUEST_POOL_SHIFT; \
250 switch (HANDLE_GET_KIND(a)) { \
251 case HANDLE_KIND_BUILTIN: \
252 if (a == MPI_MESSAGE_NO_PROC) { \
253 ptr = NULL; \
254 } else { \
255 MPIR_Assert(HANDLE_INDEX(a) < MPIR_REQUEST_BUILTIN_COUNT); \
256 ptr = MPIR_Request_builtins + HANDLE_INDEX(a); \
257 } \
258 break; \
259 case HANDLE_KIND_DIRECT: \
260 MPIR_Assert(pool == 0); \
261 ptr = MPIR_Request_direct + HANDLE_INDEX(a); \
262 break; \
263 case HANDLE_KIND_INDIRECT: \
264 blk = ((a) & REQUEST_BLOCK_MASK) >> REQUEST_BLOCK_SHIFT; \
265 idx = ((a) & REQUEST_OBJECT_MASK) >> REQUEST_OBJECT_SHIFT; \
266 ptr = ((MPIR_Request *) MPIR_Request_mem[pool].indirect[blk]) + idx; \
267 break; \
268 default: \
269 ptr = NULL; \
270 break; \
271 } \
272 } while (0)
273
274 void MPII_init_request(void);
275
276 /* To get the benefit of multiple request pool, device layer need register their per-vci lock
277 * with each pool that they are going to use, typically a 1-1 vci-pool mapping.
278 * NOTE: currently, only per-vci thread granularity utilizes multiple request pool.
279 */
MPIR_Request_register_pool_lock(int pool,MPID_Thread_mutex_t * lock)280 static inline void MPIR_Request_register_pool_lock(int pool, MPID_Thread_mutex_t * lock)
281 {
282 MPIR_Request_mem[pool].lock = lock;
283 }
284
MPIR_Request_is_persistent(MPIR_Request * req_ptr)285 static inline int MPIR_Request_is_persistent(MPIR_Request * req_ptr)
286 {
287 return (req_ptr->kind == MPIR_REQUEST_KIND__PREQUEST_SEND ||
288 req_ptr->kind == MPIR_REQUEST_KIND__PREQUEST_RECV);
289 }
290
291 /* Return whether a request is active.
292 * A persistent request and the handle to it are "inactive"
293 * if the request is not associated with any ongoing communication.
294 * A handle is "active" if it is neither null nor "inactive". */
MPIR_Request_is_active(MPIR_Request * req_ptr)295 static inline int MPIR_Request_is_active(MPIR_Request * req_ptr)
296 {
297 if (req_ptr == NULL)
298 return 0;
299 else
300 return (!MPIR_Request_is_persistent(req_ptr) || (req_ptr)->u.persist.real_request != NULL);
301 }
302
303 #define MPIR_REQUESTS_PROPERTY__NO_NULL (1 << 1)
304 #define MPIR_REQUESTS_PROPERTY__NO_GREQUESTS (1 << 2)
305 #define MPIR_REQUESTS_PROPERTY__SEND_RECV_ONLY (1 << 3)
306 #define MPIR_REQUESTS_PROPERTY__OPT_ALL (MPIR_REQUESTS_PROPERTY__NO_NULL \
307 | MPIR_REQUESTS_PROPERTY__NO_GREQUESTS \
308 | MPIR_REQUESTS_PROPERTY__SEND_RECV_ONLY)
309
310 /* NOTE: Pool-specific request creation is unsafe unless under global thread granularity.
311 */
MPIR_Request_create_from_pool(MPIR_Request_kind_t kind,int pool)312 static inline MPIR_Request *MPIR_Request_create_from_pool(MPIR_Request_kind_t kind, int pool)
313 {
314 MPIR_Request *req;
315
316 #ifdef MPICH_DEBUG_MUTEX
317 MPID_THREAD_ASSERT_IN_CS(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
318 #endif
319 req = MPIR_Handle_obj_alloc_unsafe(&MPIR_Request_mem[pool],
320 REQUEST_NUM_BLOCKS, REQUEST_NUM_INDICES);
321 if (req != NULL) {
322 /* Patch the handle for pool index. */
323 req->handle |= (pool << REQUEST_POOL_SHIFT);
324 }
325
326 if (req != NULL) {
327 MPL_DBG_MSG_P(MPIR_DBG_REQUEST, VERBOSE, "allocated request, handle=0x%08x", req->handle);
328 #ifdef MPICH_DBG_OUTPUT
329 /*MPIR_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPIR_REQUEST); */
330 if (HANDLE_GET_MPI_KIND(req->handle) != MPIR_REQUEST) {
331 int mpi_errno;
332 mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
333 __func__, __LINE__, MPI_ERR_OTHER,
334 "**invalid_handle", "**invalid_handle %d",
335 req->handle);
336 MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
337 }
338 #endif
339 /* FIXME: This makes request creation expensive. We need to
340 * trim this to the basics, with additional setup for
341 * special-purpose requests (think base class and
342 * inheritance). For example, do we really* want to set the
343 * kind to UNDEFINED? And should the RMA values be set only
344 * for RMA requests? */
345 MPIR_Object_set_ref(req, 1);
346 req->kind = kind;
347 MPIR_cc_set(&req->cc, 1);
348 req->cc_ptr = &req->cc;
349
350 req->completion_notification = NULL;
351
352 req->status.MPI_ERROR = MPI_SUCCESS;
353 MPIR_STATUS_SET_CANCEL_BIT(req->status, FALSE);
354
355 req->comm = NULL;
356
357 switch (kind) {
358 case MPIR_REQUEST_KIND__SEND:
359 MPII_REQUEST_CLEAR_DBG(req);
360 break;
361 case MPIR_REQUEST_KIND__COLL:
362 req->u.nbc.errflag = MPIR_ERR_NONE;
363 req->u.nbc.coll.host_sendbuf = NULL;
364 req->u.nbc.coll.host_recvbuf = NULL;
365 req->u.nbc.coll.datatype = MPI_DATATYPE_NULL;
366 break;
367 default:
368 break;
369 }
370
371 MPID_Request_create_hook(req);
372 } else {
373 /* FIXME: This fails to fail if debugging is turned off */
374 MPL_DBG_MSG(MPIR_DBG_REQUEST, TYPICAL, "unable to allocate a request");
375 }
376
377 return req;
378 }
379
380 /* NOTE: safe under per-vci, per-obj, or global thread granularity */
MPIR_Request_create(MPIR_Request_kind_t kind)381 static inline MPIR_Request *MPIR_Request_create(MPIR_Request_kind_t kind)
382 {
383 MPIR_Request *req;
384 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
385 MPID_THREAD_CS_ENTER(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[0].lock));
386 req = MPIR_Request_create_from_pool(kind, 0);
387 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
388 MPID_THREAD_CS_EXIT(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[0].lock));
389 return req;
390 }
391
392 #define MPIR_Request_add_ref(req_p_) \
393 do { MPIR_Object_add_ref(req_p_); } while (0)
394
395 #define MPIR_Request_release_ref(req_p_, inuse_) \
396 do { MPIR_Object_release_ref(req_p_, inuse_); } while (0)
397
get_builtin_req(int idx,MPIR_Request_kind_t kind)398 MPL_STATIC_INLINE_PREFIX MPIR_Request *get_builtin_req(int idx, MPIR_Request_kind_t kind)
399 {
400 return MPIR_Request_builtins + (idx);
401 }
402
MPIR_Request_create_complete(MPIR_Request_kind_t kind)403 MPL_STATIC_INLINE_PREFIX MPIR_Request *MPIR_Request_create_complete(MPIR_Request_kind_t kind)
404 {
405 /* pre-completed request uses kind as idx */
406 return get_builtin_req(kind, kind);
407 }
408
MPIR_Request_create_null_recv(void)409 MPL_STATIC_INLINE_PREFIX MPIR_Request *MPIR_Request_create_null_recv(void)
410 {
411 return get_builtin_req(HANDLE_INDEX(MPIR_REQUEST_NULL_RECV), MPIR_REQUEST_KIND__RECV);
412 }
413
MPIR_Request_free_with_safety(MPIR_Request * req,int need_safety)414 static inline void MPIR_Request_free_with_safety(MPIR_Request * req, int need_safety)
415 {
416 int inuse;
417 int pool = MPIR_REQUEST_POOL(req);
418
419 if (HANDLE_IS_BUILTIN(req->handle)) {
420 /* do not free builtin request objects */
421 return;
422 }
423
424 MPIR_Request_release_ref(req, &inuse);
425
426 if (need_safety) {
427 MPID_THREAD_CS_ENTER(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
428 }
429 #ifdef MPICH_DEBUG_MUTEX
430 MPID_THREAD_ASSERT_IN_CS(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
431 #endif
432 /* inform the device that we are decrementing the ref-count on
433 * this request */
434 MPID_Request_free_hook(req);
435
436 if (inuse == 0) {
437 MPL_DBG_MSG_P(MPIR_DBG_REQUEST, VERBOSE, "freeing request, handle=0x%08x", req->handle);
438
439 #ifdef MPICH_DBG_OUTPUT
440 if (HANDLE_GET_MPI_KIND(req->handle) != MPIR_REQUEST) {
441 int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
442 __func__, __LINE__, MPI_ERR_OTHER,
443 "**invalid_handle", "**invalid_handle %d",
444 req->handle);
445 MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
446 }
447
448 if (req->ref_count != 0) {
449 int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
450 __func__, __LINE__, MPI_ERR_OTHER,
451 "**invalid_refcount", "**invalid_refcount %d",
452 req->ref_count);
453 MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
454 }
455 #endif
456
457 /* FIXME: We need a better way to handle these so that we do
458 * not always need to initialize these fields and check them
459 * when we destroy a request */
460 /* FIXME: We need a way to call these routines ONLY when the
461 * related ref count has become zero. */
462 if (req->comm != NULL) {
463 MPIR_Comm_release(req->comm);
464 }
465
466 if (req->kind == MPIR_REQUEST_KIND__GREQUEST) {
467 MPL_free(req->u.ureq.greq_fns);
468 }
469
470 MPID_Request_destroy_hook(req);
471
472 if (need_safety) {
473 MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
474 MPIR_Handle_obj_free_unsafe(&MPIR_Request_mem[pool], req);
475 MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_HANDLE_MUTEX);
476 } else {
477 MPIR_Handle_obj_free_unsafe(&MPIR_Request_mem[pool], req);
478 }
479 }
480 if (need_safety) {
481 MPID_THREAD_CS_EXIT(VCI, (*(MPID_Thread_mutex_t *) MPIR_Request_mem[pool].lock));
482 }
483 }
484
MPIR_Request_free_safe(MPIR_Request * req)485 MPL_STATIC_INLINE_PREFIX void MPIR_Request_free_safe(MPIR_Request * req)
486 {
487 MPIR_Request_free_with_safety(req, 1);
488 }
489
MPIR_Request_free_unsafe(MPIR_Request * req)490 MPL_STATIC_INLINE_PREFIX void MPIR_Request_free_unsafe(MPIR_Request * req)
491 {
492 MPIR_Request_free_with_safety(req, 0);
493 }
494
MPIR_Request_free(MPIR_Request * req)495 MPL_STATIC_INLINE_PREFIX void MPIR_Request_free(MPIR_Request * req)
496 {
497 /* The default is to assume we need safety unless it's global thread granularity */
498 MPIR_Request_free_with_safety(req, 1);
499 }
500
501 /* Requests that are not created inside device (general requests, nonblocking collective
502 * requests such as sched, gentran, hcoll) should call MPIR_Request_complete.
503 * MPID_Request_complete are called inside device critical section, therefore, potentially
504 * are unsafe to call outside the device. (NOTE: this will come into effect with ch4 multi-vci.)
505 */
MPIR_Request_complete(MPIR_Request * req)506 MPL_STATIC_INLINE_PREFIX void MPIR_Request_complete(MPIR_Request * req)
507 {
508 MPIR_cc_set(&req->cc, 0);
509 MPIR_Request_free(req);
510 }
511
512 /* The "fastpath" version of MPIR_Request_completion_processing. It only handles
513 * MPIR_REQUEST_KIND__SEND and MPIR_REQUEST_KIND__RECV kinds, and it does not attempt to
514 * deal with status structures under the assumption that bleeding fast code will
515 * pass either MPI_STATUS_IGNORE or MPI_STATUSES_IGNORE as appropriate. This
516 * routine (or some a variation of it) is an unfortunately necessary stunt to
517 * get high message rates on key benchmarks for high-end systems.
518 */
MPIR_Request_completion_processing_fastpath(MPI_Request * request,MPIR_Request * request_ptr)519 MPL_STATIC_INLINE_PREFIX int MPIR_Request_completion_processing_fastpath(MPI_Request * request,
520 MPIR_Request * request_ptr)
521 {
522 int mpi_errno = MPI_SUCCESS;
523
524 MPIR_Assert(request_ptr->kind == MPIR_REQUEST_KIND__SEND ||
525 request_ptr->kind == MPIR_REQUEST_KIND__RECV);
526
527 if (request_ptr->kind == MPIR_REQUEST_KIND__SEND) {
528 /* FIXME: are Ibsend requests added to the send queue? */
529 MPII_SENDQ_FORGET(request_ptr);
530 }
531
532 /* the completion path for SEND and RECV is the same at this time, modulo
533 * the SENDQ hook above */
534 mpi_errno = request_ptr->status.MPI_ERROR;
535 MPIR_Request_free(request_ptr);
536 *request = MPI_REQUEST_NULL;
537
538 return mpi_errno;
539 }
540
541 int MPIR_Request_completion_processing(MPIR_Request *, MPI_Status *);
542 int MPIR_Request_get_error(MPIR_Request *);
543
544 MPL_STATIC_INLINE_PREFIX int MPID_Request_is_anysource(MPIR_Request *);
545 MPL_STATIC_INLINE_PREFIX int MPID_Comm_AS_enabled(MPIR_Comm *);
546 extern int MPIR_CVAR_ENABLE_FT;
547
548 /* The following routines are ULFM helpers. */
549
550 /* This routine check if the request is "anysource" but the communicator is not,
551 * which happens usually due to a failure of a process in the communicator. */
MPIR_Request_is_anysrc_mismatched(MPIR_Request * req_ptr)552 MPL_STATIC_INLINE_PREFIX int MPIR_Request_is_anysrc_mismatched(MPIR_Request * req_ptr)
553 {
554 return (MPIR_CVAR_ENABLE_FT &&
555 !MPIR_Request_is_complete(req_ptr) &&
556 MPID_Request_is_anysource(req_ptr) && !MPID_Comm_AS_enabled((req_ptr)->comm));
557 }
558
559 /* This routine handle the request when its associated process failed. */
560 int MPIR_Request_handle_proc_failed(MPIR_Request * request_ptr);
561
562 /* The following routines perform the callouts to the user routines registered
563 as part of a generalized request. They handle any language binding issues
564 that are necessary. They are used when completing, freeing, cancelling or
565 extracting the status from a generalized request. */
566 int MPIR_Grequest_cancel(MPIR_Request * request_ptr, int complete);
567 int MPIR_Grequest_query(MPIR_Request * request_ptr);
568 int MPIR_Grequest_free(MPIR_Request * request_ptr);
569
570 void MPIR_Grequest_complete(MPIR_Request * request_ptr);
571 int MPIR_Grequest_start(MPI_Grequest_query_function * query_fn,
572 MPI_Grequest_free_function * free_fn,
573 MPI_Grequest_cancel_function * cancel_fn,
574 void *extra_state, MPIR_Request ** request_ptr);
575 int MPIX_Grequest_start_impl(MPI_Grequest_query_function *,
576 MPI_Grequest_free_function *,
577 MPI_Grequest_cancel_function *,
578 MPIX_Grequest_poll_function *,
579 MPIX_Grequest_wait_function *, void *, MPIR_Request **);
580
581 /* These routines below are helpers for the Extended generalized requests. */
582
MPIR_Request_has_poll_fn(MPIR_Request * request_ptr)583 MPL_STATIC_INLINE_PREFIX int MPIR_Request_has_poll_fn(MPIR_Request * request_ptr)
584 {
585 return (request_ptr->kind == MPIR_REQUEST_KIND__GREQUEST &&
586 request_ptr->u.ureq.greq_fns != NULL && request_ptr->u.ureq.greq_fns->poll_fn != NULL);
587 }
588
MPIR_Request_has_wait_fn(MPIR_Request * request_ptr)589 MPL_STATIC_INLINE_PREFIX int MPIR_Request_has_wait_fn(MPIR_Request * request_ptr)
590 {
591 return (request_ptr->kind == MPIR_REQUEST_KIND__GREQUEST &&
592 request_ptr->u.ureq.greq_fns != NULL && request_ptr->u.ureq.greq_fns->wait_fn != NULL);
593 }
594
MPIR_Grequest_wait(MPIR_Request * request_ptr,MPI_Status * status)595 MPL_STATIC_INLINE_PREFIX int MPIR_Grequest_wait(MPIR_Request * request_ptr, MPI_Status * status)
596 {
597 int mpi_errno;
598 MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
599 mpi_errno = (request_ptr->u.ureq.greq_fns->wait_fn) (1,
600 &request_ptr->u.ureq.greq_fns->
601 grequest_extra_state, 0, status);
602 MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
603 return mpi_errno;
604 }
605
MPIR_Grequest_poll(MPIR_Request * request_ptr,MPI_Status * status)606 MPL_STATIC_INLINE_PREFIX int MPIR_Grequest_poll(MPIR_Request * request_ptr, MPI_Status * status)
607 {
608 int mpi_errno;
609 MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
610 mpi_errno =
611 (request_ptr->u.ureq.greq_fns->poll_fn) (request_ptr->u.ureq.greq_fns->grequest_extra_state,
612 status);
613 MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
614 return mpi_errno;
615 }
616
617 int MPIR_Test_state(MPIR_Request * request, int *flag, MPI_Status * status,
618 MPID_Progress_state * state);
619 int MPIR_Testall_state(int count, MPIR_Request * request_ptrs[], int *flag,
620 MPI_Status array_of_statuses[], int requests_property,
621 MPID_Progress_state * state);
622 int MPIR_Testany_state(int count, MPIR_Request * request_ptrs[], int *indx, int *flag,
623 MPI_Status * status, MPID_Progress_state * state);
624 int MPIR_Testsome_state(int incount, MPIR_Request * request_ptrs[], int *outcount,
625 int array_of_indices[], MPI_Status array_of_statuses[],
626 MPID_Progress_state * state);
627 int MPIR_Test_impl(MPIR_Request * request, int *flag, MPI_Status * status);
628 int MPIR_Testall_impl(int count, MPIR_Request * request_ptrs[], int *flag,
629 MPI_Status array_of_statuses[], int requests_property);
630 int MPIR_Testany_impl(int count, MPIR_Request * request_ptrs[],
631 int *indx, int *flag, MPI_Status * status);
632 int MPIR_Testsome_impl(int incount, MPIR_Request * request_ptrs[],
633 int *outcount, int array_of_indices[], MPI_Status array_of_statuses[]);
634
635 int MPIR_Wait_state(MPIR_Request * request_ptr, MPI_Status * status, MPID_Progress_state * state);
636 int MPIR_Waitall_state(int count, MPIR_Request * request_ptrs[], MPI_Status array_of_statuses[],
637 int request_properties, MPID_Progress_state * state);
638 int MPIR_Waitany_state(int count, MPIR_Request * request_ptrs[], int *indx, MPI_Status * status,
639 MPID_Progress_state * state);
640 int MPIR_Waitsome_state(int incount, MPIR_Request * request_ptrs[],
641 int *outcount, int array_of_indices[], MPI_Status array_of_statuses[],
642 MPID_Progress_state * state);
643 int MPIR_Wait_impl(MPIR_Request * request_ptr, MPI_Status * status);
644 int MPIR_Waitall_impl(int count, MPIR_Request * request_ptrs[], MPI_Status array_of_statuses[],
645 int request_properties);
646 int MPIR_Waitany_impl(int count, MPIR_Request * request_ptrs[], int *indx, MPI_Status * status);
647 int MPIR_Waitsome_impl(int incount, MPIR_Request * request_ptrs[],
648 int *outcount, int array_of_indices[], MPI_Status array_of_statuses[]);
649
650 int MPIR_Test(MPI_Request * request, int *flag, MPI_Status * status);
651 int MPIR_Testall(int count, MPI_Request array_of_requests[], int *flag,
652 MPI_Status array_of_statuses[]);
653 int MPIR_Wait(MPI_Request * request, MPI_Status * status);
654 int MPIR_Waitall(int count, MPI_Request array_of_requests[], MPI_Status array_of_statuses[]);
655
656 #endif /* MPIR_REQUEST_H_INCLUDED */
657