1 /* begin_generated_IBM_copyright_prolog */
2 /* */
3 /* This is an automatically generated copyright prolog. */
4 /* After initializing, DO NOT MODIFY OR MOVE */
5 /* --------------------------------------------------------------- */
6 /* Licensed Materials - Property of IBM */
7 /* Blue Gene/Q 5765-PER 5765-PRP */
8 /* */
9 /* (C) Copyright IBM Corp. 2011, 2012 All Rights Reserved */
10 /* US Government Users Restricted Rights - */
11 /* Use, duplication, or disclosure restricted */
12 /* by GSA ADP Schedule Contract with IBM Corp. */
13 /* */
14 /* --------------------------------------------------------------- */
15 /* */
16 /* end_generated_IBM_copyright_prolog */
17 /* (C)Copyright IBM Corp. 2007, 2011 */
18 /**
19 * \file src/mpid_recvq.h
20 * \brief Accessors and actors for MPID Requests
21 */
22 #ifndef __src_pt2pt_mpidi_recvq_h__
23 #define __src_pt2pt_mpidi_recvq_h__
24
25
26 struct MPIDI_Recvq_t
27 {
28 struct MPID_Request * posted_head; /**< \brief The Head of the Posted queue */
29 struct MPID_Request * posted_tail; /**< \brief The Tail of the Posted queue */
30 struct MPID_Request * unexpected_head; /**< \brief The Head of the Unexpected queue */
31 struct MPID_Request * unexpected_tail; /**< \brief The Tail of the Unexpected queue */
32 };
33 extern struct MPIDI_Recvq_t MPIDI_Recvq;
34
35
36 #ifndef OUT_OF_ORDER_HANDLING
37 #define MPIDI_Recvq_append(__Q, __req) \
38 ({ \
39 /* ---------------------------------------------- */ \
40 /* The tail request should point to the new one */ \
41 /* ---------------------------------------------- */ \
42 if (__Q ## _tail != NULL) \
43 __Q ## _tail->mpid.next = __req; \
44 else \
45 __Q ## _head = __req; \
46 /* ------------------------------------------ */ \
47 /* The tail should point to the new request */ \
48 /* ------------------------------------------ */ \
49 __Q ## _tail = __req; \
50 })
51 #else
52 #define MPIDI_Recvq_append(__Q, __req) \
53 ({ \
54 /* ---------------------------------------------- */ \
55 /* The tail request should point to the new one */ \
56 /* ---------------------------------------------- */ \
57 if (__Q ## _tail != NULL) { \
58 __Q ## _tail->mpid.next = __req; \
59 __req->mpid.prev = __Q ## _tail; \
60 } \
61 else { \
62 __Q ## _head = __req; \
63 __req->mpid.prev = NULL; \
64 } \
65 /* ------------------------------------------ */ \
66 /* The tail should point to the new request */ \
67 /* ------------------------------------------ */ \
68 __Q ## _tail = __req; \
69 })
70 #endif
71
72
73 #ifndef OUT_OF_ORDER_HANDLING
74 #define MPIDI_Recvq_remove(__Q, __req, __prev) \
75 ({ \
76 /* --------------------------------------------- */ \
77 /* Patch the next pointers to skip the request */ \
78 /* --------------------------------------------- */ \
79 if (__prev != NULL) \
80 __prev->mpid.next = __req->mpid.next; \
81 else \
82 __Q ## _head = __req->mpid.next; \
83 /* ------------------------------------------- */ \
84 /* Set tail pointer if removing the last one */ \
85 /* ------------------------------------------- */ \
86 if (__req->mpid.next == NULL) \
87 __Q ## _tail = __prev; \
88 })
89 #else
90 #define MPIDI_Recvq_remove(__Q, __req, __prev) \
91 ({ \
92 /* --------------------------------------------- */ \
93 /* Patch the next pointers to skip the request */ \
94 /* --------------------------------------------- */ \
95 if (__prev != NULL) { \
96 __prev->mpid.next = __req->mpid.next; \
97 } \
98 else \
99 __Q ## _head = __req->mpid.next; \
100 /* ------------------------------------------- */ \
101 /* Set tail pointer if removing the last one */ \
102 /* ------------------------------------------- */ \
103 if (__req->mpid.next == NULL) \
104 __Q ## _tail = __prev; \
105 else \
106 (__req->mpid.next)->mpid.prev = __prev; \
107 })
108 #endif
109
110
111 /**
112 * \brief A thread-safe version of MPIDI_Recvq_FU
113 * \param[in] source Find by Sender
114 * \param[in] tag Find by Tag
115 * \param[in] context_id Find by Context ID (communicator)
116 * \return 1/0 if the matching UE request was found or not
117 */
118 static inline int
MPIDI_Recvq_FU_r(int source,int tag,int context,MPI_Status * status)119 MPIDI_Recvq_FU_r(int source, int tag, int context, MPI_Status * status)
120 {
121 int rc = FALSE;
122 if (likely(MPIDI_Recvq.unexpected_head != NULL))
123 {
124 MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
125 rc = MPIDI_Recvq_FU(source, tag, context, status);
126 MPIU_THREAD_CS_EXIT(MSGQUEUE, 0);
127 }
128 return rc;
129 }
130
131
132 /**
133 * \brief Find a request in the unexpected queue and dequeue it, or allocate a new request and enqueue it in the posted queue
134 * \param[in] source Find by Sender
135 * \param[in] tag Find by Tag
136 * \param[in] context_id Find by Context ID (communicator)
137 * \param[out] foundp TRUE iff the request was found
138 * \return The matching UE request or the new posted request
139 */
140 #ifndef OUT_OF_ORDER_HANDLING
141 static inline MPID_Request *
MPIDI_Recvq_FDU_or_AEP(MPID_Request * newreq,int source,int tag,int context_id,int * foundp)142 MPIDI_Recvq_FDU_or_AEP(MPID_Request *newreq, int source, int tag, int context_id, int * foundp)
143 #else
144 static inline MPID_Request *
145 MPIDI_Recvq_FDU_or_AEP(MPID_Request *newreq, int source, pami_task_t pami_source, int tag, int context_id, int * foundp)
146 #endif
147 {
148 MPID_Request * rreq = NULL;
149 /* We have unexpected messages, so search unexpected queue */
150 if (unlikely(MPIDI_Recvq.unexpected_head != NULL)) {
151 #ifndef OUT_OF_ORDER_HANDLING
152 rreq = MPIDI_Recvq_FDU(source, tag, context_id, foundp);
153 #else
154 rreq = MPIDI_Recvq_FDU(source, pami_source, tag, context_id, foundp);
155 #endif
156 if (*foundp == TRUE)
157 return rreq;
158 #if (MPIDI_STATISTICS)
159 else {
160 MPID_NSTAT(mpid_statp->lateArrivals);
161 }
162 #endif
163 }
164
165 /* A matching request was not found in the unexpected queue,
166 so we need to allocate a new request and add it to the
167 posted queue */
168 rreq = newreq;
169 #ifdef MPIDI_TRACE
170 rreq->mpid.envelope.msginfo.MPIseqno=-1;
171 rreq->mpid.envelope.length=-1;
172 rreq->mpid.envelope.data=NULL;
173 #endif
174 rreq->kind = MPID_REQUEST_RECV;
175 MPIDI_Request_setMatch(rreq, tag, source, context_id);
176 MPIDI_Recvq_append(MPIDI_Recvq.posted, rreq);
177 *foundp = FALSE;
178
179 return rreq;
180 }
181
182
183 #ifdef OUT_OF_ORDER_HANDLING
184
185 /**
186 * data structures that tracks pair-wise in-coming communication.
187 */
188 typedef struct MPIDI_In_cntr {
189 uint n_OutOfOrderMsgs:16; /* the number of out-of-order messages received */
190 uint nMsgs; /* the number of received messages */
191 MPID_Request *OutOfOrderList; /* link list of out-of-order messages */
192 #ifdef MPIDI_TRACE
193 recv_status *R;
194 int totPR;
195 int nRdmaMsgs;
196 posted_recv *PR;
197 #endif
198 } MPIDI_In_cntr_t;
199
200 /**
201 * data structures that tracks pair-wise Out-going communication.
202 */
203 typedef struct MPIDI_Out_cntr {
204 uint unmatched:16; /* the number of un-matched messages */
205 uint nMsgs; /* the number of out-going messages */
206 #ifdef MPIDI_TRACE
207 send_status *S;
208 #endif
209 } MPIDI_Out_cntr_t;
210
211 /* global data to keep track of pair-wise communication, storage malloced
212 during initialization time */
213 MPIDI_In_cntr_t *MPIDI_In_cntr;
214 MPIDI_Out_cntr_t *MPIDI_Out_cntr;
215
216 #endif
217
218
219 /**
220 * \brief Find a request in the posted queue and dequeue it
221 * \param[in] source Find by Sender
222 * \param[in] tag Find by Tag
223 * \param[in] context_id Find by Context ID (communicator)
224 * \return The matching posted request or the new UE request
225 */
226 #ifndef OUT_OF_ORDER_HANDLING
227 static inline MPID_Request *
MPIDI_Recvq_FDP(size_t source,size_t tag,size_t context_id)228 MPIDI_Recvq_FDP(size_t source, size_t tag, size_t context_id)
229 #else
230 static inline MPID_Request *
231 MPIDI_Recvq_FDP(size_t source, pami_task_t pami_source, int tag, int context_id, int msg_seqno)
232 #endif
233 {
234 MPID_Request * rreq;
235 MPID_Request * prev_rreq = NULL;
236 #ifdef USE_STATISTICS
237 unsigned search_length = 0;
238 #endif
239 #ifdef MPIDI_TRACE
240 int idx;
241 idx=(msg_seqno & SEQMASK);
242 recv_status *rstatus;
243 memset(&MPIDI_In_cntr[pami_source].R[idx],0,sizeof(recv_status));
244 rstatus=&MPIDI_In_cntr[pami_source].R[idx];
245 #endif
246
247 rreq = MPIDI_Recvq.posted_head;
248
249 #ifdef OUT_OF_ORDER_HANDLING
250 MPIDI_In_cntr_t *in_cntr = &MPIDI_In_cntr[pami_source];
251 int nMsgs=(in_cntr->nMsgs+1);
252
253 if(msg_seqno == nMsgs) {
254 in_cntr->nMsgs = msg_seqno;
255 }
256
257 if( ((int)(in_cntr->nMsgs - msg_seqno)) >= 0) {
258 #endif
259 while (rreq != NULL) {
260 #ifdef USE_STATISTICS
261 ++search_length;
262 #endif
263
264 int match_src = MPIDI_Request_getMatchRank(rreq);
265 int match_tag = MPIDI_Request_getMatchTag(rreq);
266 int match_ctx = MPIDI_Request_getMatchCtxt(rreq);
267
268 int flag0 = (source == match_src);
269 flag0 |= (match_src == MPI_ANY_SOURCE);
270 int flag1 = (context_id == match_ctx);
271 int flag2 = (tag == match_tag);
272 flag2 |= (match_tag == MPI_ANY_TAG);
273 int flag = flag0 & flag1 & flag2;
274
275 #if 0
276 if ((MPIDI_Request_getMatchRank(rreq)==source || MPIDI_Request_getMatchRank(rreq)==MPI_ANY_SOURCE) &&
277 (MPIDI_Request_getMatchCtxt(rreq)==context_id) &&
278 (MPIDI_Request_getMatchTag(rreq) == tag || MPIDI_Request_getMatchTag(rreq) == MPI_ANY_TAG)
279 )
280 #else
281 if (flag)
282 #endif
283 {
284 #ifdef MPIDI_TRACE
285 rstatus->req=rreq;
286 rstatus->msgid=msg_seqno;
287 rstatus->posted=1;
288 rstatus->bufadd=rreq->mpid.userbuf;
289 rstatus->len=rreq->mpid.envelope.length;
290 rstatus->rtag=tag;
291 rstatus->rctx=context_id;
292 rstatus->matchedInHH=1;
293 rstatus->rsource=pami_source;
294 rreq->mpid.idx=idx;
295 rreq->mpid.partner_id=pami_source;
296 #endif
297 MPIDI_Recvq_remove(MPIDI_Recvq.posted, rreq, prev_rreq);
298 #ifdef USE_STATISTICS
299 MPIDI_Statistics_time(MPIDI_Statistics.recvq.unexpected_search, search_length);
300 #endif
301 return rreq;
302 }
303
304 prev_rreq = rreq;
305 rreq = rreq->mpid.next;
306 }
307 #ifdef OUT_OF_ORDER_HANDLING
308 }
309 #endif
310
311 #ifdef USE_STATISTICS
312 MPIDI_Statistics_time(MPIDI_Statistics.recvq.unexpected_search, search_length);
313 #endif
314 return NULL;
315 }
316
317
318 #ifdef OUT_OF_ORDER_HANDLING
319 /**
320 * insert a request _req2 before _req1
321 */
322 #define MPIDI_Recvq_insert(__Q, __req1, __req2) \
323 ({ \
324 (__req2)->mpid.next = __req1; \
325 if((__req1)->mpid.prev != NULL) \
326 ((__req1)->mpid.prev)->mpid.next = (__req2); \
327 else \
328 __Q ## _head = __req2; \
329 (__req2)->mpid.prev = (__req1)->mpid.prev; \
330 (__req1)->mpid.prev = (__req2); \
331 })
332
333 /**
334 * remove a request from out of order list
335 */
336 #define MPIDI_Recvq_remove_req_from_ool(req,in_cntr) \
337 ({ \
338 in_cntr->n_OutOfOrderMsgs--; \
339 if (in_cntr->n_OutOfOrderMsgs == 0) { \
340 in_cntr->OutOfOrderList=NULL; \
341 req->mpid.nextR=NULL; \
342 req->mpid.prevR=NULL; \
343 } else if (in_cntr->n_OutOfOrderMsgs > 0) { \
344 in_cntr->OutOfOrderList=req->mpid.nextR; \
345 /* remove req from out of order list */ \
346 ((MPID_Request *)(req)->mpid.prevR)->mpid.nextR = (req)->mpid.nextR; \
347 ((MPID_Request *)(req)->mpid.nextR)->mpid.prevR = (req)->mpid.prevR; \
348 (req)->mpid.nextR=NULL; \
349 (req)->mpid.prevR=NULL; \
350 } \
351 })
352 #endif
353
354
355 #endif
356