1 /* 2 * Copyright (C) by Argonne National Laboratory 3 * See COPYRIGHT in top-level directory 4 */ 5 6 #ifndef MPIDPKT_H_INCLUDED 7 #define MPIDPKT_H_INCLUDED 8 9 #include "mpir_op_util.h" 10 11 #ifdef HAVE_STDINT_H 12 #include <stdint.h> 13 #endif 14 #ifdef HAVE_INTTYPES_H 15 #include <inttypes.h> 16 #endif 17 18 /* Enable the use of data within the message packet for small messages */ 19 #define USE_EAGER_SHORT 20 #define MPIDI_EAGER_SHORT_INTS 4 21 /* FIXME: This appears to assume that sizeof(int) == 4 (or at least >= 4) */ 22 #define MPIDI_EAGER_SHORT_SIZE 16 23 24 /* This is the number of ints that can be carried within an RMA packet */ 25 #define MPIDI_RMA_IMMED_BYTES 8 26 27 /* Union for immediate data in RMA packet headers.*/ 28 typedef union { 29 char payload[MPIDI_RMA_IMMED_BYTES]; 30 #ifdef NEEDS_STRICT_ALIGNMENT 31 /* Because the data is accessed per predefined type in the packet handler 32 * of accumulate-like operations, we need extra union members to ensure 33 * aligned access. 34 * NOTE: this fix might increase the packet size (long double complex 35 * can be 32bytes), thus we only enable this fix for a few platforms which 36 * are alignment-sensitive.*/ 37 MPL_mem_alignment_t alignment; 38 #endif 39 } MPIDI_CH3_RMA_Immed_u; 40 41 /* Union over all types (integer, logical, and multi-language types) that are 42 allowed in a CAS operation. This is used to allocate enough space in the 43 packet header for immediate data. */ 44 typedef union { 45 #define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ cas_##type_name_; 46 MPIR_OP_TYPE_GROUP(C_INTEGER) 47 MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER) 48 MPIR_OP_TYPE_GROUP(LOGICAL) 49 MPIR_OP_TYPE_GROUP(BYTE) 50 MPIR_OP_TYPE_GROUP(C_INTEGER_EXTRA) 51 MPIR_OP_TYPE_GROUP(FORTRAN_INTEGER_EXTRA) 52 MPIR_OP_TYPE_GROUP(LOGICAL_EXTRA) 53 MPIR_OP_TYPE_GROUP(BYTE_EXTRA) 54 #undef MPIR_OP_TYPE_MACRO 55 } MPIDI_CH3_CAS_Immed_u; 56 57 /* Union over all types (all predefined types) that are allowed in a 58 Fetch-and-op operation. This can be too large for the packet header, so we 59 limit the immediate space in the header to FOP_IMMED_INTS. */ 60 61 /* *INDENT-OFF* */ 62 /* Indentation turned off because "indent" is getting confused with 63 * the lack of a semi-colon in the fields below */ 64 typedef union { 65 #define MPIR_OP_TYPE_MACRO(mpi_type_, c_type_, type_name_) c_type_ fop##type_name_; 66 MPIR_OP_TYPE_GROUP_ALL_BASIC 67 MPIR_OP_TYPE_GROUP_ALL_EXTRA 68 #undef MPIR_OP_TYPE_MACRO 69 } MPIDI_CH3_FOP_Immed_u; 70 /* *INDENT-ON* */ 71 72 /* 73 * Predefined packet types. This simplifies some of the code. 74 */ 75 /* FIXME: Having predefined names makes it harder to add new message types, 76 such as different RMA types. */ 77 typedef enum { 78 MPIDI_CH3_PKT_EAGER_SEND = 0, 79 #if defined(USE_EAGER_SHORT) 80 MPIDI_CH3_PKT_EAGERSHORT_SEND, 81 #endif /* defined(USE_EAGER_SHORT) */ 82 MPIDI_CH3_PKT_EAGER_SYNC_SEND, /* FIXME: no sync eager */ 83 MPIDI_CH3_PKT_EAGER_SYNC_ACK, 84 MPIDI_CH3_PKT_READY_SEND, 85 MPIDI_CH3_PKT_RNDV_REQ_TO_SEND, 86 MPIDI_CH3_PKT_RNDV_CLR_TO_SEND, 87 MPIDI_CH3_PKT_RNDV_SEND, /* FIXME: should be stream put */ 88 MPIDI_CH3_PKT_CANCEL_SEND_REQ, 89 MPIDI_CH3_PKT_CANCEL_SEND_RESP, 90 /* RMA Packets begin here */ 91 MPIDI_CH3_PKT_PUT, 92 MPIDI_CH3_PKT_PUT_IMMED, 93 MPIDI_CH3_PKT_GET, 94 MPIDI_CH3_PKT_ACCUMULATE, 95 MPIDI_CH3_PKT_ACCUMULATE_IMMED, 96 MPIDI_CH3_PKT_GET_ACCUM, 97 MPIDI_CH3_PKT_GET_ACCUM_IMMED, 98 MPIDI_CH3_PKT_FOP, 99 MPIDI_CH3_PKT_FOP_IMMED, 100 MPIDI_CH3_PKT_CAS_IMMED, 101 MPIDI_CH3_PKT_GET_RESP, 102 MPIDI_CH3_PKT_GET_RESP_IMMED, 103 MPIDI_CH3_PKT_GET_ACCUM_RESP, 104 MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED, 105 MPIDI_CH3_PKT_FOP_RESP, 106 MPIDI_CH3_PKT_FOP_RESP_IMMED, 107 MPIDI_CH3_PKT_CAS_RESP_IMMED, 108 MPIDI_CH3_PKT_LOCK, 109 MPIDI_CH3_PKT_LOCK_ACK, 110 MPIDI_CH3_PKT_LOCK_OP_ACK, 111 MPIDI_CH3_PKT_UNLOCK, 112 MPIDI_CH3_PKT_FLUSH, 113 MPIDI_CH3_PKT_ACK, /* ACK packet for FLUSH, UNLOCK, DECR_AT_COUNTER */ 114 MPIDI_CH3_PKT_DECR_AT_COUNTER, 115 /* RMA Packets end here */ 116 MPIDI_CH3_PKT_FLOW_CNTL_UPDATE, /* FIXME: Unused */ 117 MPIDI_CH3_PKT_CLOSE, 118 MPIDI_CH3_PKT_REVOKE, 119 #ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS 120 /* Dynamic Connection Management */ 121 MPIDI_CH3_PKT_CONN_ACK, 122 MPIDI_CH3_PKT_ACCEPT_ACK, 123 #endif 124 MPIDI_CH3_PKT_END_CH3, 125 /* The channel can define additional types by defining the value 126 * MPIDI_CH3_PKT_ENUM */ 127 #if defined(MPIDI_CH3_PKT_ENUM) 128 MPIDI_CH3_PKT_ENUM, 129 #endif 130 MPIDI_CH3_PKT_END_ALL, 131 MPIDI_CH3_PKT_INVALID = -1 /* forces a signed enum to quash warnings */ 132 } MPIDI_CH3_Pkt_type_t; 133 134 /* These pkt_flags can be "OR'ed" together */ 135 typedef enum { 136 MPIDI_CH3_PKT_FLAG_NONE = 0, 137 MPIDI_CH3_PKT_FLAG_RMA_LOCK_SHARED = 1, 138 MPIDI_CH3_PKT_FLAG_RMA_LOCK_EXCLUSIVE = 2, 139 MPIDI_CH3_PKT_FLAG_RMA_UNLOCK = 4, 140 MPIDI_CH3_PKT_FLAG_RMA_FLUSH = 8, 141 MPIDI_CH3_PKT_FLAG_RMA_REQ_ACK = 16, 142 MPIDI_CH3_PKT_FLAG_RMA_DECR_AT_COUNTER = 32, 143 MPIDI_CH3_PKT_FLAG_RMA_NOCHECK = 64, 144 MPIDI_CH3_PKT_FLAG_RMA_ACK = 128, 145 MPIDI_CH3_PKT_FLAG_RMA_LOCK_GRANTED = 256, 146 MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_QUEUED = 512, 147 MPIDI_CH3_PKT_FLAG_RMA_LOCK_QUEUED_DATA_DISCARDED = 1024, 148 MPIDI_CH3_PKT_FLAG_RMA_LOCK_DISCARDED = 2048, 149 MPIDI_CH3_PKT_FLAG_RMA_UNLOCK_NO_ACK = 4096, 150 MPIDI_CH3_PKT_FLAG_RMA_IMMED_RESP = 8192, 151 MPIDI_CH3_PKT_FLAG_RMA_STREAM = 16384 152 } MPIDI_CH3_Pkt_flags_t; 153 154 typedef struct MPIDI_CH3_Pkt_send { 155 MPIDI_CH3_Pkt_type_t type; /* XXX - uint8_t to conserve space ??? */ 156 MPIDI_Message_match match; 157 MPI_Request sender_req_id; /* needed for ssend and send cancel */ 158 intptr_t data_sz; 159 #if defined(MPID_USE_SEQUENCE_NUMBERS) 160 MPID_Seqnum_t seqnum; 161 #endif 162 } MPIDI_CH3_Pkt_send_t; 163 164 /* NOTE: Normal and synchronous eager sends, as well as all ready-mode sends, 165 use the same structure but have a different type value. */ 166 typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_send_t; 167 typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_eager_sync_send_t; 168 typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_ready_send_t; 169 170 #if defined(USE_EAGER_SHORT) 171 typedef struct MPIDI_CH3_Pkt_eagershort_send { 172 MPIDI_CH3_Pkt_type_t type; /* XXX - uint8_t to conserve space ??? */ 173 MPIDI_Message_match match; 174 intptr_t data_sz; 175 #if defined(MPID_USE_SEQUENCE_NUMBERS) 176 MPID_Seqnum_t seqnum; 177 #endif 178 int data[MPIDI_EAGER_SHORT_INTS]; /* FIXME: Experimental for now */ 179 } MPIDI_CH3_Pkt_eagershort_send_t; 180 #endif /* defined(USE_EAGER_SHORT) */ 181 182 typedef struct MPIDI_CH3_Pkt_eager_sync_ack { 183 MPIDI_CH3_Pkt_type_t type; 184 MPI_Request sender_req_id; 185 } MPIDI_CH3_Pkt_eager_sync_ack_t; 186 187 typedef MPIDI_CH3_Pkt_send_t MPIDI_CH3_Pkt_rndv_req_to_send_t; 188 189 typedef struct MPIDI_CH3_Pkt_rndv_clr_to_send { 190 MPIDI_CH3_Pkt_type_t type; 191 MPI_Request sender_req_id; 192 MPI_Request receiver_req_id; 193 } MPIDI_CH3_Pkt_rndv_clr_to_send_t; 194 195 typedef struct MPIDI_CH3_Pkt_rndv_send { 196 MPIDI_CH3_Pkt_type_t type; 197 MPI_Request receiver_req_id; 198 } MPIDI_CH3_Pkt_rndv_send_t; 199 200 typedef struct MPIDI_CH3_Pkt_cancel_send_req { 201 MPIDI_CH3_Pkt_type_t type; 202 MPIDI_Message_match match; 203 MPI_Request sender_req_id; 204 } MPIDI_CH3_Pkt_cancel_send_req_t; 205 206 typedef struct MPIDI_CH3_Pkt_cancel_send_resp { 207 MPIDI_CH3_Pkt_type_t type; 208 MPI_Request sender_req_id; 209 int ack; 210 } MPIDI_CH3_Pkt_cancel_send_resp_t; 211 212 /* *INDENT-OFF* */ 213 /* Indentation turned off because "indent" is getting confused with 214 * the lack of a semi-colon in the field below */ 215 #if defined(MPIDI_CH3_PKT_DEFS) 216 MPIDI_CH3_PKT_DEFS 217 #endif 218 /* *INDENT-ON* */ 219 220 #define MPIDI_CH3_PKT_RMA_GET_TARGET_DATATYPE(pkt_, datatype_, err_) \ 221 { \ 222 /* This macro returns target_datatype in RMA operation \ 223 packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ 224 err_ = MPI_SUCCESS; \ 225 switch((pkt_).type) { \ 226 case (MPIDI_CH3_PKT_PUT): \ 227 case (MPIDI_CH3_PKT_PUT_IMMED): \ 228 datatype_ = (pkt_).put.datatype; \ 229 break; \ 230 case (MPIDI_CH3_PKT_GET): \ 231 datatype_ = (pkt_).get.datatype; \ 232 break; \ 233 case (MPIDI_CH3_PKT_ACCUMULATE): \ 234 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 235 datatype_ = (pkt_).accum.datatype; \ 236 break; \ 237 case (MPIDI_CH3_PKT_GET_ACCUM): \ 238 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 239 datatype_ = (pkt_).get_accum.datatype; \ 240 break; \ 241 case (MPIDI_CH3_PKT_CAS_IMMED): \ 242 datatype_ = (pkt_).cas.datatype; \ 243 break; \ 244 case (MPIDI_CH3_PKT_FOP): \ 245 case (MPIDI_CH3_PKT_FOP_IMMED): \ 246 datatype_ = (pkt_).fop.datatype; \ 247 break; \ 248 default: \ 249 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 250 } \ 251 } 252 253 #define MPIDI_CH3_PKT_RMA_GET_TARGET_COUNT(pkt_, count_, err_) \ 254 { \ 255 /* This macro returns target_count in RMA operation \ 256 packets. (PUT, GET, ACC, GACC, CAS, FOP) */ \ 257 err_ = MPI_SUCCESS; \ 258 switch((pkt_).type) { \ 259 case (MPIDI_CH3_PKT_PUT): \ 260 case (MPIDI_CH3_PKT_PUT_IMMED): \ 261 count_ = (pkt_).put.count; \ 262 break; \ 263 case (MPIDI_CH3_PKT_GET): \ 264 count_ = (pkt_).get.count; \ 265 break; \ 266 case (MPIDI_CH3_PKT_ACCUMULATE): \ 267 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 268 count_ = (pkt_).accum.count; \ 269 break; \ 270 case (MPIDI_CH3_PKT_GET_ACCUM): \ 271 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 272 count_ = (pkt_).get_accum.count; \ 273 break; \ 274 case (MPIDI_CH3_PKT_CAS_IMMED): \ 275 case (MPIDI_CH3_PKT_FOP): \ 276 case (MPIDI_CH3_PKT_FOP_IMMED): \ 277 count_ = 1; \ 278 break; \ 279 default: \ 280 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 281 } \ 282 } 283 284 #define MPIDI_CH3_PKT_RMA_GET_IMMED_DATA_PTR(pkt_, immed_data_, err_) \ 285 { \ 286 /* This macro returns pointer to immed data in RMA operation \ 287 packets (PUT, ACC, GACC, FOP, CAS) and RMA response \ 288 packets (GET_RESP, GACC_RESP, FOP_RESP, CAS_RESP). */ \ 289 err_ = MPI_SUCCESS; \ 290 switch((pkt_).type) { \ 291 case (MPIDI_CH3_PKT_PUT_IMMED): \ 292 immed_data_ = &((pkt_).put.info.data); \ 293 break; \ 294 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 295 immed_data_ = &((pkt_).accum.info.data); \ 296 break; \ 297 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 298 immed_data_ = &((pkt_).get_accum.info.data); \ 299 break; \ 300 case (MPIDI_CH3_PKT_FOP_IMMED): \ 301 immed_data_ = &((pkt_).fop.info.data); \ 302 break; \ 303 case (MPIDI_CH3_PKT_CAS_IMMED): \ 304 /* Note that here we return pointer of origin data, not \ 305 pointer of compare data. */ \ 306 immed_data_ = &((pkt_).cas.origin_data); \ 307 break; \ 308 case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ 309 immed_data_ = &((pkt_).get_resp.info.data); \ 310 break; \ 311 case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ 312 immed_data_ = &((pkt_).get_accum_resp.info.data); \ 313 break; \ 314 case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ 315 immed_data_ = &((pkt_).fop_resp.info.data); \ 316 break; \ 317 case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ 318 immed_data_ = &((pkt_).cas_resp.info.data); \ 319 break; \ 320 default: \ 321 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 322 } \ 323 } 324 325 #define MPIDI_CH3_PKT_RMA_GET_FLAGS(pkt_, flags_, err_) \ 326 { \ 327 /* This macro returns pkt_flags in RMA operation packets (PUT, GET, \ 328 ACC, GACC, FOP, CAS), RMA operation response packets \ 329 (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ 330 packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ 331 LOCK_OP_ACK) */ \ 332 err_ = MPI_SUCCESS; \ 333 switch((pkt_).type) { \ 334 case (MPIDI_CH3_PKT_PUT): \ 335 case (MPIDI_CH3_PKT_PUT_IMMED): \ 336 flags_ = (pkt_).put.pkt_flags; \ 337 break; \ 338 case (MPIDI_CH3_PKT_GET): \ 339 flags_ = (pkt_).get.pkt_flags; \ 340 break; \ 341 case (MPIDI_CH3_PKT_ACCUMULATE): \ 342 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 343 flags_ = (pkt_).accum.pkt_flags; \ 344 break; \ 345 case (MPIDI_CH3_PKT_GET_ACCUM): \ 346 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 347 flags_ = (pkt_).get_accum.pkt_flags; \ 348 break; \ 349 case (MPIDI_CH3_PKT_CAS_IMMED): \ 350 flags_ = (pkt_).cas.pkt_flags; \ 351 break; \ 352 case (MPIDI_CH3_PKT_FOP): \ 353 case (MPIDI_CH3_PKT_FOP_IMMED): \ 354 flags_ = (pkt_).fop.pkt_flags; \ 355 break; \ 356 case (MPIDI_CH3_PKT_GET_RESP): \ 357 case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ 358 flags_ = (pkt_).get_resp.pkt_flags; \ 359 break; \ 360 case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ 361 case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ 362 flags_ = (pkt_).get_accum_resp.pkt_flags; \ 363 break; \ 364 case (MPIDI_CH3_PKT_FOP_RESP): \ 365 case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ 366 flags_ = (pkt_).fop_resp.pkt_flags; \ 367 break; \ 368 case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ 369 flags_ = (pkt_).cas_resp.pkt_flags; \ 370 break; \ 371 case (MPIDI_CH3_PKT_LOCK): \ 372 flags_ = (pkt_).lock.pkt_flags; \ 373 break; \ 374 case (MPIDI_CH3_PKT_UNLOCK): \ 375 flags_ = (pkt_).unlock.pkt_flags; \ 376 break; \ 377 case (MPIDI_CH3_PKT_LOCK_ACK): \ 378 flags_ = (pkt_).lock_ack.pkt_flags; \ 379 break; \ 380 case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ 381 flags_ = (pkt_).lock_op_ack.pkt_flags; \ 382 break; \ 383 default: \ 384 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 385 } \ 386 } 387 388 #define MPIDI_CH3_PKT_RMA_GET_OP(pkt_, op_, err_) \ 389 { \ 390 /* This macro returns op in RMA operation packets (ACC, GACC, \ 391 FOP) */ \ 392 err_ = MPI_SUCCESS; \ 393 switch((pkt_).type) { \ 394 case (MPIDI_CH3_PKT_ACCUMULATE): \ 395 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 396 op_ = (pkt_).accum.op; \ 397 break; \ 398 case (MPIDI_CH3_PKT_GET_ACCUM): \ 399 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 400 op_ = (pkt_).get_accum.op; \ 401 break; \ 402 case (MPIDI_CH3_PKT_FOP): \ 403 case (MPIDI_CH3_PKT_FOP_IMMED): \ 404 op_ = (pkt_).fop.op; \ 405 break; \ 406 default: \ 407 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 408 } \ 409 } 410 411 #define MPIDI_CH3_PKT_RMA_ERASE_FLAGS(pkt_, err_) \ 412 { \ 413 /* This macro erases pkt_flags in RMA operation packets (PUT, GET, \ 414 ACC, GACC, FOP, CAS), RMA operation response packets \ 415 (GET_RESP, GET_ACCUM_RESP, FOP_RESP, CAS_RESP), RMA control \ 416 packets (UNLOCK) and RMA control response packets (LOCK_ACK, \ 417 LOCK_OP_ACK) */ \ 418 err_ = MPI_SUCCESS; \ 419 switch((pkt_).type) { \ 420 case (MPIDI_CH3_PKT_PUT): \ 421 case (MPIDI_CH3_PKT_PUT_IMMED): \ 422 (pkt_).put.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 423 break; \ 424 case (MPIDI_CH3_PKT_GET): \ 425 (pkt_).get.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 426 break; \ 427 case (MPIDI_CH3_PKT_ACCUMULATE): \ 428 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 429 (pkt_).accum.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 430 break; \ 431 case (MPIDI_CH3_PKT_GET_ACCUM): \ 432 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 433 (pkt_).get_accum.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 434 break; \ 435 case (MPIDI_CH3_PKT_CAS_IMMED): \ 436 (pkt_).cas.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 437 break; \ 438 case (MPIDI_CH3_PKT_FOP): \ 439 case (MPIDI_CH3_PKT_FOP_IMMED): \ 440 (pkt_).fop.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 441 break; \ 442 case (MPIDI_CH3_PKT_GET_RESP): \ 443 case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ 444 (pkt_).get_resp.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 445 break; \ 446 case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ 447 case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ 448 (pkt_).get_accum_resp.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 449 break; \ 450 case (MPIDI_CH3_PKT_FOP_RESP): \ 451 case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ 452 (pkt_).fop_resp.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 453 break; \ 454 case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ 455 (pkt_).cas_resp.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 456 break; \ 457 case (MPIDI_CH3_PKT_LOCK): \ 458 (pkt_).lock.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 459 break; \ 460 case (MPIDI_CH3_PKT_UNLOCK): \ 461 (pkt_).unlock.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 462 break; \ 463 case (MPIDI_CH3_PKT_LOCK_ACK): \ 464 (pkt_).lock_ack.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 465 break; \ 466 case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ 467 (pkt_).lock_op_ack.pkt_flags = MPIDI_CH3_PKT_FLAG_NONE; \ 468 break; \ 469 default: \ 470 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 471 } \ 472 } 473 474 #define MPIDI_CH3_PKT_RMA_GET_SOURCE_WIN_HANDLE(pkt_, win_hdl_, err_) \ 475 { \ 476 /* This macro returns source_win_handle in RMA operation \ 477 packets (PUT, GET, ACC, GACC, CAS, FOP), RMA operation \ 478 response packets (GET_RESP, GACC_RESP, CAS_RESP, FOP_RESP), \ 479 RMA control packets (LOCK, UNLOCK, FLUSH), and RMA control \ 480 response packets (LOCK_ACK, LOCK_OP_ACK, ACK). */ \ 481 err_ = MPI_SUCCESS; \ 482 switch((pkt_).type) { \ 483 case (MPIDI_CH3_PKT_PUT): \ 484 case (MPIDI_CH3_PKT_PUT_IMMED): \ 485 win_hdl_ = (pkt_).put.source_win_handle; \ 486 break; \ 487 case (MPIDI_CH3_PKT_ACCUMULATE): \ 488 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 489 win_hdl_ = (pkt_).accum.source_win_handle; \ 490 break; \ 491 case (MPIDI_CH3_PKT_LOCK): \ 492 win_hdl_ = (pkt_).lock.source_win_handle; \ 493 break; \ 494 case (MPIDI_CH3_PKT_UNLOCK): \ 495 win_hdl_ = (pkt_).unlock.source_win_handle; \ 496 break; \ 497 case (MPIDI_CH3_PKT_FLUSH): \ 498 win_hdl_ = (pkt_).flush.source_win_handle; \ 499 break; \ 500 case (MPIDI_CH3_PKT_LOCK_ACK): \ 501 win_hdl_ = (pkt_).lock_ack.source_win_handle; \ 502 break; \ 503 case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ 504 win_hdl_ = (pkt_).lock_op_ack.source_win_handle; \ 505 break; \ 506 case (MPIDI_CH3_PKT_ACK): \ 507 win_hdl_ = (pkt_).ack.source_win_handle; \ 508 break; \ 509 default: \ 510 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 511 } \ 512 } 513 514 #define MPIDI_CH3_PKT_RMA_GET_TARGET_WIN_HANDLE(pkt_, win_hdl_, err_) \ 515 { \ 516 /* This macro returns target_win_handle in RMA operation \ 517 packets (PUT, GET, ACC, GACC, CAS, FOP) and RMA control \ 518 packets (LOCK, UNLOCK, FLUSH, DECR_AT_CNT) */ \ 519 err_ = MPI_SUCCESS; \ 520 switch((pkt_).type) { \ 521 case (MPIDI_CH3_PKT_PUT): \ 522 case (MPIDI_CH3_PKT_PUT_IMMED): \ 523 win_hdl_ = (pkt_).put.target_win_handle; \ 524 break; \ 525 case (MPIDI_CH3_PKT_GET): \ 526 win_hdl_ = (pkt_).get.target_win_handle; \ 527 break; \ 528 case (MPIDI_CH3_PKT_ACCUMULATE): \ 529 case (MPIDI_CH3_PKT_ACCUMULATE_IMMED): \ 530 win_hdl_ = (pkt_).accum.target_win_handle; \ 531 break; \ 532 case (MPIDI_CH3_PKT_GET_ACCUM): \ 533 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 534 win_hdl_ = (pkt_).get_accum.target_win_handle; \ 535 break; \ 536 case (MPIDI_CH3_PKT_CAS_IMMED): \ 537 win_hdl_ = (pkt_).cas.target_win_handle; \ 538 break; \ 539 case (MPIDI_CH3_PKT_FOP): \ 540 case (MPIDI_CH3_PKT_FOP_IMMED): \ 541 win_hdl_ = (pkt_).fop.target_win_handle; \ 542 break; \ 543 case (MPIDI_CH3_PKT_LOCK): \ 544 win_hdl_ = (pkt_).lock.target_win_handle; \ 545 break; \ 546 case (MPIDI_CH3_PKT_UNLOCK): \ 547 win_hdl_ = (pkt_).unlock.target_win_handle; \ 548 break; \ 549 case (MPIDI_CH3_PKT_FLUSH): \ 550 win_hdl_ = (pkt_).flush.target_win_handle; \ 551 break; \ 552 case (MPIDI_CH3_PKT_DECR_AT_COUNTER): \ 553 win_hdl_ = (pkt_).decr_at_cnt.target_win_handle; \ 554 break; \ 555 default: \ 556 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 557 } \ 558 } 559 560 #define MPIDI_CH3_PKT_RMA_GET_REQUEST_HANDLE(pkt_, request_hdl_, err_) \ 561 { \ 562 err_ = MPI_SUCCESS; \ 563 switch((pkt_).type) { \ 564 case (MPIDI_CH3_PKT_GET): \ 565 request_hdl_ = (pkt_).get.request_handle; \ 566 break; \ 567 case (MPIDI_CH3_PKT_GET_ACCUM): \ 568 case (MPIDI_CH3_PKT_GET_ACCUM_IMMED): \ 569 request_hdl_ = (pkt_).get_accum.request_handle; \ 570 break; \ 571 case (MPIDI_CH3_PKT_CAS_IMMED): \ 572 request_hdl_ = (pkt_).cas.request_handle; \ 573 break; \ 574 case (MPIDI_CH3_PKT_FOP): \ 575 case (MPIDI_CH3_PKT_FOP_IMMED): \ 576 request_hdl_ = (pkt_).fop.request_handle; \ 577 break; \ 578 case (MPIDI_CH3_PKT_GET_RESP): \ 579 case (MPIDI_CH3_PKT_GET_RESP_IMMED): \ 580 request_hdl_ = (pkt_).get_resp.request_handle; \ 581 break; \ 582 case (MPIDI_CH3_PKT_GET_ACCUM_RESP): \ 583 case (MPIDI_CH3_PKT_GET_ACCUM_RESP_IMMED): \ 584 request_hdl_ = (pkt_).get_accum_resp.request_handle; \ 585 break; \ 586 case (MPIDI_CH3_PKT_FOP_RESP): \ 587 case (MPIDI_CH3_PKT_FOP_RESP_IMMED): \ 588 request_hdl_ = (pkt_).fop_resp.request_handle; \ 589 break; \ 590 case (MPIDI_CH3_PKT_CAS_RESP_IMMED): \ 591 request_hdl_ = (pkt_).cas_resp.request_handle; \ 592 break; \ 593 case (MPIDI_CH3_PKT_LOCK): \ 594 request_hdl_ = (pkt_).lock.request_handle; \ 595 break; \ 596 case (MPIDI_CH3_PKT_LOCK_ACK): \ 597 request_hdl_ = (pkt_).lock_ack.request_handle; \ 598 break; \ 599 case (MPIDI_CH3_PKT_LOCK_OP_ACK): \ 600 request_hdl_ = (pkt_).lock_op_ack.request_handle; \ 601 break; \ 602 default: \ 603 MPIR_ERR_SETANDJUMP1(err_, MPI_ERR_OTHER, "**invalidpkt", "**invalidpkt %d", (pkt_).type); \ 604 } \ 605 } 606 607 608 /* This macro judges if the RMA operation is a read operation, 609 * which means, it will triffer the issuing of response data from 610 * the target to the origin */ 611 #define MPIDI_CH3I_RMA_PKT_IS_READ_OP(pkt_) \ 612 ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \ 613 (pkt_).type == MPIDI_CH3_PKT_GET_ACCUM || \ 614 (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \ 615 (pkt_).type == MPIDI_CH3_PKT_FOP || \ 616 (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \ 617 (pkt_).type == MPIDI_CH3_PKT_GET) 618 619 /* This macro judges if the RMA operation is a immed operation */ 620 #define MPIDI_CH3I_RMA_PKT_IS_IMMED_OP(pkt_) \ 621 ((pkt_).type == MPIDI_CH3_PKT_GET_ACCUM_IMMED || \ 622 (pkt_).type == MPIDI_CH3_PKT_FOP_IMMED || \ 623 (pkt_).type == MPIDI_CH3_PKT_CAS_IMMED || \ 624 (pkt_).type == MPIDI_CH3_PKT_PUT_IMMED || \ 625 (pkt_).type == MPIDI_CH3_PKT_ACCUMULATE_IMMED) 626 627 typedef struct MPIDI_CH3_Pkt_put { 628 MPIDI_CH3_Pkt_type_t type; 629 int pkt_flags; 630 void *addr; 631 int count; 632 MPI_Datatype datatype; 633 MPI_Win target_win_handle; 634 MPI_Win source_win_handle; 635 union { 636 int flattened_type_size; 637 MPIDI_CH3_RMA_Immed_u data; 638 } info; 639 } MPIDI_CH3_Pkt_put_t; 640 641 typedef struct MPIDI_CH3_Pkt_get { 642 MPIDI_CH3_Pkt_type_t type; 643 int pkt_flags; 644 void *addr; 645 int count; 646 MPI_Datatype datatype; 647 struct { 648 int flattened_type_size; 649 } info; 650 MPI_Request request_handle; 651 MPI_Win target_win_handle; 652 } MPIDI_CH3_Pkt_get_t; 653 654 typedef struct MPIDI_CH3_Pkt_get_resp { 655 MPIDI_CH3_Pkt_type_t type; 656 MPI_Request request_handle; 657 /* followings are used to decrement ack_counter at origin */ 658 int target_rank; 659 int pkt_flags; 660 /* Followings are to piggyback IMMED data */ 661 struct { 662 /* note that we use struct here in order 663 * to consistently access data 664 * by "pkt->info.data". */ 665 MPIDI_CH3_RMA_Immed_u data; 666 } info; 667 } MPIDI_CH3_Pkt_get_resp_t; 668 669 typedef struct MPIDI_CH3_Pkt_accum { 670 MPIDI_CH3_Pkt_type_t type; 671 int pkt_flags; 672 void *addr; 673 int count; 674 MPI_Datatype datatype; 675 MPI_Op op; 676 MPI_Win target_win_handle; 677 MPI_Win source_win_handle; 678 union { 679 int flattened_type_size; 680 MPIDI_CH3_RMA_Immed_u data; 681 } info; 682 } MPIDI_CH3_Pkt_accum_t; 683 684 typedef struct MPIDI_CH3_Pkt_get_accum { 685 MPIDI_CH3_Pkt_type_t type; 686 int pkt_flags; 687 MPI_Request request_handle; /* For get_accumulate response */ 688 void *addr; 689 int count; 690 MPI_Datatype datatype; 691 MPI_Op op; 692 MPI_Win target_win_handle; 693 union { 694 int flattened_type_size; 695 MPIDI_CH3_RMA_Immed_u data; 696 } info; 697 } MPIDI_CH3_Pkt_get_accum_t; 698 699 typedef struct MPIDI_CH3_Pkt_get_accum_resp { 700 MPIDI_CH3_Pkt_type_t type; 701 MPI_Request request_handle; 702 /* followings are used to decrement ack_counter at origin */ 703 int target_rank; 704 int pkt_flags; 705 /* Followings are to piggyback IMMED data */ 706 struct { 707 /* note that we use struct here in order 708 * to consistently access data 709 * by "pkt->info.data". */ 710 MPIDI_CH3_RMA_Immed_u data; 711 } info; 712 } MPIDI_CH3_Pkt_get_accum_resp_t; 713 714 typedef struct MPIDI_CH3_Pkt_cas { 715 MPIDI_CH3_Pkt_type_t type; 716 int pkt_flags; 717 MPI_Datatype datatype; 718 void *addr; 719 MPI_Request request_handle; 720 MPI_Win target_win_handle; 721 MPIDI_CH3_CAS_Immed_u origin_data; 722 MPIDI_CH3_CAS_Immed_u compare_data; 723 } MPIDI_CH3_Pkt_cas_t; 724 725 typedef struct MPIDI_CH3_Pkt_cas_resp { 726 MPIDI_CH3_Pkt_type_t type; 727 MPI_Request request_handle; 728 struct { 729 /* note that we use struct here in order 730 * to consistently access data 731 * by "pkt->info.data". */ 732 MPIDI_CH3_CAS_Immed_u data; 733 } info; 734 /* followings are used to decrement ack_counter at orign */ 735 int target_rank; 736 int pkt_flags; 737 } MPIDI_CH3_Pkt_cas_resp_t; 738 739 typedef struct MPIDI_CH3_Pkt_fop { 740 MPIDI_CH3_Pkt_type_t type; 741 int pkt_flags; 742 MPI_Datatype datatype; 743 void *addr; 744 MPI_Op op; 745 MPI_Request request_handle; 746 MPI_Win target_win_handle; 747 struct { 748 /* note that we use struct here in order 749 * to consistently access data 750 * by "pkt->info.data". */ 751 MPIDI_CH3_RMA_Immed_u data; 752 } info; 753 } MPIDI_CH3_Pkt_fop_t; 754 755 typedef struct MPIDI_CH3_Pkt_fop_resp { 756 MPIDI_CH3_Pkt_type_t type; 757 MPI_Request request_handle; 758 struct { 759 /* note that we use struct here in order 760 * to consistently access data 761 * by "pkt->info.data". */ 762 MPIDI_CH3_RMA_Immed_u data; 763 } info; 764 /* followings are used to decrement ack_counter at orign */ 765 int target_rank; 766 int pkt_flags; 767 } MPIDI_CH3_Pkt_fop_resp_t; 768 769 typedef struct MPIDI_CH3_Pkt_lock { 770 MPIDI_CH3_Pkt_type_t type; 771 int pkt_flags; 772 MPI_Win target_win_handle; 773 /* Note that either source_win_handle 774 * or request_handle will be used. Here 775 * we need both of them because PUT/GET 776 * may be converted to LOCK packet, 777 * PUT has source_win_handle area and 778 * GET has request_handle area. */ 779 MPI_Win source_win_handle; 780 MPI_Request request_handle; 781 } MPIDI_CH3_Pkt_lock_t; 782 783 typedef struct MPIDI_CH3_Pkt_unlock { 784 MPIDI_CH3_Pkt_type_t type; 785 MPI_Win target_win_handle; 786 MPI_Win source_win_handle; 787 int pkt_flags; 788 } MPIDI_CH3_Pkt_unlock_t; 789 790 typedef struct MPIDI_CH3_Pkt_flush { 791 MPIDI_CH3_Pkt_type_t type; 792 MPI_Win target_win_handle; 793 MPI_Win source_win_handle; 794 } MPIDI_CH3_Pkt_flush_t; 795 796 typedef struct MPIDI_CH3_Pkt_lock_ack { 797 MPIDI_CH3_Pkt_type_t type; 798 int pkt_flags; 799 /* note that either source_win_handle 800 * or request_handle is used. */ 801 MPI_Win source_win_handle; 802 MPI_Request request_handle; 803 int target_rank; 804 } MPIDI_CH3_Pkt_lock_ack_t; 805 806 typedef struct MPIDI_CH3_Pkt_lock_op_ack { 807 MPIDI_CH3_Pkt_type_t type; 808 int pkt_flags; 809 /* note that either source_win_handle 810 * or request_handle is used. */ 811 MPI_Win source_win_handle; 812 MPI_Request request_handle; 813 int target_rank; 814 } MPIDI_CH3_Pkt_lock_op_ack_t; 815 816 /* This ACK packet is the acknowledgement 817 * for FLUSH, UNLOCK and DECR_AT_COUNTER 818 * packet */ 819 typedef struct MPIDI_CH3_Pkt_ack { 820 MPIDI_CH3_Pkt_type_t type; 821 MPI_Win source_win_handle; 822 int target_rank; 823 int pkt_flags; 824 } MPIDI_CH3_Pkt_ack_t; 825 826 typedef struct MPIDI_CH3_Pkt_decr_at_counter { 827 MPIDI_CH3_Pkt_type_t type; 828 MPI_Win target_win_handle; 829 MPI_Win source_win_handle; 830 int pkt_flags; 831 } MPIDI_CH3_Pkt_decr_at_counter_t; 832 833 typedef struct MPIDI_CH3_Pkt_close { 834 MPIDI_CH3_Pkt_type_t type; 835 int ack; 836 } MPIDI_CH3_Pkt_close_t; 837 838 #ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS 839 /* packet types used in dynamic process connection. */ 840 typedef struct MPIDI_CH3_Pkt_conn_ack { 841 MPIDI_CH3_Pkt_type_t type; 842 int ack; 843 } MPIDI_CH3_Pkt_conn_ack_t; 844 845 typedef MPIDI_CH3_Pkt_conn_ack_t MPIDI_CH3_Pkt_accept_ack_t; 846 #endif /* end of MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS */ 847 848 typedef struct MPIDI_CH3_Pkt_revoke { 849 MPIDI_CH3_Pkt_type_t type; 850 MPIR_Context_id_t revoked_comm; 851 } MPIDI_CH3_Pkt_revoke_t; 852 853 typedef union MPIDI_CH3_Pkt { 854 MPIDI_CH3_Pkt_type_t type; 855 MPIDI_CH3_Pkt_eager_send_t eager_send; 856 #if defined(USE_EAGER_SHORT) 857 MPIDI_CH3_Pkt_eagershort_send_t eagershort_send; 858 #endif /* defined(USE_EAGER_SHORT) */ 859 MPIDI_CH3_Pkt_eager_sync_send_t eager_sync_send; 860 MPIDI_CH3_Pkt_eager_sync_ack_t eager_sync_ack; 861 MPIDI_CH3_Pkt_eager_send_t ready_send; 862 MPIDI_CH3_Pkt_rndv_req_to_send_t rndv_req_to_send; 863 MPIDI_CH3_Pkt_rndv_clr_to_send_t rndv_clr_to_send; 864 MPIDI_CH3_Pkt_rndv_send_t rndv_send; 865 MPIDI_CH3_Pkt_cancel_send_req_t cancel_send_req; 866 MPIDI_CH3_Pkt_cancel_send_resp_t cancel_send_resp; 867 MPIDI_CH3_Pkt_put_t put; 868 MPIDI_CH3_Pkt_get_t get; 869 MPIDI_CH3_Pkt_get_resp_t get_resp; 870 MPIDI_CH3_Pkt_accum_t accum; 871 MPIDI_CH3_Pkt_get_accum_t get_accum; 872 MPIDI_CH3_Pkt_lock_t lock; 873 MPIDI_CH3_Pkt_lock_ack_t lock_ack; 874 MPIDI_CH3_Pkt_lock_op_ack_t lock_op_ack; 875 MPIDI_CH3_Pkt_unlock_t unlock; 876 MPIDI_CH3_Pkt_flush_t flush; 877 MPIDI_CH3_Pkt_ack_t ack; 878 MPIDI_CH3_Pkt_decr_at_counter_t decr_at_cnt; 879 MPIDI_CH3_Pkt_close_t close; 880 #ifndef MPIDI_CH3_HAS_NO_DYNAMIC_PROCESS 881 MPIDI_CH3_Pkt_conn_ack_t conn_ack; 882 MPIDI_CH3_Pkt_accept_ack_t accept_ack; 883 #endif 884 MPIDI_CH3_Pkt_cas_t cas; 885 MPIDI_CH3_Pkt_cas_resp_t cas_resp; 886 MPIDI_CH3_Pkt_fop_t fop; 887 MPIDI_CH3_Pkt_fop_resp_t fop_resp; 888 MPIDI_CH3_Pkt_get_accum_resp_t get_accum_resp; 889 MPIDI_CH3_Pkt_revoke_t revoke; 890 #if defined(MPIDI_CH3_PKT_DECL) 891 MPIDI_CH3_PKT_DECL 892 #endif 893 } MPIDI_CH3_Pkt_t; 894 895 /* Extended header packet types */ 896 897 typedef struct MPIDI_CH3_Ext_pkt_stream { 898 MPI_Aint stream_offset; 899 } MPIDI_CH3_Ext_pkt_stream_t; 900 901 #if defined(MPID_USE_SEQUENCE_NUMBERS) 902 typedef struct MPIDI_CH3_Pkt_send_container { 903 MPIDI_CH3_Pkt_send_t pkt; 904 struct MPIDI_CH3_Pkt_send_container_s *next; 905 } MPIDI_CH3_Pkt_send_container_t; 906 #endif 907 908 #endif /* MPIDPKT_H_INCLUDED */ 909