1 /*
2 * Copyright (c) 2019 Amazon.com, Inc. or its affiliates.
3 * All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #ifndef _RXR_PKT_TYPE_REQ_H
35 #define _RXR_PKT_TYPE_REQ_H
36
37 /*
38 * This file contain REQ packet type related struct and functions
39 * REQ packets can be classifed into 4 categories:
40 * RTM (Request To Message) is used by message
41 * RTW (Request To Write) is used by RMA write
42 * RTR (Request To Read) is used by RMA read
43 * RTA (Request To Atomic) is used by Atomic
44 *
45 * For each REQ packet type need to have the following:
46 *
47 * 1. a header struct
48 * 2. an init() function called by rxr_pkt_init_ctrl()
49 * 3. a handle_sent() function called by rxr_pkt_post_ctrl()
50 * 4. a handle_send_completion() function called by
51 * rxr_pkt_handle_send_completion()
52 * 5. a proc() function called by
53 * rxr_pkt_proc_req()
54 *
55 * Some req packet types are so similar that they can share
56 * some functions.
57 */
58
59 /*
60 * Utilities shared by all REQ packets
61 *
62 * Packet Header Flags
63 */
64 #define RXR_REQ_OPT_RAW_ADDR_HDR BIT_ULL(0)
65 #define RXR_REQ_OPT_CQ_DATA_HDR BIT_ULL(1)
66 #define RXR_REQ_MSG BIT_ULL(2)
67 #define RXR_REQ_TAGGED BIT_ULL(3)
68 #define RXR_REQ_RMA BIT_ULL(4)
69 #define RXR_REQ_ATOMIC BIT_ULL(5)
70
71 /*
72 * Extra Feature Flags
73 */
74 #define RXR_REQ_FEATURE_RDMA_READ BIT_ULL(0)
75
76 /*
77 * Utility struct and functions for
78 * REQ packet types
79 */
80 struct rxr_req_opt_raw_addr_hdr {
81 uint32_t addr_len;
82 char raw_addr[0];
83 };
84
85 struct rxr_req_opt_cq_data_hdr {
86 int64_t cq_data;
87 };
88
89 void rxr_pkt_proc_req_common_hdr(struct rxr_pkt_entry *pkt_entry);
90
91 size_t rxr_pkt_req_base_hdr_size(struct rxr_pkt_entry *pkt_entry);
92
93 size_t rxr_pkt_req_max_data_size(struct rxr_ep *ep, fi_addr_t addr, int pkt_type);
94
95 /*
96 * Structs and funcitons for RTM (Message) packet types
97 * There are 4 message protocols
98 * Eager message protocol,
99 * Medium message protocol,
100 * Long message protocol,
101 * Read message protocol (message by read)
102 * Each protocol employes two packet types: non-tagged and tagged.
103 * Thus altogether there are 8 RTM packet types.
104 */
105
106 /*
107 * Utility structs and functions shared by all
108 * RTM packet types
109 */
110 struct rxr_rtm_base_hdr {
111 uint8_t type;
112 uint8_t version;
113 uint16_t flags;
114 uint32_t msg_id;
115 };
116
117 static inline
rxr_get_rtm_base_hdr(void * pkt)118 struct rxr_rtm_base_hdr *rxr_get_rtm_base_hdr(void *pkt)
119 {
120 return (struct rxr_rtm_base_hdr *)pkt;
121 }
122
123 static inline
rxr_pkt_msg_id(struct rxr_pkt_entry * pkt_entry)124 uint32_t rxr_pkt_msg_id(struct rxr_pkt_entry *pkt_entry)
125 {
126 struct rxr_rtm_base_hdr *rtm_hdr;
127
128 rtm_hdr = rxr_get_rtm_base_hdr(pkt_entry->pkt);
129 /* only msg and atomic request has msg_id */
130 assert(rtm_hdr->flags & (RXR_REQ_MSG | RXR_REQ_ATOMIC));
131 return rtm_hdr->msg_id;
132 }
133
134 size_t rxr_pkt_rtm_total_len(struct rxr_pkt_entry *pkt_entry);
135
136 static inline
rxr_pkt_rtm_tag(struct rxr_pkt_entry * pkt_entry)137 uint64_t rxr_pkt_rtm_tag(struct rxr_pkt_entry *pkt_entry)
138 {
139 size_t offset;
140 uint64_t *tagptr;
141
142 /*
143 * In consideration of performance, this function did not cast header
144 * into different header types to get tag, but assume tag is always
145 * the last member of header.
146 */
147 offset = rxr_pkt_req_base_hdr_size(pkt_entry) - sizeof(uint64_t);
148 tagptr = (uint64_t *)((char *)pkt_entry->pkt + offset);
149 return *tagptr;
150 }
151
152 static inline
rxr_pkt_rtm_settag(struct rxr_pkt_entry * pkt_entry,uint64_t tag)153 void rxr_pkt_rtm_settag(struct rxr_pkt_entry *pkt_entry, uint64_t tag)
154 {
155 size_t offset;
156 uint64_t *tagptr;
157
158 offset = rxr_pkt_req_base_hdr_size(pkt_entry) - sizeof(uint64_t);
159 /* tag is always the last member */
160 tagptr = (uint64_t *)((char *)pkt_entry->pkt + offset);
161 *tagptr = tag;
162 }
163
164 /*
165 * Header structs for each REQ packe type
166 */
167 struct rxr_eager_msgrtm_hdr {
168 struct rxr_rtm_base_hdr hdr;
169 };
170
171 struct rxr_eager_tagrtm_hdr {
172 struct rxr_rtm_base_hdr hdr;
173 uint64_t tag;
174 };
175
176 struct rxr_medium_rtm_base_hdr {
177 struct rxr_rtm_base_hdr hdr;
178 uint64_t data_len;
179 uint64_t offset;
180 };
181
182 struct rxr_medium_msgrtm_hdr {
183 struct rxr_medium_rtm_base_hdr hdr;
184 };
185
186 struct rxr_medium_tagrtm_hdr {
187 struct rxr_medium_rtm_base_hdr hdr;
188 uint64_t tag;
189 };
190
191 static inline
rxr_get_medium_rtm_base_hdr(void * pkt)192 struct rxr_medium_rtm_base_hdr *rxr_get_medium_rtm_base_hdr(void *pkt)
193 {
194 return (struct rxr_medium_rtm_base_hdr *)pkt;
195 }
196
197 struct rxr_long_rtm_base_hdr {
198 struct rxr_rtm_base_hdr hdr;
199 uint64_t data_len;
200 uint32_t tx_id;
201 uint32_t credit_request;
202 };
203
204 static inline
rxr_get_long_rtm_base_hdr(void * pkt)205 struct rxr_long_rtm_base_hdr *rxr_get_long_rtm_base_hdr(void *pkt)
206 {
207 return (struct rxr_long_rtm_base_hdr *)pkt;
208 }
209
210 struct rxr_long_msgrtm_hdr {
211 struct rxr_long_rtm_base_hdr hdr;
212 };
213
214 struct rxr_long_tagrtm_hdr {
215 struct rxr_long_rtm_base_hdr hdr;
216 uint64_t tag;
217 };
218
219 struct rxr_read_rtm_base_hdr {
220 struct rxr_rtm_base_hdr hdr;
221 uint64_t data_len;
222 uint32_t tx_id;
223 uint32_t read_iov_count;
224 };
225
226 static inline
rxr_get_read_rtm_base_hdr(void * pkt)227 struct rxr_read_rtm_base_hdr *rxr_get_read_rtm_base_hdr(void *pkt)
228 {
229 return (struct rxr_read_rtm_base_hdr *)pkt;
230 }
231
232 struct rxr_read_msgrtm_hdr {
233 struct rxr_read_rtm_base_hdr hdr;
234 };
235
236 struct rxr_read_tagrtm_hdr {
237 struct rxr_read_rtm_base_hdr hdr;
238 uint64_t tag;
239 };
240
241 static inline
rxr_read_rtm_pkt_type(int op)242 int rxr_read_rtm_pkt_type(int op)
243 {
244 assert(op == ofi_op_tagged || op == ofi_op_msg);
245 return (op == ofi_op_tagged) ? RXR_READ_TAGRTM_PKT
246 : RXR_READ_MSGRTM_PKT;
247 }
248
249 /*
250 * init() functions for RTM packets
251 */
252 ssize_t rxr_pkt_init_eager_msgrtm(struct rxr_ep *ep,
253 struct rxr_tx_entry *tx_entry,
254 struct rxr_pkt_entry *pkt_entry);
255
256 ssize_t rxr_pkt_init_eager_tagrtm(struct rxr_ep *ep,
257 struct rxr_tx_entry *tx_entry,
258 struct rxr_pkt_entry *pkt_entry);
259
260 ssize_t rxr_pkt_init_medium_msgrtm(struct rxr_ep *ep,
261 struct rxr_tx_entry *tx_entry,
262 struct rxr_pkt_entry *pkt_entry);
263
264 ssize_t rxr_pkt_init_medium_tagrtm(struct rxr_ep *ep,
265 struct rxr_tx_entry *tx_entry,
266 struct rxr_pkt_entry *pkt_entry);
267
268 ssize_t rxr_pkt_init_long_msgrtm(struct rxr_ep *ep,
269 struct rxr_tx_entry *tx_entry,
270 struct rxr_pkt_entry *pkt_entry);
271
272 ssize_t rxr_pkt_init_long_tagrtm(struct rxr_ep *ep,
273 struct rxr_tx_entry *tx_entry,
274 struct rxr_pkt_entry *pkt_entry);
275
276 ssize_t rxr_pkt_init_read_msgrtm(struct rxr_ep *ep,
277 struct rxr_tx_entry *tx_entry,
278 struct rxr_pkt_entry *pkt_entry);
279
280 ssize_t rxr_pkt_init_read_tagrtm(struct rxr_ep *ep,
281 struct rxr_tx_entry *tx_entry,
282 struct rxr_pkt_entry *pkt_entry);
283 /*
284 * handle_sent() functions for RTM packets
285 */
286 static inline
rxr_pkt_handle_eager_rtm_sent(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)287 void rxr_pkt_handle_eager_rtm_sent(struct rxr_ep *ep,
288 struct rxr_pkt_entry *pkt_entry)
289 {
290 /* there is nothing to be done for eager RTM */
291 return;
292 }
293
294 void rxr_pkt_handle_medium_rtm_sent(struct rxr_ep *ep,
295 struct rxr_pkt_entry *pkt_entry);
296
297 void rxr_pkt_handle_long_rtm_sent(struct rxr_ep *ep,
298 struct rxr_pkt_entry *pkt_entry);
299
300 static inline
rxr_pkt_handle_read_rtm_sent(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)301 void rxr_pkt_handle_read_rtm_sent(struct rxr_ep *ep,
302 struct rxr_pkt_entry *pkt_entry)
303 {
304 }
305
306 /*
307 * handle_send_completion() functions for RTM packet types
308 */
309 void rxr_pkt_handle_eager_rtm_send_completion(struct rxr_ep *ep,
310 struct rxr_pkt_entry *pkt_entry);
311
312 void rxr_pkt_handle_medium_rtm_send_completion(struct rxr_ep *ep,
313 struct rxr_pkt_entry *pkt_entry);
314
315 void rxr_pkt_handle_long_rtm_send_completion(struct rxr_ep *ep,
316 struct rxr_pkt_entry *pkt_entry);
317
318 static inline
rxr_pkt_handle_read_rtm_send_completion(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)319 void rxr_pkt_handle_read_rtm_send_completion(struct rxr_ep *ep,
320 struct rxr_pkt_entry *pkt_entry)
321 {
322 }
323
324 /*
325 * proc() functions for RTM packet types
326 */
327 void rxr_pkt_rtm_init_rx_entry(struct rxr_pkt_entry *pkt_entry,
328 struct rxr_rx_entry *rx_entry);
329
330 /* This function is called by both
331 * rxr_pkt_handle_rtm_recv() and
332 * rxr_msg_handle_unexp_match()
333 */
334 ssize_t rxr_pkt_proc_matched_rtm(struct rxr_ep *ep,
335 struct rxr_rx_entry *rx_entry,
336 struct rxr_pkt_entry *pkt_entry);
337
338 ssize_t rxr_pkt_proc_rtm_rta(struct rxr_ep *ep,
339 struct rxr_pkt_entry *pkt_entry);
340 /*
341 * This function is shared by all RTM packet types which handle
342 * reordering
343 */
344 void rxr_pkt_handle_rtm_rta_recv(struct rxr_ep *ep,
345 struct rxr_pkt_entry *pkt_entry);
346
347 /* Structs and functions for RTW packet types
348 * There are 3 write protocols
349 * Eager write protocol,
350 * Long write protocol and
351 * Read write protocol (write by read)
352 * Each protocol correspond to a packet type
353 */
354
355 /*
356 * Header structs
357 */
358 struct rxr_rtw_base_hdr {
359 uint8_t type;
360 uint8_t version;
361 uint16_t flags;
362 /* end of rxr_base_hdr */
363 uint32_t rma_iov_count;
364 };
365
366 static inline
rxr_get_rtw_base_hdr(void * pkt)367 struct rxr_rtw_base_hdr *rxr_get_rtw_base_hdr(void *pkt)
368 {
369 return (struct rxr_rtw_base_hdr *)pkt;
370 }
371
372 struct rxr_eager_rtw_hdr {
373 uint8_t type;
374 uint8_t version;
375 uint16_t flags;
376 /* end of rxr_base_hdr */
377 uint32_t rma_iov_count;
378 struct fi_rma_iov rma_iov[0];
379 };
380
381 struct rxr_long_rtw_hdr {
382 uint8_t type;
383 uint8_t version;
384 uint16_t flags;
385 /* end of rxr_base_hdr */
386 uint32_t rma_iov_count;
387 uint64_t data_len;
388 uint32_t tx_id;
389 uint32_t credit_request;
390 struct fi_rma_iov rma_iov[0];
391 };
392
393 struct rxr_read_rtw_hdr {
394 uint8_t type;
395 uint8_t version;
396 uint16_t flags;
397 /* end of rxr_base_hdr */
398 uint32_t rma_iov_count;
399 uint64_t data_len;
400 uint32_t tx_id;
401 uint32_t read_iov_count;
402 struct fi_rma_iov rma_iov[0];
403 };
404
405 /*
406 * init() functions for each RTW packet types
407 */
408 ssize_t rxr_pkt_init_eager_rtw(struct rxr_ep *ep,
409 struct rxr_tx_entry *tx_entry,
410 struct rxr_pkt_entry *pkt_entry);
411
412 ssize_t rxr_pkt_init_long_rtw(struct rxr_ep *ep,
413 struct rxr_tx_entry *tx_entry,
414 struct rxr_pkt_entry *pkt_entry);
415
416 ssize_t rxr_pkt_init_read_rtw(struct rxr_ep *ep,
417 struct rxr_tx_entry *tx_entry,
418 struct rxr_pkt_entry *pkt_entry);
419 /*
420 * handle_sent() functions
421 */
422 static inline
rxr_pkt_handle_eager_rtw_sent(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)423 void rxr_pkt_handle_eager_rtw_sent(struct rxr_ep *ep,
424 struct rxr_pkt_entry *pkt_entry)
425 {
426 /* For eager RTW, there is nothing to be done here */
427 return;
428 }
429
430 void rxr_pkt_handle_long_rtw_sent(struct rxr_ep *ep,
431 struct rxr_pkt_entry *pkt_entry);
432
433 static inline
rxr_pkt_handle_read_rtw_sent(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)434 void rxr_pkt_handle_read_rtw_sent(struct rxr_ep *ep,
435 struct rxr_pkt_entry *pkt_entry)
436 {
437 }
438
439 /*
440 * handle_send_completion() functions
441 */
442 void rxr_pkt_handle_eager_rtw_send_completion(struct rxr_ep *ep,
443 struct rxr_pkt_entry *pkt_entry);
444
445 void rxr_pkt_handle_long_rtw_send_completion(struct rxr_ep *ep,
446 struct rxr_pkt_entry *pkt_entry);
447
448 static inline
rxr_pkt_handle_read_rtw_send_completion(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)449 void rxr_pkt_handle_read_rtw_send_completion(struct rxr_ep *ep,
450 struct rxr_pkt_entry *pkt_entry)
451 {
452 }
453
454 /*
455 * handle_recv() functions
456 */
457 void rxr_pkt_handle_eager_rtw_recv(struct rxr_ep *ep,
458 struct rxr_pkt_entry *pkt_entry);
459
460 void rxr_pkt_handle_long_rtw_recv(struct rxr_ep *ep,
461 struct rxr_pkt_entry *pkt_entry);
462
463 void rxr_pkt_handle_read_rtw_recv(struct rxr_ep *ep,
464 struct rxr_pkt_entry *pkt_entry);
465
466 /* Structs and functions for RTR packet types
467 * There are 3 read protocols
468 * Short protocol,
469 * Long read protocol and
470 * RDMA read protocol
471 * Each protocol correspond to a packet type
472 */
473
474 /*
475 * Header structs
476 */
477 struct rxr_rtr_hdr {
478 uint8_t type;
479 uint8_t version;
480 uint16_t flags;
481 /* end of rxr_base_hdr */
482 uint32_t rma_iov_count;
483 uint64_t data_len;
484 uint32_t read_req_rx_id;
485 uint32_t read_req_window;
486 struct fi_rma_iov rma_iov[0];
487 };
488
489 static inline
rxr_get_rtr_hdr(void * pkt)490 struct rxr_rtr_hdr *rxr_get_rtr_hdr(void *pkt)
491 {
492 return (struct rxr_rtr_hdr *)pkt;
493 }
494
495 /*
496 * init() functions for each RTW packet types
497 */
498 ssize_t rxr_pkt_init_short_rtr(struct rxr_ep *ep,
499 struct rxr_tx_entry *tx_entry,
500 struct rxr_pkt_entry *pkt_entry);
501
502 ssize_t rxr_pkt_init_long_rtr(struct rxr_ep *ep,
503 struct rxr_tx_entry *tx_entry,
504 struct rxr_pkt_entry *pkt_entry);
505
506 /*
507 * handle_sent() functions
508 */
509 void rxr_pkt_handle_rtr_sent(struct rxr_ep *ep,
510 struct rxr_pkt_entry *pkt_entry);
511
512 /*
513 * handle_send_completion() functions
514 */
515 void rxr_pkt_handle_rtr_send_completion(struct rxr_ep *ep,
516 struct rxr_pkt_entry *pkt_entry);
517 /*
518 * handle_recv() functions
519 */
520 void rxr_pkt_handle_rtr_recv(struct rxr_ep *ep,
521 struct rxr_pkt_entry *pkt_entry);
522
523 /* Structs and functions for RTW packet types
524 * There are 2 atomic protocols
525 * write atomic protocol and,
526 * read/compare atomic protocol and
527 * Each protocol correspond to a packet type
528 */
529 struct rxr_rta_hdr {
530 uint8_t type;
531 uint8_t version;
532 uint16_t flags;
533 uint32_t msg_id;
534 /* end of rtm_base_hdr, atomic packet need msg_id for reordering */
535 uint32_t rma_iov_count;
536 uint32_t atomic_datatype;
537 uint32_t atomic_op;
538 uint32_t tx_id;
539 struct fi_rma_iov rma_iov[0];
540 };
541
542 static inline
rxr_get_rta_hdr(void * pkt)543 struct rxr_rta_hdr *rxr_get_rta_hdr(void *pkt)
544 {
545 return (struct rxr_rta_hdr *)pkt;
546 }
547
548 ssize_t rxr_pkt_init_write_rta(struct rxr_ep *ep, struct rxr_tx_entry *tx_entry, struct rxr_pkt_entry *pkt_entry);
549
550 ssize_t rxr_pkt_init_fetch_rta(struct rxr_ep *ep, struct rxr_tx_entry *tx_entry, struct rxr_pkt_entry *pkt_entry);
551
552 ssize_t rxr_pkt_init_compare_rta(struct rxr_ep *ep, struct rxr_tx_entry *tx_entry, struct rxr_pkt_entry *pkt_entry);
553
554 static inline
rxr_pkt_handle_rta_sent(struct rxr_ep * ep,struct rxr_pkt_entry * pkt_entry)555 void rxr_pkt_handle_rta_sent(struct rxr_ep *ep,
556 struct rxr_pkt_entry *pkt_entry)
557 {
558 }
559
560 void rxr_pkt_handle_write_rta_send_completion(struct rxr_ep *ep,
561 struct rxr_pkt_entry *pkt_entry);
562
563 /* no action to be taken for compare_rta and fetch rta's send completion therefore
564 * there are not functions named rxr_pkt_handle_compare/fetch_rta_send_completion()
565 */
566
567 int rxr_pkt_proc_write_rta(struct rxr_ep *ep,
568 struct rxr_pkt_entry *pkt_entry);
569
570 int rxr_pkt_proc_fetch_rta(struct rxr_ep *ep,
571 struct rxr_pkt_entry *pkt_entry);
572
573 int rxr_pkt_proc_compare_rta(struct rxr_ep *ep,
574 struct rxr_pkt_entry *pkt_entry);
575
576 void rxr_pkt_handle_rta_recv(struct rxr_ep *ep, struct rxr_pkt_entry *pkt_entry);
577 #endif
578