1 /** 2 * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED. 3 * 4 * See file LICENSE for terms. 5 */ 6 7 #ifndef UCT_RC_VERBS_H 8 #define UCT_RC_VERBS_H 9 10 #include <uct/ib/rc/base/rc_iface.h> 11 #include <uct/ib/rc/base/rc_ep.h> 12 #include <ucs/type/class.h> 13 14 15 #define UCT_RC_VERBS_IFACE_FOREACH_TXWQE(_iface, _i, _wc, _num_wcs) \ 16 status = uct_ib_poll_cq((_iface)->super.cq[UCT_IB_DIR_TX], &_num_wcs, _wc); \ 17 if (status != UCS_OK) { \ 18 return 0; \ 19 } \ 20 UCS_STATS_UPDATE_COUNTER((_iface)->stats, \ 21 UCT_RC_IFACE_STAT_TX_COMPLETION, _num_wcs); \ 22 for (_i = 0; _i < _num_wcs; ++_i) 23 24 25 enum { 26 UCT_RC_VERBS_ADDR_HAS_ATOMIC_MR = UCS_BIT(0) 27 }; 28 29 30 typedef struct uct_rc_verbs_ep_address { 31 uint8_t flags; 32 uct_ib_uint24_t qp_num; 33 uint64_t flush_addr; 34 uint32_t flush_rkey; 35 } UCS_S_PACKED uct_rc_verbs_ep_address_t; 36 37 38 typedef struct uct_rc_verbs_txcnt { 39 uint16_t pi; /* producer (post_send) count */ 40 uint16_t ci; /* consumer (ibv_poll_cq) completion count */ 41 } uct_rc_verbs_txcnt_t; 42 43 44 /** 45 * RC verbs communication context. 46 */ 47 typedef struct uct_rc_verbs_ep { 48 uct_rc_ep_t super; 49 uct_rc_verbs_txcnt_t txcnt; 50 uct_ib_fence_info_t fi; 51 struct ibv_qp *qp; 52 struct { 53 uintptr_t remote_addr; 54 uint32_t rkey; 55 } flush; 56 } uct_rc_verbs_ep_t; 57 58 59 /** 60 * RC verbs interface configuration. 61 */ 62 typedef struct uct_rc_verbs_iface_config { 63 uct_rc_iface_config_t super; 64 size_t max_am_hdr; 65 unsigned tx_max_wr; 66 } uct_rc_verbs_iface_config_t; 67 68 69 /** 70 * RC verbs interface. 71 */ 72 typedef struct uct_rc_verbs_iface { 73 uct_rc_iface_t super; 74 struct ibv_srq *srq; 75 struct ibv_send_wr inl_am_wr; 76 struct ibv_send_wr inl_rwrite_wr; 77 struct ibv_sge inl_sge[2]; 78 uct_rc_am_short_hdr_t am_inl_hdr; 79 ucs_mpool_t short_desc_mp; 80 uct_rc_iface_send_desc_t *fc_desc; /* used when max_inline is zero */ 81 struct ibv_mr *flush_mr; /* MR for writing dummy value to flush */ 82 void *flush_mem; 83 struct { 84 size_t short_desc_size; 85 size_t max_inline; 86 size_t max_send_sge; 87 unsigned tx_max_wr; 88 } config; 89 } uct_rc_verbs_iface_t; 90 91 92 ucs_status_t uct_rc_verbs_iface_flush_mem_create(uct_rc_verbs_iface_t *iface); 93 94 UCS_CLASS_DECLARE(uct_rc_verbs_ep_t, const uct_ep_params_t *); 95 UCS_CLASS_DECLARE_NEW_FUNC(uct_rc_verbs_ep_t, uct_ep_t, const uct_ep_params_t *); 96 UCS_CLASS_DECLARE_DELETE_FUNC(uct_rc_verbs_ep_t, uct_ep_t); 97 98 ucs_status_t uct_rc_verbs_ep_put_short(uct_ep_h tl_ep, const void *buffer, 99 unsigned length, uint64_t remote_addr, 100 uct_rkey_t rkey); 101 102 ssize_t uct_rc_verbs_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb, 103 void *arg, uint64_t remote_addr, 104 uct_rkey_t rkey); 105 106 ucs_status_t uct_rc_verbs_ep_put_zcopy(uct_ep_h tl_ep, 107 const uct_iov_t *iov, size_t iovcnt, 108 uint64_t remote_addr, uct_rkey_t rkey, 109 uct_completion_t *comp); 110 111 ucs_status_t uct_rc_verbs_ep_get_bcopy(uct_ep_h tl_ep, 112 uct_unpack_callback_t unpack_cb, 113 void *arg, size_t length, 114 uint64_t remote_addr, uct_rkey_t rkey, 115 uct_completion_t *comp); 116 117 ucs_status_t uct_rc_verbs_ep_get_zcopy(uct_ep_h tl_ep, 118 const uct_iov_t *iov, size_t iovcnt, 119 uint64_t remote_addr, uct_rkey_t rkey, 120 uct_completion_t *comp); 121 122 ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, 123 const void *buffer, unsigned length); 124 125 ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, 126 uct_pack_callback_t pack_cb, void *arg, 127 unsigned flags); 128 129 ucs_status_t uct_rc_verbs_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, 130 unsigned header_length, const uct_iov_t *iov, 131 size_t iovcnt, unsigned flags, 132 uct_completion_t *comp); 133 134 ucs_status_t uct_rc_verbs_ep_atomic_cswap64(uct_ep_h tl_ep, uint64_t compare, uint64_t swap, 135 uint64_t remote_addr, uct_rkey_t rkey, 136 uint64_t *result, uct_completion_t *comp); 137 138 ucs_status_t uct_rc_verbs_ep_atomic64_post(uct_ep_h tl_ep, unsigned opcode, uint64_t value, 139 uint64_t remote_addr, uct_rkey_t rkey); 140 141 ucs_status_t uct_rc_verbs_ep_atomic64_fetch(uct_ep_h tl_ep, uct_atomic_op_t opcode, 142 uint64_t value, uint64_t *result, 143 uint64_t remote_addr, uct_rkey_t rkey, 144 uct_completion_t *comp); 145 146 ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags, 147 uct_completion_t *comp); 148 149 ucs_status_t uct_rc_verbs_ep_fence(uct_ep_h tl_ep, unsigned flags); 150 151 ucs_status_t uct_rc_verbs_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, 152 uct_rc_fc_request_t *req); 153 154 ucs_status_t uct_rc_verbs_ep_handle_failure(uct_rc_verbs_ep_t *ep, 155 ucs_status_t status); 156 157 ucs_status_t uct_rc_verbs_ep_get_address(uct_ep_h tl_ep, uct_ep_addr_t *addr); 158 159 ucs_status_t uct_rc_verbs_ep_connect_to_ep(uct_ep_h tl_ep, 160 const uct_device_addr_t *dev_addr, 161 const uct_ep_addr_t *ep_addr); 162 163 #endif 164