1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IB_CLIENTS_IBD_H 28 #define _SYS_IB_CLIENTS_IBD_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #ifdef __cplusplus 33 extern "C" { 34 #endif 35 36 /* 37 * IETF defined IPoIB encapsulation header, with 2b of ethertype 38 * followed by 2 reserved bytes. This is at the start of the 39 * datagram sent to and received over the wire by the driver. 40 */ 41 typedef struct ipoib_header { 42 ushort_t ipoib_type; 43 ushort_t ipoib_mbz; 44 } ipoib_hdr_t; 45 46 #define IPOIB_HDRSIZE sizeof (struct ipoib_header) 47 48 /* 49 * IETF defined IPoIB link address; IBA QPN, followed by GID, 50 * which has a prefix and suffix, as reported via ARP. 51 */ 52 typedef struct ipoib_mac { 53 uint32_t ipoib_qpn; 54 uint32_t ipoib_gidpref[2]; 55 uint32_t ipoib_gidsuff[2]; 56 } ipoib_mac_t; 57 58 #define IPOIB_ADDRL sizeof (struct ipoib_mac) 59 60 /* 61 * Pseudo header prepended to datagram in DLIOCRAW transmit path 62 * and when GLD hands the datagram to the gldm_send entry point. 63 */ 64 typedef struct ipoib_ptxhdr { 65 ipoib_mac_t ipoib_dest; 66 ipoib_hdr_t ipoib_rhdr; 67 } ipoib_ptxhdr_t; 68 69 #define IPOIBDLSAP(p, offset) ((ipoib_ptxhdr_t *)((caddr_t)(p)+offset)) 70 71 /* 72 * The pseudo-GRH structure that sits before the data in the 73 * receive buffer, and is overlaid on top of the real GRH. 74 * The driver sets the ipoib_vertcflow to 0 if the pseudo-GRH 75 * does not hold valid information. If it is indicated valid, 76 * the driver must additionally provide the sender's qpn in 77 * network byte order in ipoib_sqpn, and not touch the 78 * remaining parts which were DMA'ed in by the IBA hardware. 79 */ 80 typedef struct ipoib_pgrh { 81 uint32_t ipoib_vertcflow; 82 uint32_t ipoib_sqpn; 83 uint32_t ipoib_sgid_pref[2]; 84 uint32_t ipoib_sgid_suff[2]; 85 uint32_t ipoib_dgid_pref[2]; 86 uint32_t ipoib_dgid_suff[2]; 87 } ipoib_pgrh_t; 88 89 /* 90 * The GRH is also dma'ed into recv buffers, thus space needs 91 * to be allocated for them. 92 */ 93 #define IPOIB_GRH_SIZE sizeof (ipoib_pgrh_t) 94 95 #if defined(_KERNEL) && !defined(_BOOT) 96 97 #include <sys/ib/ibtl/ibti.h> 98 #include <sys/ib/ib_pkt_hdrs.h> 99 #include <sys/list.h> 100 #include <sys/gld.h> 101 #include <sys/modhash.h> 102 103 #define IBD_HIWAT (64*1024) /* drv flow control high water */ 104 #define IBD_LOWAT (1024) /* drv flow control low water */ 105 #define IBD_IDNUM 0 /* ibd module ID; zero works */ 106 107 #define IBD_MAX_SQSEG 3 108 #define IBD_MAX_RQSEG 1 109 110 typedef struct ibd_copybuf_s { 111 ibt_mr_hdl_t ic_mr_hdl; 112 ibt_wr_ds_t ic_sgl; 113 ibt_mr_desc_t ic_mr_desc; 114 uint8_t *ic_bufaddr; 115 } ibd_copybuf_t; 116 117 typedef struct ibd_mblkbuf_s { 118 ibt_mr_hdl_t im_mr_hdl; 119 ibt_mr_desc_t im_mr_desc; 120 } ibd_mblkbuf_t; 121 122 /* 123 * Structure to encapsulate various types of async requests. 124 */ 125 typedef struct ibd_acache_rq { 126 struct list_node rq_list; /* list of pending work */ 127 int rq_op; /* what operation */ 128 ipoib_mac_t rq_mac; 129 ib_gid_t rq_gid; 130 void *rq_ptr; 131 } ibd_req_t; 132 133 134 typedef struct ibd_mcache { 135 struct list_node mc_list; /* full/non list */ 136 uint8_t mc_jstate; 137 boolean_t mc_fullreap; 138 ibt_mcg_info_t mc_info; 139 ibd_req_t mc_req; /* to queue LEAVE req */ 140 } ibd_mce_t; 141 142 typedef struct ibd_acache_s { 143 struct list_node ac_list; /* free/active list */ 144 ibt_ud_dest_hdl_t ac_dest; 145 ipoib_mac_t ac_mac; 146 uint32_t ac_ref; 147 ibd_mce_t *ac_mce; /* for MCG AHs */ 148 } ibd_ace_t; 149 150 typedef enum {IBD_WQE_SEND, IBD_WQE_RECV} ibd_wqe_type_t; 151 152 typedef struct ibd_wqe_s { 153 struct ibd_wqe_s *w_next; 154 struct ibd_wqe_s *w_prev; 155 ibd_wqe_type_t w_type; 156 ibd_copybuf_t w_copybuf; 157 mblk_t *im_mblk; 158 } ibd_wqe_t; 159 160 typedef struct ibd_swqe_s { 161 ibd_wqe_t w_ibd_swqe; 162 ibt_send_wr_t w_swr; 163 ibt_wr_ds_t w_smblk_sgl[IBD_MAX_SQSEG]; 164 ibd_mblkbuf_t w_smblkbuf[IBD_MAX_SQSEG]; 165 ibd_ace_t *w_ahandle; 166 void *w_mdtinfo; 167 } ibd_swqe_t; 168 169 #define swqe_next w_ibd_swqe.w_next 170 #define swqe_prev w_ibd_swqe.w_prev 171 #define swqe_type w_ibd_swqe.w_type 172 #define swqe_copybuf w_ibd_swqe.w_copybuf 173 #define swqe_im_mblk w_ibd_swqe.im_mblk 174 #define SWQE_TO_WQE(swqe) (ibd_wqe_t *)&((swqe)->w_ibd_swqe) 175 #define WQE_TO_SWQE(wqe) (ibd_swqe_t *)wqe 176 177 typedef struct ibd_rwqe_s { 178 ibd_wqe_t w_ibd_rwqe; 179 struct ibd_state_s *w_state; 180 ibt_recv_wr_t w_rwr; 181 boolean_t w_freeing_wqe; 182 frtn_t w_freemsg_cb; 183 } ibd_rwqe_t; 184 185 #define rwqe_next w_ibd_rwqe.w_next 186 #define rwqe_prev w_ibd_rwqe.w_prev 187 #define rwqe_type w_ibd_rwqe.w_type 188 #define rwqe_copybuf w_ibd_rwqe.w_copybuf 189 #define rwqe_im_mblk w_ibd_rwqe.im_mblk 190 #define RWQE_TO_WQE(rwqe) (ibd_wqe_t *)&((rwqe)->w_ibd_rwqe) 191 #define WQE_TO_RWQE(wqe) (ibd_rwqe_t *)wqe 192 193 194 typedef struct ibd_list_s { 195 ibd_wqe_t *dl_head; 196 ibd_wqe_t *dl_tail; 197 union { 198 boolean_t pending_sends; 199 uint32_t bufs_outstanding; 200 } ustat; 201 uint32_t dl_cnt; 202 kmutex_t dl_mutex; 203 } ibd_list_t; 204 205 #define dl_pending_sends ustat.pending_sends 206 #define dl_bufs_outstanding ustat.bufs_outstanding 207 208 /* 209 * This structure maintains information per port per HCA 210 * (per network interface). 211 */ 212 typedef struct ibd_state_s { 213 dev_info_t *id_dip; 214 ibt_clnt_hdl_t id_ibt_hdl; 215 ibt_hca_hdl_t id_hca_hdl; 216 ibt_pd_hdl_t id_pd_hdl; 217 218 uint32_t id_max_sqseg; 219 ibd_list_t id_tx_list; 220 uint32_t id_tx_sends; 221 kmutex_t id_txcomp_lock; 222 ibt_cq_hdl_t id_scq_hdl; 223 ibt_wc_t *id_txwcs; 224 225 uint32_t id_num_rwqe; 226 ibd_list_t id_rx_list; 227 ibt_cq_hdl_t id_rcq_hdl; 228 void *id_fifos; 229 int id_nfifos; 230 ibt_wc_t *id_wcs; 231 232 ibt_channel_hdl_t id_chnl_hdl; 233 ib_pkey_t id_pkey; 234 uint16_t id_pkix; 235 uint8_t id_port; 236 ibt_mcg_info_t *id_mcinfo; 237 238 gld_mac_info_t *id_macinfo; 239 ib_gid_t id_sgid; 240 ib_qpn_t id_qpnum; 241 ipoib_mac_t id_macaddr; 242 ib_gid_t id_mgid; 243 ipoib_mac_t id_bcaddr; 244 245 int id_mtu; 246 uchar_t id_scope; 247 248 struct list id_req_list; 249 250 kmutex_t id_acache_req_lock; 251 kcondvar_t id_acache_req_cv; 252 kt_did_t id_async_thrid; 253 254 kmutex_t id_ac_mutex; 255 mod_hash_t *id_ah_active_hash; 256 struct list id_ah_free; 257 struct list id_ah_active; 258 ipoib_mac_t id_ah_addr; 259 ibd_req_t id_ah_req; 260 char id_ah_op; 261 ibd_ace_t *id_ac_list; 262 263 kmutex_t id_mc_mutex; 264 struct list id_mc_full; 265 struct list id_mc_non; 266 ibd_req_t id_multi_req; 267 ipoib_mac_t id_multi_addr; 268 char id_multi_op; 269 boolean_t id_multi_queued; 270 271 kmutex_t id_trap_lock; 272 kcondvar_t id_trap_cv; 273 boolean_t id_trap_stop; 274 uint32_t id_trap_inprog; 275 276 int id_prom_op; 277 ibd_req_t id_prom_req; 278 279 kmutex_t id_sched_lock; 280 boolean_t id_sched_queued; 281 ibd_req_t id_sched_req; 282 283 kmutex_t id_link_mutex; 284 int32_t id_link_state; 285 uint64_t id_link_speed; 286 287 uint64_t id_ah_error; 288 uint64_t id_rx_short; 289 uint64_t id_num_intrs; 290 uint64_t id_tx_short; 291 uint32_t id_num_swqe; 292 } ibd_state_t; 293 294 #endif /* _KERNEL && !_BOOT */ 295 296 #ifdef __cplusplus 297 } 298 #endif 299 300 #endif /* _SYS_IB_CLIENTS_IBD_H */ 301