1 /*
2 * Copyright (c) 2010-2014 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #if !defined(RDMA_VERBS_H)
34 #define RDMA_VERBS_H
35
36 #include <assert.h>
37 #include <infiniband/verbs.h>
38 #include <rdma/rdma_cma.h>
39 #include <errno.h>
40
41 #ifdef __cplusplus
42 extern "C" {
43 #endif
44
rdma_seterrno(int ret)45 static inline int rdma_seterrno(int ret)
46 {
47 if (ret) {
48 errno = ret;
49 ret = -1;
50 }
51 return ret;
52 }
53
54 /*
55 * Shared receive queues.
56 */
57 int rdma_create_srq(struct rdma_cm_id *id, struct ibv_pd *pd,
58 struct ibv_srq_init_attr *attr);
59 int rdma_create_srq_ex(struct rdma_cm_id *id, struct ibv_srq_init_attr_ex *attr);
60
61 void rdma_destroy_srq(struct rdma_cm_id *id);
62
63
64 /*
65 * Memory registration helpers.
66 */
67 static inline struct ibv_mr *
rdma_reg_msgs(struct rdma_cm_id * id,void * addr,size_t length)68 rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
69 {
70 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
71 }
72
73 static inline struct ibv_mr *
rdma_reg_read(struct rdma_cm_id * id,void * addr,size_t length)74 rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
75 {
76 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
77 IBV_ACCESS_REMOTE_READ);
78 }
79
80 static inline struct ibv_mr *
rdma_reg_write(struct rdma_cm_id * id,void * addr,size_t length)81 rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
82 {
83 return ibv_reg_mr(id->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
84 IBV_ACCESS_REMOTE_WRITE);
85 }
86
87 static inline int
rdma_dereg_mr(struct ibv_mr * mr)88 rdma_dereg_mr(struct ibv_mr *mr)
89 {
90 return rdma_seterrno(ibv_dereg_mr(mr));
91 }
92
93
94 /*
95 * Vectored send, receive, and RDMA operations.
96 * Support multiple scatter-gather entries.
97 */
98 static inline int
rdma_post_recvv(struct rdma_cm_id * id,void * context,struct ibv_sge * sgl,int nsge)99 rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
100 int nsge)
101 {
102 struct ibv_recv_wr wr, *bad;
103
104 wr.wr_id = (uintptr_t) context;
105 wr.next = NULL;
106 wr.sg_list = sgl;
107 wr.num_sge = nsge;
108
109 if (id->srq)
110 return rdma_seterrno(ibv_post_srq_recv(id->srq, &wr, &bad));
111 else
112 return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad));
113 }
114
115 static inline int
rdma_post_sendv(struct rdma_cm_id * id,void * context,struct ibv_sge * sgl,int nsge,int flags)116 rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
117 int nsge, int flags)
118 {
119 struct ibv_send_wr wr, *bad;
120
121 wr.wr_id = (uintptr_t) context;
122 wr.next = NULL;
123 wr.sg_list = sgl;
124 wr.num_sge = nsge;
125 wr.opcode = IBV_WR_SEND;
126 wr.send_flags = flags;
127
128 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
129 }
130
131 static inline int
rdma_post_readv(struct rdma_cm_id * id,void * context,struct ibv_sge * sgl,int nsge,int flags,uint64_t remote_addr,uint32_t rkey)132 rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
133 int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
134 {
135 struct ibv_send_wr wr, *bad;
136
137 wr.wr_id = (uintptr_t) context;
138 wr.next = NULL;
139 wr.sg_list = sgl;
140 wr.num_sge = nsge;
141 wr.opcode = IBV_WR_RDMA_READ;
142 wr.send_flags = flags;
143 wr.wr.rdma.remote_addr = remote_addr;
144 wr.wr.rdma.rkey = rkey;
145
146 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
147 }
148
149 static inline int
rdma_post_writev(struct rdma_cm_id * id,void * context,struct ibv_sge * sgl,int nsge,int flags,uint64_t remote_addr,uint32_t rkey)150 rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
151 int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
152 {
153 struct ibv_send_wr wr, *bad;
154
155 wr.wr_id = (uintptr_t) context;
156 wr.next = NULL;
157 wr.sg_list = sgl;
158 wr.num_sge = nsge;
159 wr.opcode = IBV_WR_RDMA_WRITE;
160 wr.send_flags = flags;
161 wr.wr.rdma.remote_addr = remote_addr;
162 wr.wr.rdma.rkey = rkey;
163
164 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
165 }
166
167 /*
168 * Simple send, receive, and RDMA calls.
169 */
170 static inline int
rdma_post_recv(struct rdma_cm_id * id,void * context,void * addr,size_t length,struct ibv_mr * mr)171 rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
172 size_t length, struct ibv_mr *mr)
173 {
174 struct ibv_sge sge;
175
176 assert((addr >= mr->addr) &&
177 (((uint8_t *) addr + length) <= ((uint8_t *) mr->addr + mr->length)));
178 sge.addr = (uint64_t) (uintptr_t) addr;
179 sge.length = (uint32_t) length;
180 sge.lkey = mr->lkey;
181
182 return rdma_post_recvv(id, context, &sge, 1);
183 }
184
185 static inline int
rdma_post_send(struct rdma_cm_id * id,void * context,void * addr,size_t length,struct ibv_mr * mr,int flags)186 rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
187 size_t length, struct ibv_mr *mr, int flags)
188 {
189 struct ibv_sge sge;
190
191 sge.addr = (uint64_t) (uintptr_t) addr;
192 sge.length = (uint32_t) length;
193 sge.lkey = mr ? mr->lkey : 0;
194
195 return rdma_post_sendv(id, context, &sge, 1, flags);
196 }
197
198 static inline int
rdma_post_read(struct rdma_cm_id * id,void * context,void * addr,size_t length,struct ibv_mr * mr,int flags,uint64_t remote_addr,uint32_t rkey)199 rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
200 size_t length, struct ibv_mr *mr, int flags,
201 uint64_t remote_addr, uint32_t rkey)
202 {
203 struct ibv_sge sge;
204
205 sge.addr = (uint64_t) (uintptr_t) addr;
206 sge.length = (uint32_t) length;
207 sge.lkey = mr->lkey;
208
209 return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
210 }
211
212 static inline int
rdma_post_write(struct rdma_cm_id * id,void * context,void * addr,size_t length,struct ibv_mr * mr,int flags,uint64_t remote_addr,uint32_t rkey)213 rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
214 size_t length, struct ibv_mr *mr, int flags,
215 uint64_t remote_addr, uint32_t rkey)
216 {
217 struct ibv_sge sge;
218
219 sge.addr = (uint64_t) (uintptr_t) addr;
220 sge.length = (uint32_t) length;
221 sge.lkey = mr ? mr->lkey : 0;
222
223 return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
224 }
225
226 static inline int
rdma_post_ud_send(struct rdma_cm_id * id,void * context,void * addr,size_t length,struct ibv_mr * mr,int flags,struct ibv_ah * ah,uint32_t remote_qpn)227 rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
228 size_t length, struct ibv_mr *mr, int flags,
229 struct ibv_ah *ah, uint32_t remote_qpn)
230 {
231 struct ibv_send_wr wr, *bad;
232 struct ibv_sge sge;
233
234 sge.addr = (uint64_t) (uintptr_t) addr;
235 sge.length = (uint32_t) length;
236 sge.lkey = mr ? mr->lkey : 0;
237
238 wr.wr_id = (uintptr_t) context;
239 wr.next = NULL;
240 wr.sg_list = &sge;
241 wr.num_sge = 1;
242 wr.opcode = IBV_WR_SEND;
243 wr.send_flags = flags;
244 wr.wr.ud.ah = ah;
245 wr.wr.ud.remote_qpn = remote_qpn;
246 wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
247
248 return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
249 }
250
251 static inline int
rdma_get_send_comp(struct rdma_cm_id * id,struct ibv_wc * wc)252 rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
253 {
254 struct ibv_cq *cq;
255 void *context;
256 int ret;
257
258 do {
259 ret = ibv_poll_cq(id->send_cq, 1, wc);
260 if (ret)
261 break;
262
263 ret = ibv_req_notify_cq(id->send_cq, 0);
264 if (ret)
265 return rdma_seterrno(ret);
266
267 ret = ibv_poll_cq(id->send_cq, 1, wc);
268 if (ret)
269 break;
270
271 ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
272 if (ret)
273 return ret;
274
275 assert(cq == id->send_cq && context == id);
276 ibv_ack_cq_events(id->send_cq, 1);
277 } while (1);
278
279 return (ret < 0) ? rdma_seterrno(ret) : ret;
280 }
281
282 static inline int
rdma_get_recv_comp(struct rdma_cm_id * id,struct ibv_wc * wc)283 rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
284 {
285 struct ibv_cq *cq;
286 void *context;
287 int ret;
288
289 do {
290 ret = ibv_poll_cq(id->recv_cq, 1, wc);
291 if (ret)
292 break;
293
294 ret = ibv_req_notify_cq(id->recv_cq, 0);
295 if (ret)
296 return rdma_seterrno(ret);
297
298 ret = ibv_poll_cq(id->recv_cq, 1, wc);
299 if (ret)
300 break;
301
302 ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
303 if (ret)
304 return ret;
305
306 assert(cq == id->recv_cq && context == id);
307 ibv_ack_cq_events(id->recv_cq, 1);
308 } while (1);
309
310 return (ret < 0) ? rdma_seterrno(ret) : ret;
311 }
312
313 #ifdef __cplusplus
314 }
315 #endif
316
317 #endif /* RDMA_CMA_H */
318