1 /*
2 * Copyright (c) 2014-2016, Cisco Systems, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
33 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #include "config.h"
38
39 #include <asm/types.h>
40 #include <assert.h>
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <netinet/in.h>
44 #include <poll.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48
49 #include <rdma/fabric.h>
50 #include <rdma/fi_cm.h>
51 #include <rdma/fi_domain.h>
52 #include <rdma/fi_endpoint.h>
53 #include <rdma/fi_rma.h>
54 #include <rdma/fi_errno.h>
55 #include "ofi.h"
56
57 #include "usd.h"
58 #include "usd_post.h"
59
60 #include "usdf.h"
61 #include "usdf_dgram.h"
62 #include "usdf_av.h"
63
_usdf_iov_len(const struct iovec * iov,size_t count)64 static inline size_t _usdf_iov_len(const struct iovec *iov, size_t count)
65 {
66 size_t len;
67 size_t i;
68
69 for (i = 0, len = 0; i < count; i++)
70 len += iov[i].iov_len;
71
72 return len;
73 }
74
_usdf_find_hdr(struct usd_wq * wq)75 static inline struct usd_udp_hdr *_usdf_find_hdr(struct usd_wq *wq)
76 {
77 uint8_t *copybuf;
78
79 copybuf = wq->uwq_copybuf + (wq->uwq_post_index * USD_SEND_MAX_COPY);
80
81 return (struct usd_udp_hdr *) copybuf;
82 }
83
_usdf_adjust_hdr(struct usd_udp_hdr * hdr,struct usd_qp_impl * qp,size_t len)84 static inline void _usdf_adjust_hdr(struct usd_udp_hdr *hdr,
85 struct usd_qp_impl *qp, size_t len)
86 {
87 hdr->uh_ip.tot_len = htons(len + sizeof(struct usd_udp_hdr) -
88 sizeof(struct ether_header));
89 hdr->uh_udp.len = htons(len + sizeof(struct usd_udp_hdr) -
90 sizeof(struct ether_header) -
91 sizeof(struct iphdr));
92 hdr->uh_udp.source =
93 qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
94 }
95
_usdf_adjust_prefix_hdr(struct usd_udp_hdr * hdr,struct usd_qp_impl * qp,size_t len,size_t padding)96 static inline void _usdf_adjust_prefix_hdr(struct usd_udp_hdr *hdr,
97 struct usd_qp_impl *qp, size_t len, size_t padding)
98 {
99
100 hdr->uh_ip.tot_len = htons(len - padding - sizeof(struct ether_header));
101 hdr->uh_udp.len = htons(len - padding - sizeof(struct ether_header) -
102 sizeof(struct iphdr));
103 hdr->uh_udp.source =
104 qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
105 }
106
_usdf_adjust_post_info(struct usd_wq * wq,uint32_t last_post,void * context,size_t len)107 static inline void _usdf_adjust_post_info(struct usd_wq *wq, uint32_t last_post,
108 void *context, size_t len)
109 {
110 struct usd_wq_post_info *info;
111
112 info = &wq->uwq_post_info[last_post];
113 info->wp_context = context;
114 info->wp_len = len;
115 }
116
117 ssize_t
usdf_dgram_recv(struct fid_ep * fep,void * buf,size_t len,void * desc,fi_addr_t src_addr,void * context)118 usdf_dgram_recv(struct fid_ep *fep, void *buf, size_t len,
119 void *desc, fi_addr_t src_addr, void *context)
120 {
121 struct usdf_ep *ep;
122 struct usd_qp_impl *qp;
123 struct usd_recv_desc rxd;
124 uint32_t index;
125
126 ep = ep_ftou(fep);
127 qp = to_qpi(ep->e.dg.ep_qp);
128
129 index = qp->uq_rq.urq_post_index;
130 rxd.urd_context = context;
131 rxd.urd_iov[0].iov_base = (uint8_t *)ep->e.dg.ep_hdr_buf +
132 (index * USDF_HDR_BUF_ENTRY) +
133 (USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr));
134 rxd.urd_iov[0].iov_len = sizeof(struct usd_udp_hdr);
135 rxd.urd_iov[1].iov_base = buf;
136 rxd.urd_iov[1].iov_len = len;
137 rxd.urd_iov_cnt = 2;
138 rxd.urd_next = NULL;
139
140 ep->e.dg.ep_hdr_ptr[index] = rxd.urd_iov[0].iov_base;
141 index = (index + 1) & qp->uq_rq.urq_post_index_mask;
142 ep->e.dg.ep_hdr_ptr[index] = rxd.urd_iov[0].iov_base;
143
144 return usd_post_recv(ep->e.dg.ep_qp, &rxd);
145 }
146
147 ssize_t
usdf_dgram_recvv(struct fid_ep * fep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t src_addr,void * context)148 usdf_dgram_recvv(struct fid_ep *fep, const struct iovec *iov, void **desc,
149 size_t count, fi_addr_t src_addr, void *context)
150 {
151 struct usdf_ep *ep;
152 struct usd_recv_desc rxd;
153 struct usd_qp_impl *qp;
154 uint32_t index;
155 size_t i;
156
157 ep = ep_ftou(fep);
158 qp = to_qpi(ep->e.dg.ep_qp);
159
160 rxd.urd_context = context;
161 rxd.urd_iov[0].iov_base = ((uint8_t *)ep->e.dg.ep_hdr_buf) +
162 qp->uq_rq.urq_post_index * USDF_HDR_BUF_ENTRY;
163 rxd.urd_iov[0].iov_len = sizeof(struct usd_udp_hdr);
164 memcpy(&rxd.urd_iov[1], iov, sizeof(*iov) * count);
165 rxd.urd_iov_cnt = count + 1;
166 rxd.urd_next = NULL;
167
168 index = qp->uq_rq.urq_post_index;
169 for (i = 0; i < count; ++i) {
170 ep->e.dg.ep_hdr_ptr[index] = rxd.urd_iov[0].iov_base;
171 index = (index + 1) & qp->uq_rq.urq_post_index_mask;
172 }
173
174 return usd_post_recv(ep->e.dg.ep_qp, &rxd);
175 }
176
177 ssize_t
usdf_dgram_recvmsg(struct fid_ep * fep,const struct fi_msg * msg,uint64_t flags)178 usdf_dgram_recvmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t flags)
179 {
180 struct usdf_ep *ep;
181 struct usd_qp_impl *qp;
182 struct usd_rq *rq;
183 struct vnic_rq *vrq;
184 struct rq_enet_desc *desc;
185 const struct iovec *iovp;
186 uint8_t *hdr_ptr;
187 uint32_t index;
188 unsigned i;
189
190 ep = ep_ftou(fep);
191 qp = to_qpi(ep->e.dg.ep_qp);
192 rq = &qp->uq_rq;
193 vrq = &rq->urq_vnic_rq;
194 desc = rq->urq_next_desc;
195 index = rq->urq_post_index;
196
197 iovp = msg->msg_iov;
198 rq->urq_context[index] = msg->context;
199 hdr_ptr = ((uint8_t *)ep->e.dg.ep_hdr_buf) +
200 (index * USDF_HDR_BUF_ENTRY);
201 rq_enet_desc_enc(desc, (dma_addr_t) hdr_ptr,
202 RQ_ENET_TYPE_ONLY_SOP, sizeof(struct usd_udp_hdr));
203 ep->e.dg.ep_hdr_ptr[index] = (struct usd_udp_hdr *) hdr_ptr;
204
205 index = (index + 1) & rq->urq_post_index_mask;
206 desc = (struct rq_enet_desc *)
207 ((uintptr_t)rq->urq_desc_ring + (index << 4));
208
209 for (i = 0; i < msg->iov_count; ++i) {
210 rq->urq_context[index] = msg->context;
211 rq_enet_desc_enc(desc, (dma_addr_t) iovp[i].iov_base,
212 RQ_ENET_TYPE_NOT_SOP, iovp[i].iov_len);
213 ep->e.dg.ep_hdr_ptr[index] = (struct usd_udp_hdr *) hdr_ptr;
214
215 index = (index + 1) & rq->urq_post_index_mask;
216 desc = (struct rq_enet_desc *)
217 ((uintptr_t)rq->urq_desc_ring + (index << 4));
218 }
219
220 if ((flags & FI_MORE) == 0) {
221 wmb();
222 iowrite32(index, &vrq->ctrl->posted_index);
223 }
224
225 rq->urq_next_desc = desc;
226 rq->urq_post_index = index;
227 rq->urq_recv_credits -= msg->iov_count + 1;
228
229 return 0;
230 }
231
232 ssize_t
usdf_dgram_send(struct fid_ep * fep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,void * context)233 usdf_dgram_send(struct fid_ep *fep, const void *buf, size_t len, void *desc,
234 fi_addr_t dest_addr, void *context)
235 {
236 struct usdf_dest *dest;
237 struct usdf_ep *ep;
238 uint32_t flags;
239
240 ep = ep_ftou(fep);
241 dest = (struct usdf_dest *)(uintptr_t) dest_addr;
242 flags = (ep->ep_tx_completion) ? USD_SF_SIGNAL : 0;
243
244 assert(len <= ep->max_msg_size);
245
246 if (len + sizeof(struct usd_udp_hdr) <= USD_SEND_MAX_COPY) {
247 return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest,
248 buf, len, flags,
249 context);
250 } else if (ep->e.dg.tx_op_flags & FI_INJECT) {
251 USDF_DBG_SYS(EP_DATA,
252 "given inject length (%zu) exceeds max inject length (%d)\n",
253 len + sizeof(struct usd_udp_hdr),
254 USD_SEND_MAX_COPY);
255 return -FI_ENOSPC;
256 }
257
258 return usd_post_send_one(ep->e.dg.ep_qp, &dest->ds_dest, buf, len,
259 flags, context);
260 }
261
262 static ssize_t
_usdf_dgram_send_iov_copy(struct usdf_ep * ep,struct usd_dest * dest,const struct iovec * iov,size_t count,void * context,uint8_t cq_entry)263 _usdf_dgram_send_iov_copy(struct usdf_ep *ep, struct usd_dest *dest,
264 const struct iovec *iov, size_t count, void *context,
265 uint8_t cq_entry)
266 {
267 struct usd_wq *wq;
268 struct usd_qp_impl *qp;
269 struct usd_udp_hdr *hdr;
270 uint32_t last_post;
271 size_t len;
272 unsigned i;
273
274 qp = to_qpi(ep->e.dg.ep_qp);
275 wq = &qp->uq_wq;
276
277 hdr = _usdf_find_hdr(wq);
278 memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
279
280 len = 0;
281 for (i = 0; i < count; i++) {
282 memcpy((char *) hdr + sizeof(*hdr) + len, iov[i].iov_base,
283 iov[i].iov_len);
284 len += iov[i].iov_len;
285 }
286
287 assert(len <= ep->max_msg_size);
288
289 _usdf_adjust_hdr(hdr, qp, len);
290
291 last_post = _usd_post_send_one(wq, hdr, len + sizeof(*hdr), cq_entry);
292
293 _usdf_adjust_post_info(wq, last_post, context, len);
294
295 return 0;
296 }
297
_usdf_dgram_send_iov(struct usdf_ep * ep,struct usd_dest * dest,const struct iovec * iov,size_t count,void * context,uint8_t cq_entry)298 static ssize_t _usdf_dgram_send_iov(struct usdf_ep *ep, struct usd_dest *dest,
299 const struct iovec *iov, size_t count, void *context, uint8_t
300 cq_entry)
301 {
302 struct iovec send_iov[USDF_DGRAM_MAX_SGE];
303 struct usd_udp_hdr *hdr;
304 struct usd_qp_impl *qp;
305 struct usd_wq *wq;
306 uint32_t last_post;
307 size_t len;
308
309 qp = to_qpi(ep->e.dg.ep_qp);
310 wq = &qp->uq_wq;
311
312 len = _usdf_iov_len(iov, count);
313 hdr = _usdf_find_hdr(wq);
314 memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
315 _usdf_adjust_hdr(hdr, qp, len);
316
317 assert(len <= ep->max_msg_size);
318
319 send_iov[0].iov_base = hdr;
320 send_iov[0].iov_len = sizeof(*hdr);
321 memcpy(&send_iov[1], iov, sizeof(struct iovec) * count);
322
323 last_post = _usd_post_send_iov(wq, send_iov, count + 1,
324 cq_entry);
325 _usdf_adjust_post_info(wq, last_post, context, len);
326
327 return FI_SUCCESS;
328 }
329
330 ssize_t
usdf_dgram_sendv(struct fid_ep * fep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t dest_addr,void * context)331 usdf_dgram_sendv(struct fid_ep *fep, const struct iovec *iov, void **desc,
332 size_t count, fi_addr_t dest_addr, void *context)
333 {
334 struct usd_dest *dest;
335 struct usdf_ep *ep;
336 size_t len;
337
338 ep = ep_ftou(fep);
339 len = sizeof(struct usd_udp_hdr);
340 dest = (struct usd_dest *)(uintptr_t) dest_addr;
341
342 len += _usdf_iov_len(iov, count);
343 assert(len <= ep->max_msg_size);
344
345 if (len <= USD_SEND_MAX_COPY) {
346 return _usdf_dgram_send_iov_copy(ep, dest, iov, count, context,
347 ep->ep_tx_completion);
348 } else if (ep->e.dg.tx_op_flags & FI_INJECT) {
349 USDF_DBG_SYS(EP_DATA,
350 "given inject length (%zu) exceeds max inject length (%d)\n",
351 len, USD_SEND_MAX_COPY);
352 return -FI_ENOSPC;
353 }
354
355 if (count > ep->e.dg.tx_iov_limit) {
356 USDF_DBG_SYS(EP_DATA, "max iov count exceeded: %zu\n", count);
357 return -FI_ENOSPC;
358 }
359
360 return _usdf_dgram_send_iov(ep, dest, iov, count, context,
361 ep->ep_tx_completion);
362 }
363
364 ssize_t
usdf_dgram_sendmsg(struct fid_ep * fep,const struct fi_msg * msg,uint64_t flags)365 usdf_dgram_sendmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t flags)
366 {
367 struct usd_dest *dest;
368 struct usdf_ep *ep;
369 uint8_t completion;
370 size_t len;
371
372 ep = ep_ftou(fep);
373 len = sizeof(struct usd_udp_hdr);
374 dest = (struct usd_dest *)(uintptr_t) msg->addr;
375 completion = ep->ep_tx_dflt_signal_comp || (flags & FI_COMPLETION);
376
377 len += _usdf_iov_len(msg->msg_iov, msg->iov_count);
378 assert(len <= ep->max_msg_size);
379
380 if (len <= USD_SEND_MAX_COPY) {
381 return _usdf_dgram_send_iov_copy(ep, dest, msg->msg_iov,
382 msg->iov_count,
383 msg->context,
384 completion);
385 } else if (flags & FI_INJECT) {
386 USDF_DBG_SYS(EP_DATA,
387 "given inject length (%zu) exceeds max inject length (%d)\n",
388 len, USD_SEND_MAX_COPY);
389 return -FI_ENOSPC;
390 }
391
392 if (msg->iov_count > ep->e.dg.tx_iov_limit) {
393 USDF_DBG_SYS(EP_DATA, "max iov count exceeded: %zu\n",
394 msg->iov_count);
395 return -FI_ENOSPC;
396 }
397
398 return _usdf_dgram_send_iov(ep, dest, msg->msg_iov, msg->iov_count,
399 msg->context, completion);
400 }
401
402 ssize_t
usdf_dgram_inject(struct fid_ep * fep,const void * buf,size_t len,fi_addr_t dest_addr)403 usdf_dgram_inject(struct fid_ep *fep, const void *buf, size_t len,
404 fi_addr_t dest_addr)
405 {
406 struct usdf_dest *dest;
407 struct usdf_ep *ep;
408
409 ep = ep_ftou(fep);
410 dest = (struct usdf_dest *)(uintptr_t) dest_addr;
411
412 if (len + sizeof(struct usd_udp_hdr) > USD_SEND_MAX_COPY) {
413 USDF_DBG_SYS(EP_DATA,
414 "given inject length (%zu) exceeds max inject length (%d)\n",
415 len + sizeof(struct usd_udp_hdr),
416 USD_SEND_MAX_COPY);
417 return -FI_ENOSPC;
418 }
419
420 /*
421 * fi_inject never generates a completion
422 */
423 return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest, buf, len,
424 0, NULL);
425 }
426
usdf_dgram_prefix_inject(struct fid_ep * fep,const void * buf,size_t len,fi_addr_t dest_addr)427 ssize_t usdf_dgram_prefix_inject(struct fid_ep *fep, const void *buf,
428 size_t len, fi_addr_t dest_addr)
429 {
430 return usdf_dgram_inject(fep, ((uint8_t *)buf) + USDF_HDR_BUF_ENTRY,
431 len - USDF_HDR_BUF_ENTRY, dest_addr);
432 }
433
usdf_dgram_rx_size_left(struct fid_ep * fep)434 ssize_t usdf_dgram_rx_size_left(struct fid_ep *fep)
435 {
436 struct usdf_ep *ep;
437
438 USDF_DBG_SYS(EP_DATA, "\n");
439
440 if (fep == NULL)
441 return -FI_EINVAL;
442
443 ep = ep_ftou(fep);
444
445 if (!(ep->flags & USDF_EP_ENABLED))
446 return -FI_EOPBADSTATE;
447
448 return usd_get_recv_credits(ep->e.dg.ep_qp) /
449 (ep->e.dg.rx_iov_limit + 1);
450 }
451
usdf_dgram_tx_size_left(struct fid_ep * fep)452 ssize_t usdf_dgram_tx_size_left(struct fid_ep *fep)
453 {
454 struct usdf_ep *ep;
455
456 USDF_DBG_SYS(EP_DATA, "\n");
457
458 if (fep == NULL)
459 return -FI_EINVAL;
460
461 ep = ep_ftou(fep);
462
463 if (!(ep->flags & USDF_EP_ENABLED))
464 return -FI_EOPBADSTATE;
465
466 return usd_get_send_credits(ep->e.dg.ep_qp) /
467 (ep->e.dg.tx_iov_limit + 1);
468 }
469
470 /*
471 * Versions that rely on user to reserve space for header at start of buffer
472 */
473 ssize_t
usdf_dgram_prefix_recv(struct fid_ep * fep,void * buf,size_t len,void * desc,fi_addr_t src_addr,void * context)474 usdf_dgram_prefix_recv(struct fid_ep *fep, void *buf, size_t len,
475 void *desc, fi_addr_t src_addr, void *context)
476 {
477 struct usdf_ep *ep;
478 struct usd_qp_impl *qp;
479 struct usd_recv_desc rxd;
480 uint32_t index;
481
482 ep = ep_ftou(fep);
483 qp = to_qpi(ep->e.dg.ep_qp);
484
485 index = qp->uq_rq.urq_post_index;
486 rxd.urd_context = context;
487 rxd.urd_iov[0].iov_base = (uint8_t *)buf +
488 USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
489 rxd.urd_iov[0].iov_len = len -
490 (USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr));
491 rxd.urd_iov_cnt = 1;
492 rxd.urd_next = NULL;
493
494 ep->e.dg.ep_hdr_ptr[index] = rxd.urd_iov[0].iov_base;
495
496 return usd_post_recv(ep->e.dg.ep_qp, &rxd);
497 }
498
499 ssize_t
usdf_dgram_prefix_recvv(struct fid_ep * fep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t src_addr,void * context)500 usdf_dgram_prefix_recvv(struct fid_ep *fep, const struct iovec *iov,
501 void **desc, size_t count, fi_addr_t src_addr, void *context)
502 {
503 struct usdf_ep *ep;
504 struct usd_recv_desc rxd;
505 struct usd_qp_impl *qp;
506 uint32_t index;
507 size_t i;
508
509 ep = ep_ftou(fep);
510 qp = to_qpi(ep->e.dg.ep_qp);
511
512 rxd.urd_context = context;
513 memcpy(&rxd.urd_iov[0], iov, sizeof(*iov) * count);
514 rxd.urd_iov[0].iov_base = (uint8_t *)rxd.urd_iov[0].iov_base +
515 USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
516 rxd.urd_iov[0].iov_len -= (USDF_HDR_BUF_ENTRY -
517 sizeof(struct usd_udp_hdr));
518
519 rxd.urd_iov_cnt = count;
520 rxd.urd_next = NULL;
521
522 index = qp->uq_rq.urq_post_index;
523 for (i = 0; i < count; ++i) {
524 ep->e.dg.ep_hdr_ptr[index] = rxd.urd_iov[0].iov_base;
525 index = (index + 1) & qp->uq_rq.urq_post_index_mask;
526 }
527
528 return usd_post_recv(ep->e.dg.ep_qp, &rxd);
529 }
530
531 ssize_t
usdf_dgram_prefix_recvmsg(struct fid_ep * fep,const struct fi_msg * msg,uint64_t flags)532 usdf_dgram_prefix_recvmsg(struct fid_ep *fep, const struct fi_msg *msg, uint64_t flags)
533 {
534 struct usdf_ep *ep;
535 struct usd_qp_impl *qp;
536 struct usd_rq *rq;
537 struct vnic_rq *vrq;
538 struct rq_enet_desc *desc;
539 uint8_t *hdr_ptr;
540 const struct iovec *iovp;
541 uint32_t index;
542 unsigned i;
543
544 ep = ep_ftou(fep);
545 qp = to_qpi(ep->e.dg.ep_qp);
546 rq = &qp->uq_rq;
547 vrq = &rq->urq_vnic_rq;
548 desc = rq->urq_next_desc;
549 index = rq->urq_post_index;
550
551 iovp = msg->msg_iov;
552 rq->urq_context[index] = msg->context;
553 hdr_ptr = ((uint8_t *)iovp[0].iov_base) +
554 (USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr));
555 rq_enet_desc_enc(desc, (dma_addr_t) hdr_ptr,
556 RQ_ENET_TYPE_ONLY_SOP,
557 iovp[0].iov_len -
558 (USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr)));
559 ep->e.dg.ep_hdr_ptr[index] = (struct usd_udp_hdr *) hdr_ptr;
560
561 index = (index+1) & rq->urq_post_index_mask;
562 desc = (struct rq_enet_desc *) ((uintptr_t)rq->urq_desc_ring
563 + (index<<4));
564
565 for (i = 1; i < msg->iov_count; ++i) {
566 rq->urq_context[index] = msg->context;
567 rq_enet_desc_enc(desc, (dma_addr_t) iovp[i].iov_base,
568 RQ_ENET_TYPE_NOT_SOP, iovp[i].iov_len);
569 ep->e.dg.ep_hdr_ptr[index] = (struct usd_udp_hdr *) hdr_ptr;
570
571 index = (index+1) & rq->urq_post_index_mask;
572 desc = (struct rq_enet_desc *) ((uintptr_t)rq->urq_desc_ring
573 + (index<<4));
574 }
575
576 if ((flags & FI_MORE) == 0) {
577 wmb();
578 iowrite32(index, &vrq->ctrl->posted_index);
579 }
580
581 rq->urq_next_desc = desc;
582 rq->urq_post_index = index;
583 rq->urq_recv_credits -= msg->iov_count;
584
585 return 0;
586 }
587
588 ssize_t
usdf_dgram_prefix_send(struct fid_ep * fep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,void * context)589 usdf_dgram_prefix_send(struct fid_ep *fep, const void *buf, size_t len,
590 void *desc, fi_addr_t dest_addr, void *context)
591 {
592 struct usd_udp_hdr *hdr;
593 struct usd_qp_impl *qp;
594 struct usdf_dest *dest;
595 struct usdf_ep *ep;
596 struct usd_wq *wq;
597 uint32_t last_post;
598 uint32_t flags;
599 size_t padding;
600
601 ep = ep_ftou(fep);
602 dest = (struct usdf_dest *)(uintptr_t) dest_addr;
603 padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
604 flags = (ep->ep_tx_completion) ? USD_SF_SIGNAL : 0;
605
606 assert(len <= ep->max_msg_size);
607
608 if (ep->e.dg.tx_op_flags & FI_INJECT) {
609 if ((len - padding) > USD_SEND_MAX_COPY) {
610 USDF_DBG_SYS(EP_DATA,
611 "given inject length (%zu) exceeds max inject length (%d)\n",
612 len, USD_SEND_MAX_COPY);
613 return -FI_ENOSPC;
614 }
615
616 return usd_post_send_one_copy(ep->e.dg.ep_qp, &dest->ds_dest,
617 ((uint8_t *)buf) + USDF_HDR_BUF_ENTRY, len -
618 USDF_HDR_BUF_ENTRY, flags,
619 context);
620 }
621
622 qp = to_qpi(ep->e.dg.ep_qp);
623 wq = &qp->uq_wq;
624
625 hdr = (struct usd_udp_hdr *) ((char *) buf + padding);
626 memcpy(hdr, &dest->ds_dest.ds_dest.ds_udp.u_hdr, sizeof(*hdr));
627
628 _usdf_adjust_prefix_hdr(hdr, qp, len, padding);
629
630 last_post = _usd_post_send_one(wq, hdr, len - padding,
631 ep->ep_tx_completion);
632
633 _usdf_adjust_post_info(wq, last_post, context, len - USDF_HDR_BUF_ENTRY);
634
635 return FI_SUCCESS;
636 }
637
638 static ssize_t
_usdf_dgram_send_iov_prefix(struct usdf_ep * ep,struct usd_dest * dest,const struct iovec * iov,size_t count,void * context,uint8_t cq_entry)639 _usdf_dgram_send_iov_prefix(struct usdf_ep *ep,
640 struct usd_dest *dest, const struct iovec *iov,
641 size_t count, void *context, uint8_t cq_entry)
642 {
643 struct iovec send_iov[USDF_DGRAM_MAX_SGE];
644 struct usd_udp_hdr *hdr;
645 struct usd_qp_impl *qp;
646 uint32_t last_post;
647 struct usd_wq *wq;
648 size_t padding;
649 size_t len;
650
651 qp = to_qpi(ep->e.dg.ep_qp);
652 wq = &qp->uq_wq;
653
654 len = _usdf_iov_len(iov, count);
655 padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
656
657 assert(len <= ep->max_msg_size);
658
659 hdr = (struct usd_udp_hdr *) ((char *) iov[0].iov_base +
660 padding);
661 memcpy(hdr, &dest->ds_dest.ds_udp.u_hdr, sizeof(*hdr));
662
663 _usdf_adjust_prefix_hdr(hdr, qp, len, padding);
664
665 memcpy(send_iov, iov, sizeof(struct iovec) * count);
666 send_iov[0].iov_base = hdr;
667 send_iov[0].iov_len -= padding;
668
669 last_post = _usd_post_send_iov(wq, send_iov, count, cq_entry);
670 _usdf_adjust_post_info(wq, last_post, context, len - USDF_HDR_BUF_ENTRY);
671
672 return FI_SUCCESS;
673 }
674
675 ssize_t
usdf_dgram_prefix_sendv(struct fid_ep * fep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t dest_addr,void * context)676 usdf_dgram_prefix_sendv(struct fid_ep *fep, const struct iovec *iov,
677 void **desc, size_t count, fi_addr_t dest_addr, void *context)
678 {
679 struct iovec send_iov[USDF_DGRAM_MAX_SGE];
680 struct usd_dest *dest;
681 struct usdf_ep *ep;
682 size_t len;
683 size_t padding;
684
685 ep = ep_ftou(fep);
686 dest = (struct usd_dest *)(uintptr_t) dest_addr;
687 len = _usdf_iov_len(iov, count);
688 padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
689
690 assert(len <= ep->max_msg_size);
691
692 if (count > ep->e.dg.tx_iov_limit) {
693 USDF_DBG_SYS(EP_DATA, "max iov count exceeded: %zu\n", count);
694 return -FI_ENOSPC;
695 }
696
697 if ((len - padding) <= USD_SEND_MAX_COPY) {
698 /* _usdf_dgram_send_iov_copy isn't prefix aware and allocates
699 * its own prefix. reorganize iov[0] base to point to data and
700 * len to reflect data length.
701 */
702 memcpy(send_iov, iov, sizeof(struct iovec) * count);
703 send_iov[0].iov_base = ((char *) send_iov[0].iov_base +
704 USDF_HDR_BUF_ENTRY);
705 send_iov[0].iov_len -= USDF_HDR_BUF_ENTRY;
706
707 return _usdf_dgram_send_iov_copy(ep, dest, send_iov, count,
708 context, ep->ep_tx_completion);
709 } else if (ep->e.dg.tx_op_flags & FI_INJECT) {
710 USDF_DBG_SYS(EP_DATA,
711 "given inject length (%zu) exceeds max inject length (%d)\n",
712 len, USD_SEND_MAX_COPY);
713 return -FI_ENOSPC;
714 }
715
716 return _usdf_dgram_send_iov_prefix(ep, dest, iov, count, context,
717 ep->ep_tx_completion);
718 }
719
720 ssize_t
usdf_dgram_prefix_sendmsg(struct fid_ep * fep,const struct fi_msg * msg,uint64_t flags)721 usdf_dgram_prefix_sendmsg(struct fid_ep *fep, const struct fi_msg *msg,
722 uint64_t flags)
723 {
724 struct iovec send_iov[USDF_DGRAM_MAX_SGE];
725 struct usd_dest *dest;
726 struct usdf_ep *ep;
727 uint8_t completion;
728 size_t len;
729 size_t padding;
730
731 ep = ep_ftou(fep);
732 dest = (struct usd_dest *)(uintptr_t) msg->addr;
733 len = _usdf_iov_len(msg->msg_iov, msg->iov_count);
734 completion = ep->ep_tx_dflt_signal_comp || (flags & FI_COMPLETION);
735 padding = USDF_HDR_BUF_ENTRY - sizeof(struct usd_udp_hdr);
736
737 assert(len <= ep->max_msg_size);
738
739 if (msg->iov_count > ep->e.dg.tx_iov_limit) {
740 USDF_DBG_SYS(EP_DATA, "max iov count exceeded: %zu\n",
741 msg->iov_count);
742 return -FI_ENOSPC;
743 }
744
745 if ((len - padding) <= USD_SEND_MAX_COPY) {
746 /* _usdf_dgram_send_iov_copy isn't prefix aware and allocates
747 * its own prefix. reorganize iov[0] base to point to data and
748 * len to reflect data length.
749 */
750 memcpy(send_iov, msg->msg_iov,
751 sizeof(struct iovec) * msg->iov_count);
752 send_iov[0].iov_base = ((char *) send_iov[0].iov_base +
753 USDF_HDR_BUF_ENTRY);
754 send_iov[0].iov_len -= USDF_HDR_BUF_ENTRY;
755
756 return _usdf_dgram_send_iov_copy(ep, dest, send_iov,
757 msg->iov_count, msg->context, completion);
758 } else if (flags & FI_INJECT) {
759 USDF_DBG_SYS(EP_DATA,
760 "given inject length (%zu) exceeds max inject length (%d)\n",
761 len, USD_SEND_MAX_COPY);
762 return -FI_ENOSPC;
763 }
764
765 return _usdf_dgram_send_iov_prefix(ep, dest, msg->msg_iov,
766 msg->iov_count, msg->context, completion);
767 }
768
usdf_dgram_prefix_rx_size_left(struct fid_ep * fep)769 ssize_t usdf_dgram_prefix_rx_size_left(struct fid_ep *fep)
770 {
771 struct usdf_ep *ep;
772
773 USDF_DBG_SYS(EP_DATA, "\n");
774
775 if (fep == NULL)
776 return -FI_EINVAL;
777
778 ep = ep_ftou(fep);
779
780 if (!(ep->flags & USDF_EP_ENABLED))
781 return -FI_EOPBADSTATE;
782
783 /* prefix_recvv can post up to iov_limit descriptors
784 */
785 return (usd_get_recv_credits(ep->e.dg.ep_qp) / ep->e.dg.rx_iov_limit);
786 }
787
usdf_dgram_prefix_tx_size_left(struct fid_ep * fep)788 ssize_t usdf_dgram_prefix_tx_size_left(struct fid_ep *fep)
789 {
790 struct usdf_ep *ep;
791
792 USDF_DBG_SYS(EP_DATA, "\n");
793
794 if (fep == NULL)
795 return -FI_EINVAL;
796
797 ep = ep_ftou(fep);
798
799 if (!(ep->flags & USDF_EP_ENABLED))
800 return -FI_EOPBADSTATE;
801
802 /* prefix_sendvcan post up to iov_limit descriptors
803 */
804 return (usd_get_send_credits(ep->e.dg.ep_qp) / ep->e.dg.tx_iov_limit);
805 }
806