1 /*
2 * Copyright (c) 2014-2017, Cisco Systems, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
33 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #include "config.h"
38
39 #include <asm/types.h>
40 #include <assert.h>
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <netinet/in.h>
44 #include <poll.h>
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <string.h>
48
49 #include <rdma/fabric.h>
50 #include <rdma/fi_cm.h>
51 #include <rdma/fi_domain.h>
52 #include <rdma/fi_endpoint.h>
53 #include <rdma/fi_rma.h>
54 #include <rdma/fi_errno.h>
55 #include "ofi.h"
56 #include "ofi_enosys.h"
57
58 #include "usnic_direct.h"
59 #include "usdf.h"
60
61 static
usdf_dereg_mr(fid_t fid)62 int usdf_dereg_mr(fid_t fid)
63 {
64 struct usdf_mr *mr;
65 int ret;
66
67 mr = container_of(fid, struct usdf_mr, mr_fid.fid);
68 ret = usd_dereg_mr(mr->mr_mr);
69 if (ret == 0) {
70 free(mr);
71 }
72 return ret;
73 }
74
75 static struct fi_ops usdf_mr_ops = {
76 .size = sizeof(struct fi_ops),
77 .close = usdf_dereg_mr,
78 .bind = fi_no_bind,
79 .control = fi_no_control,
80 .ops_open = fi_no_ops_open,
81 };
82
83 int
usdf_reg_mr(struct fid * fid,const void * buf,size_t len,uint64_t access,uint64_t offset,uint64_t requested_key,uint64_t flags,struct fid_mr ** mr_o,void * context)84 usdf_reg_mr(struct fid *fid, const void *buf, size_t len,
85 uint64_t access, uint64_t offset, uint64_t requested_key,
86 uint64_t flags, struct fid_mr **mr_o, void *context)
87 {
88 struct usdf_mr *mr;
89 struct usdf_domain *udp;
90 int ret;
91 struct fid_domain *domain;
92
93 if (flags != 0) {
94 return -FI_EBADFLAGS;
95 }
96
97 if (fid->fclass != FI_CLASS_DOMAIN) {
98 USDF_DBG("memory registration only supported "
99 "for struct fid_domain\n");
100 return -FI_EINVAL;
101 }
102 domain = container_of(fid, struct fid_domain, fid);
103
104 mr = calloc(1, sizeof *mr);
105 if (mr == NULL) {
106 return -FI_ENOMEM;
107 }
108
109 mr->mr_fid.fid.fclass = FI_CLASS_MR;
110 mr->mr_fid.fid.context = context;
111 mr->mr_fid.fid.ops = &usdf_mr_ops;
112
113 udp = container_of(domain, struct usdf_domain, dom_fid.fid);
114 ret = usd_reg_mr(udp->dom_dev, (void *) buf, len, &mr->mr_mr);
115 if (ret != 0) {
116 goto fail;
117 }
118
119 *mr_o = &mr->mr_fid;
120 return 0;
121
122 fail:
123 free(mr);
124 return ret;
125 }
126
127 /* We dont have proper support for regv and regattr. This is just
128 * a simple mapping to usdf_reg_mr. We can do this because we forced
129 * mr_iov_limit = 1 (made this mapping possible) by default.
130 */
usdf_regv_mr(struct fid * fid,const struct iovec * iov,size_t count,uint64_t access,uint64_t offset,uint64_t requested_key,uint64_t flags,struct fid_mr ** mr,void * context)131 int usdf_regv_mr(struct fid *fid, const struct iovec *iov,
132 size_t count, uint64_t access,
133 uint64_t offset, uint64_t requested_key,
134 uint64_t flags, struct fid_mr **mr, void *context)
135 {
136 if (count > USDF_MR_IOV_LIMIT) {
137 USDF_DBG_SYS(DOMAIN, "usnic provider only support 1 iov.\n");
138 return -FI_EINVAL;
139 }
140
141 return usdf_reg_mr(fid, iov[0].iov_base, iov[0].iov_len, access,
142 offset, requested_key, flags, mr, context);
143 }
144
usdf_regattr(struct fid * fid,const struct fi_mr_attr * attr,uint64_t flags,struct fid_mr ** mr)145 int usdf_regattr(struct fid *fid, const struct fi_mr_attr *attr,
146 uint64_t flags, struct fid_mr **mr)
147 {
148 if (attr->iov_count > USDF_MR_IOV_LIMIT) {
149 USDF_DBG_SYS(DOMAIN, "usnic provider only support 1 iov.\n");
150 return -FI_EINVAL;
151 }
152
153 return usdf_reg_mr(fid, attr->mr_iov[0].iov_base,
154 attr->mr_iov[0].iov_len,
155 attr->access,
156 attr->offset,
157 attr->requested_key,
158 flags, mr, attr->context);
159 }
160