1 /*
2 * Copyright (c) 2016, Cisco Systems, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
33 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 /* This needs to be included for usdf.h */
38 #include "ofi.h"
39 #include "ofi_enosys.h"
40 #include "ofi_util.h"
41
42 #include "usdf.h"
43 #include "usdf_cq.h"
44 #include "usdf_wait.h"
45
46
47 /* Necessary to support top-of-file struct declarations. */
48 static int usdf_wait_wait(struct fid_wait *wait_fid, int timeout);
49 static int usdf_wait_close(struct fid *waitset);
50 static int usdf_wait_control(struct fid *fid, int command, void *arg);
51
52 static struct fi_ops_wait usdf_wait_ops = {
53 .size = sizeof(struct fi_ops_wait),
54 .wait = usdf_wait_wait,
55 };
56
57 static struct fi_ops usdf_wait_fi_ops = {
58 .size = sizeof(struct fi_ops),
59 .close = usdf_wait_close,
60 .bind = fi_no_bind,
61 .control = usdf_wait_control,
62 .ops_open = fi_no_ops_open
63 };
64
usdf_wait_trywait(struct fid * fwait)65 static int usdf_wait_trywait(struct fid *fwait)
66 {
67 struct usdf_wait *wait;
68 struct dlist_entry *item;
69 struct fid_list_entry *entry;
70 int ret = FI_SUCCESS;
71
72 wait = wait_fidtou(fwait);
73
74 dlist_foreach(&wait->list, item) {
75 entry = container_of(item, struct fid_list_entry, entry);
76
77 switch (entry->fid->fclass) {
78 case FI_CLASS_EQ:
79 continue;
80 case FI_CLASS_CQ:
81 ret = usdf_cq_trywait(entry->fid);
82 if (ret)
83 return ret;
84 break;
85 default:
86 USDF_DBG_SYS(FABRIC, "invalid fid %p\n", entry->fid);
87 return -FI_EINVAL;
88 }
89 }
90
91 return ret;
92 }
93
usdf_trywait(struct fid_fabric * fabric,struct fid ** fids,int count)94 int usdf_trywait(struct fid_fabric *fabric, struct fid **fids, int count)
95 {
96 size_t i;
97 int ret;
98
99 for (i = 0; i < count; i++) {
100 assert(fids[i]);
101
102 switch (fids[i]->fclass) {
103 case FI_CLASS_EQ:
104 continue;
105 case FI_CLASS_CQ:
106 ret = usdf_cq_trywait(fids[i]);
107 break;
108 case FI_CLASS_WAIT:
109 ret = usdf_wait_trywait(fids[i]);
110 break;
111 default:
112 USDF_DBG_SYS(FABRIC, "invalid fid\n");
113 return -FI_EINVAL;
114 }
115
116 if (ret)
117 return ret;
118 }
119
120 return FI_SUCCESS;
121 }
122
123 /* Since a domain hasn't been opened at the time of wait object creation, open a
124 * device temporarily to check for the group interrupt capability.
125 */
usdf_wait_check_support(struct usdf_fabric * fabric_priv)126 static int usdf_wait_check_support(struct usdf_fabric *fabric_priv)
127 {
128 struct usd_open_params params = {
129 .flags = UOPF_SKIP_PD_ALLOC,
130 .cmd_fd = -1,
131 .context = NULL
132 };
133 struct usd_device *dev;
134 int ret;
135
136 ret = usd_open_with_params(fabric_priv->fab_dev_attrs->uda_devname,
137 ¶ms, &dev);
138 if (ret) {
139 USDF_DBG_SYS(FABRIC,
140 "opening device to check fd support failed.\n");
141 return ret;
142 }
143
144 if (!usd_get_cap(dev, USD_CAP_GRP_INTR)) {
145 USDF_WARN_SYS(FABRIC, "FD request invalid.\n");
146 USDF_WARN_SYS(FABRIC, "group interrupts not supported.\n");
147 ret = usd_close(dev);
148 if (ret)
149 USDF_WARN_SYS(FABRIC, "closing usd device failed: %s\n",
150 strerror(ret));
151
152 return -FI_EOPNOTSUPP;
153 }
154
155 return usd_close(dev);
156 }
157
158 /* Non-static because this is exported due to being returned as a callback for
159 * fabric ops.
160 *
161 * Supporting wait objects in the usNIC provider is done using an epoll
162 * context. When fi_wait_open is called an epoll context is created using
163 * epoll_create1. This simplifies multi-CQ support and also allows us to get
164 * around a limitation of the usNIC provider. IB completion channels are opened
165 * on the domain because we have a context associated with the domain. At
166 * fi_wait_open time, we only have access to the fabric. It isn't guaranteed
167 * that a domain has been opened yet. The epoll context approach allows us to
168 * defer creating the completion channel for the CQ until CQ open time.
169 */
usdf_wait_open(struct fid_fabric * fabric,struct fi_wait_attr * attr,struct fid_wait ** waitset)170 int usdf_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
171 struct fid_wait **waitset)
172 {
173 struct usdf_wait *wait_priv;
174 struct usdf_fabric *fabric_priv;
175 ofi_epoll_t epfd;
176 int ret;
177
178 USDF_TRACE_SYS(FABRIC, "\n");
179
180 switch (attr->wait_obj) {
181 case FI_WAIT_UNSPEC:
182 case FI_WAIT_FD:
183 break;
184 default:
185 USDF_WARN_SYS(FABRIC, "unsupported wait object type\n");
186 ret = -FI_EINVAL;
187 goto error;
188 }
189
190 fabric_priv = fab_fidtou(fabric);
191 ret = usdf_wait_check_support(fabric_priv);
192 if (ret)
193 goto error;
194
195 ret = ofi_epoll_create(&epfd);
196 if (ret) {
197 USDF_WARN_SYS(FABRIC, "failed to create epoll fd[%d]\n", errno);
198 goto error;
199 }
200
201 USDF_DBG_SYS(FABRIC, "successfully created epoll fd: %d\n", epfd);
202
203 wait_priv = calloc(1, sizeof(*wait_priv));
204 if (!wait_priv) {
205 USDF_WARN_SYS(FABRIC,
206 "unable to allocate memory for usdf_wait obj");
207 ret = -FI_ENOMEM;
208 goto calloc_fail;
209 }
210
211 wait_priv->wait_fid.fid.fclass = FI_CLASS_WAIT;
212 wait_priv->wait_fid.fid.ops = &usdf_wait_fi_ops;
213 wait_priv->wait_fid.ops = &usdf_wait_ops;
214 wait_priv->wait_fid.fid.context = 0;
215 wait_priv->wait_fabric = fabric_priv;
216 wait_priv->wait_obj = attr->wait_obj;
217 wait_priv->object.epfd = epfd;
218
219 ofi_atomic_initialize32(&wait_priv->wait_refcnt, 0);
220 fastlock_init(&wait_priv->lock);
221 dlist_init(&wait_priv->list);
222
223 ofi_atomic_inc32(&wait_priv->wait_fabric->fab_refcnt);
224
225 *waitset = &wait_priv->wait_fid;
226
227 return FI_SUCCESS;
228
229 calloc_fail:
230 ofi_epoll_close(epfd);
231 error:
232 *waitset = NULL;
233 return ret;
234 }
235
236 /* Close a wait object. Make sure all resources associated with the wait object
237 * have been closed.
238 */
usdf_wait_close(struct fid * waitset)239 static int usdf_wait_close(struct fid *waitset)
240 {
241 struct usdf_wait *wait_priv;
242
243 USDF_TRACE_SYS(FABRIC, "\n");
244 if (!waitset) {
245 USDF_WARN_SYS(FABRIC, "invalid input.\n");
246 return -FI_EINVAL;
247 }
248
249 wait_priv = wait_ftou(waitset);
250
251 if (ofi_atomic_get32(&wait_priv->wait_refcnt) > 0) {
252 USDF_DBG_SYS(FABRIC,
253 "failed to close waitset with non-zero refcnt");
254 return -FI_EBUSY;
255 }
256
257 switch (wait_priv->wait_obj) {
258 case FI_WAIT_UNSPEC:
259 case FI_WAIT_FD:
260 ofi_epoll_close(wait_priv->object.epfd);
261 break;
262 default:
263 USDF_WARN_SYS(FABRIC,
264 "unsupported wait object type\n");
265 return -FI_EINVAL;
266 }
267
268 ofi_atomic_dec32(&wait_priv->wait_fabric->fab_refcnt);
269 free(wait_priv);
270
271 return FI_SUCCESS;
272 }
273
usdf_wait_wait(struct fid_wait * fwait,int timeout)274 static int usdf_wait_wait(struct fid_wait *fwait, int timeout)
275 {
276 struct usdf_wait *wait;
277 void *context;
278 int ret = FI_SUCCESS;
279 int nevents;
280
281 USDF_TRACE_SYS(FABRIC, "\n");
282 wait = wait_ftou(fwait);
283
284 ret = usdf_wait_trywait(&fwait->fid);
285 if (ret) {
286 if (ret == -FI_EAGAIN)
287 return FI_SUCCESS;
288
289 return ret;
290 }
291
292 nevents = ofi_epoll_wait(wait->object.epfd, &context, 1, timeout);
293 if (nevents == 0) {
294 ret = -FI_ETIMEDOUT;
295 } else if (nevents < 0) {
296 USDF_DBG_SYS(FABRIC, "epoll wait failed\n");
297 ret = nevents;
298 }
299
300 return ret;
301 }
302
usdf_wait_get_wait(struct usdf_wait * wait_priv,void * arg)303 static int usdf_wait_get_wait(struct usdf_wait *wait_priv, void *arg)
304 {
305 USDF_TRACE_SYS(FABRIC, "\n");
306
307 if (!arg || !wait_priv) {
308 USDF_WARN_SYS(FABRIC, "invalid input\n");
309 return -FI_EINVAL;
310 }
311
312 switch (wait_priv->wait_obj) {
313 case FI_WAIT_UNSPEC:
314 case FI_WAIT_FD:
315 #ifdef HAVE_EPOLL
316 *(int *) arg = wait_priv->object.epfd;
317 #else
318 return -FI_ENOSYS;
319 #endif
320 break;
321 default:
322 USDF_DBG_SYS(FABRIC, "unsupported wait type\n");
323 return -FI_EINVAL;
324 }
325
326 return FI_SUCCESS;
327 }
328
usdf_wait_control(struct fid * fid,int command,void * arg)329 static int usdf_wait_control(struct fid *fid, int command, void *arg)
330 {
331 struct usdf_wait *wait_priv;
332
333 USDF_TRACE_SYS(FABRIC, "\n");
334
335 wait_priv = container_of(fid, struct usdf_wait, wait_fid.fid);
336
337 switch (command) {
338 case FI_GETWAIT:
339 break;
340 default:
341 USDF_DBG_SYS(FABRIC, "unsupported control command\n");
342 return -FI_EINVAL;
343 }
344
345 return usdf_wait_get_wait(wait_priv, arg);
346 }
347