1 /*
2 * Copyright (c) 2016 Intel Corporation, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <stdlib.h>
34 #include <string.h>
35
36 #include <ofi_enosys.h>
37 #include <ofi_util.h>
38 #include <ofi_coll.h>
39
ofi_ep_bind_cq(struct util_ep * ep,struct util_cq * cq,uint64_t flags)40 int ofi_ep_bind_cq(struct util_ep *ep, struct util_cq *cq, uint64_t flags)
41 {
42 int ret;
43
44 ret = ofi_check_bind_cq_flags(ep, cq, flags);
45 if (ret)
46 return ret;
47
48 if (flags & FI_TRANSMIT) {
49 ep->tx_cq = cq;
50 if (!(flags & FI_SELECTIVE_COMPLETION)) {
51 ep->tx_op_flags |= FI_COMPLETION;
52 ep->tx_msg_flags = FI_COMPLETION;
53 }
54 ofi_atomic_inc32(&cq->ref);
55 }
56
57 if (flags & FI_RECV) {
58 ep->rx_cq = cq;
59 if (!(flags & FI_SELECTIVE_COMPLETION)) {
60 ep->rx_op_flags |= FI_COMPLETION;
61 ep->rx_msg_flags = FI_COMPLETION;
62 }
63 ofi_atomic_inc32(&cq->ref);
64 }
65
66 if (flags & (FI_TRANSMIT | FI_RECV)) {
67 return fid_list_insert(&cq->ep_list,
68 &cq->ep_list_lock,
69 &ep->ep_fid.fid);
70 }
71
72 return FI_SUCCESS;
73 }
74
ofi_ep_bind_eq(struct util_ep * ep,struct util_eq * eq)75 int ofi_ep_bind_eq(struct util_ep *ep, struct util_eq *eq)
76 {
77 if (ep->eq)
78 ofi_atomic_dec32(&ep->eq->ref);
79
80 ep->eq = eq;
81 ofi_atomic_inc32(&eq->ref);
82 return 0;
83 }
84
ofi_ep_bind_av(struct util_ep * util_ep,struct util_av * av)85 int ofi_ep_bind_av(struct util_ep *util_ep, struct util_av *av)
86 {
87 if (util_ep->av) {
88 FI_WARN(util_ep->av->prov, FI_LOG_EP_CTRL,
89 "duplicate AV binding\n");
90 return -FI_EINVAL;
91 }
92 util_ep->av = av;
93 ofi_atomic_inc32(&av->ref);
94
95 fastlock_acquire(&av->ep_list_lock);
96 dlist_insert_tail(&util_ep->av_entry, &av->ep_list);
97 fastlock_release(&av->ep_list_lock);
98
99 return 0;
100 }
101
ofi_ep_bind_cntr(struct util_ep * ep,struct util_cntr * cntr,uint64_t flags)102 int ofi_ep_bind_cntr(struct util_ep *ep, struct util_cntr *cntr, uint64_t flags)
103 {
104 if (flags & ~(FI_TRANSMIT | FI_RECV | FI_READ | FI_WRITE |
105 FI_REMOTE_READ | FI_REMOTE_WRITE)) {
106 FI_WARN(ep->domain->fabric->prov, FI_LOG_EP_CTRL,
107 "Unsupported bind flags\n");
108 return -FI_EBADFLAGS;
109 }
110
111 if (((flags & FI_TRANSMIT) && ep->tx_cntr) ||
112 ((flags & FI_RECV) && ep->rx_cntr) ||
113 ((flags & FI_READ) && ep->rd_cntr) ||
114 ((flags & FI_WRITE) && ep->wr_cntr) ||
115 ((flags & FI_REMOTE_READ) && ep->rem_rd_cntr) ||
116 ((flags & FI_REMOTE_WRITE) && ep->rem_wr_cntr)) {
117 FI_WARN(ep->domain->fabric->prov, FI_LOG_EP_CTRL,
118 "Duplicate counter binding\n");
119 return -FI_EINVAL;
120 }
121
122 if (flags & FI_TRANSMIT) {
123 ep->tx_cntr = cntr;
124 ep->tx_cntr_inc = ofi_cntr_inc;
125 ofi_atomic_inc32(&cntr->ref);
126 }
127
128 if (flags & FI_RECV) {
129 ep->rx_cntr = cntr;
130 ep->rx_cntr_inc = ofi_cntr_inc;
131 ofi_atomic_inc32(&cntr->ref);
132 }
133
134 if (flags & FI_READ) {
135 ep->rd_cntr = cntr;
136 ep->rd_cntr_inc = ofi_cntr_inc;
137 ofi_atomic_inc32(&cntr->ref);
138 }
139
140 if (flags & FI_WRITE) {
141 ep->wr_cntr = cntr;
142 ep->wr_cntr_inc = ofi_cntr_inc;
143 ofi_atomic_inc32(&cntr->ref);
144 }
145
146 if (flags & FI_REMOTE_READ) {
147 ep->rem_rd_cntr = cntr;
148 ep->rem_rd_cntr_inc = ofi_cntr_inc;
149 ofi_atomic_inc32(&cntr->ref);
150 }
151
152 if (flags & FI_REMOTE_WRITE) {
153 ep->rem_wr_cntr = cntr;
154 ep->rem_wr_cntr_inc = ofi_cntr_inc;
155 ofi_atomic_inc32(&cntr->ref);
156 }
157
158 ep->flags |= OFI_CNTR_ENABLED;
159
160 return fid_list_insert(&cntr->ep_list, &cntr->ep_list_lock,
161 &ep->ep_fid.fid);
162 }
163
ofi_ep_bind(struct util_ep * util_ep,struct fid * fid,uint64_t flags)164 int ofi_ep_bind(struct util_ep *util_ep, struct fid *fid, uint64_t flags)
165 {
166 int ret;
167 struct util_av *av;
168 struct util_cq *cq;
169 struct util_eq *eq;
170 struct util_cntr *cntr;
171
172 ret = ofi_ep_bind_valid(util_ep->domain->prov, fid, flags);
173 if (ret)
174 return ret;
175
176 switch (fid->fclass) {
177 case FI_CLASS_CQ:
178 cq = container_of(fid, struct util_cq, cq_fid.fid);
179 return ofi_ep_bind_cq(util_ep, cq, flags);
180 case FI_CLASS_EQ:
181 eq = container_of(fid, struct util_eq, eq_fid.fid);
182 return ofi_ep_bind_eq(util_ep, eq);
183 case FI_CLASS_AV:
184 av = container_of(fid, struct util_av, av_fid.fid);
185 return ofi_ep_bind_av(util_ep, av);
186 case FI_CLASS_CNTR:
187 cntr = container_of(fid, struct util_cntr, cntr_fid.fid);
188 return ofi_ep_bind_cntr(util_ep, cntr, flags);
189 }
190
191 return -FI_EINVAL;
192 }
193
util_coll_init_cid_mask(struct bitmask * mask)194 static inline int util_coll_init_cid_mask(struct bitmask *mask)
195 {
196 int err = ofi_bitmask_create(mask, OFI_MAX_GROUP_ID);
197 if (err)
198 return err;
199
200 ofi_bitmask_set_all(mask);
201
202 /* reserving the first bit in context id to whole av set */
203 ofi_bitmask_unset(mask, OFI_WORLD_GROUP_ID);
204
205 return FI_SUCCESS;
206 }
207
ofi_endpoint_init(struct fid_domain * domain,const struct util_prov * util_prov,struct fi_info * info,struct util_ep * ep,void * context,ofi_ep_progress_func progress)208 int ofi_endpoint_init(struct fid_domain *domain, const struct util_prov *util_prov,
209 struct fi_info *info, struct util_ep *ep, void *context,
210 ofi_ep_progress_func progress)
211 {
212 struct util_domain *util_domain;
213 int ret;
214
215 util_domain = container_of(domain, struct util_domain, domain_fid);
216
217 if (!info || !info->ep_attr || !info->rx_attr || !info->tx_attr)
218 return -FI_EINVAL;
219
220 ret = ofi_prov_check_info(util_prov,
221 util_domain->fabric->fabric_fid.api_version,
222 info);
223 if (ret)
224 return ret;
225
226 ep->ep_fid.fid.fclass = FI_CLASS_EP;
227 ep->ep_fid.fid.context = context;
228 ep->domain = util_domain;
229 ep->caps = info->caps;
230 ep->flags = 0;
231 ep->progress = progress;
232 ep->tx_op_flags = info->tx_attr->op_flags;
233 ep->rx_op_flags = info->rx_attr->op_flags;
234 ep->tx_msg_flags = 0;
235 ep->rx_msg_flags = 0;
236 ep->inject_op_flags =
237 ((info->tx_attr->op_flags &
238 ~(FI_COMPLETION | FI_INJECT_COMPLETE |
239 FI_TRANSMIT_COMPLETE | FI_DELIVERY_COMPLETE)) | FI_INJECT);
240 ep->tx_cntr_inc = ofi_cntr_inc_noop;
241 ep->rx_cntr_inc = ofi_cntr_inc_noop;
242 ep->rd_cntr_inc = ofi_cntr_inc_noop;
243 ep->wr_cntr_inc = ofi_cntr_inc_noop;
244 ep->rem_rd_cntr_inc = ofi_cntr_inc_noop;
245 ep->rem_wr_cntr_inc = ofi_cntr_inc_noop;
246 ep->type = info->ep_attr->type;
247 ofi_atomic_inc32(&util_domain->ref);
248 if (util_domain->eq)
249 ofi_ep_bind_eq(ep, util_domain->eq);
250 fastlock_init(&ep->lock);
251 if (ep->domain->threading != FI_THREAD_SAFE) {
252 ep->lock_acquire = ofi_fastlock_acquire_noop;
253 ep->lock_release = ofi_fastlock_release_noop;
254 } else {
255 ep->lock_acquire = ofi_fastlock_acquire;
256 ep->lock_release = ofi_fastlock_release;
257 }
258 if (ep->caps & FI_COLLECTIVE) {
259 ep->coll_cid_mask = calloc(1, sizeof(*ep->coll_cid_mask));
260 if (!ep->coll_cid_mask)
261 return -FI_ENOMEM;
262 util_coll_init_cid_mask(ep->coll_cid_mask);
263 } else {
264 ep->coll_cid_mask = NULL;
265 }
266 slist_init(&ep->coll_ready_queue);
267 return 0;
268 }
269
ofi_endpoint_close(struct util_ep * util_ep)270 int ofi_endpoint_close(struct util_ep *util_ep)
271 {
272 if (util_ep->tx_cq) {
273 fid_list_remove(&util_ep->tx_cq->ep_list,
274 &util_ep->tx_cq->ep_list_lock,
275 &util_ep->ep_fid.fid);
276 ofi_atomic_dec32(&util_ep->tx_cq->ref);
277 }
278
279 if (util_ep->rx_cq) {
280 fid_list_remove(&util_ep->rx_cq->ep_list,
281 &util_ep->rx_cq->ep_list_lock,
282 &util_ep->ep_fid.fid);
283 ofi_atomic_dec32(&util_ep->rx_cq->ref);
284 }
285
286 if (util_ep->rx_cntr) {
287 fid_list_remove(&util_ep->rx_cntr->ep_list,
288 &util_ep->rx_cntr->ep_list_lock,
289 &util_ep->ep_fid.fid);
290 ofi_atomic_dec32(&util_ep->rx_cntr->ref);
291 }
292
293 if (util_ep->tx_cntr) {
294 fid_list_remove(&util_ep->tx_cntr->ep_list,
295 &util_ep->tx_cntr->ep_list_lock,
296 &util_ep->ep_fid.fid);
297 ofi_atomic_dec32(&util_ep->tx_cntr->ref);
298 }
299
300 if (util_ep->rd_cntr) {
301 fid_list_remove(&util_ep->rd_cntr->ep_list,
302 &util_ep->rd_cntr->ep_list_lock,
303 &util_ep->ep_fid.fid);
304 ofi_atomic_dec32(&util_ep->rd_cntr->ref);
305 }
306
307 if (util_ep->wr_cntr) {
308 fid_list_remove(&util_ep->wr_cntr->ep_list,
309 &util_ep->wr_cntr->ep_list_lock,
310 &util_ep->ep_fid.fid);
311 ofi_atomic_dec32(&util_ep->wr_cntr->ref);
312 }
313
314 if (util_ep->rem_rd_cntr) {
315 fid_list_remove(&util_ep->rem_rd_cntr->ep_list,
316 &util_ep->rem_rd_cntr->ep_list_lock,
317 &util_ep->ep_fid.fid);
318 ofi_atomic_dec32(&util_ep->rem_rd_cntr->ref);
319 }
320
321 if (util_ep->rem_wr_cntr) {
322 fid_list_remove(&util_ep->rem_wr_cntr->ep_list,
323 &util_ep->rem_wr_cntr->ep_list_lock,
324 &util_ep->ep_fid.fid);
325 ofi_atomic_dec32(&util_ep->rem_wr_cntr->ref);
326 }
327
328 if (util_ep->av) {
329 fastlock_acquire(&util_ep->av->ep_list_lock);
330 dlist_remove(&util_ep->av_entry);
331 fastlock_release(&util_ep->av->ep_list_lock);
332
333 ofi_atomic_dec32(&util_ep->av->ref);
334 }
335
336 if (util_ep->coll_cid_mask) {
337 ofi_bitmask_free(util_ep->coll_cid_mask);
338 free(util_ep->coll_cid_mask);
339 }
340
341 if (util_ep->eq)
342 ofi_atomic_dec32(&util_ep->eq->ref);
343 ofi_atomic_dec32(&util_ep->domain->ref);
344 fastlock_destroy(&util_ep->lock);
345 return 0;
346 }
347