1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED.
3 *
4 * Copyright (C) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED.
5 * See file LICENSE for terms.
6 */
7
8 #ifdef HAVE_CONFIG_H
9 # include "config.h"
10 #endif
11
12 #include "uct_iface.h"
13 #include "uct_cm.h"
14
15 #include <uct/api/uct.h>
16 #include <ucs/async/async.h>
17 #include <ucs/sys/string.h>
18 #include <ucs/time/time.h>
19 #include <ucs/debug/debug.h>
20
21
22 #ifdef ENABLE_STATS
23 static ucs_stats_class_t uct_ep_stats_class = {
24 .name = "uct_ep",
25 .num_counters = UCT_EP_STAT_LAST,
26 .counter_names = {
27 [UCT_EP_STAT_AM] = "am",
28 [UCT_EP_STAT_PUT] = "put",
29 [UCT_EP_STAT_GET] = "get",
30 [UCT_EP_STAT_ATOMIC] = "atomic",
31 #if IBV_HW_TM
32 [UCT_EP_STAT_TAG] = "tag",
33 #endif
34 [UCT_EP_STAT_BYTES_SHORT] = "bytes_short",
35 [UCT_EP_STAT_BYTES_BCOPY] = "bytes_bcopy",
36 [UCT_EP_STAT_BYTES_ZCOPY] = "bytes_zcopy",
37 [UCT_EP_STAT_NO_RES] = "no_res",
38 [UCT_EP_STAT_FLUSH] = "flush",
39 [UCT_EP_STAT_FLUSH_WAIT] = "flush_wait",
40 [UCT_EP_STAT_PENDING] = "pending",
41 [UCT_EP_STAT_FENCE] = "fence"
42 }
43 };
44
45 static ucs_stats_class_t uct_iface_stats_class = {
46 .name = "uct_iface",
47 .num_counters = UCT_IFACE_STAT_LAST,
48 .counter_names = {
49 [UCT_IFACE_STAT_RX_AM] = "rx_am",
50 [UCT_IFACE_STAT_RX_AM_BYTES] = "rx_am_bytes",
51 [UCT_IFACE_STAT_TX_NO_DESC] = "tx_no_desc",
52 [UCT_IFACE_STAT_FLUSH] = "flush",
53 [UCT_IFACE_STAT_FLUSH_WAIT] = "flush_wait",
54 [UCT_IFACE_STAT_FENCE] = "fence"
55 }
56 };
57 #endif
58
59
uct_iface_stub_am_handler(void * arg,void * data,size_t length,unsigned flags)60 static ucs_status_t uct_iface_stub_am_handler(void *arg, void *data,
61 size_t length, unsigned flags)
62 {
63 const size_t dump_len = 64;
64 uint8_t id = (uintptr_t)arg;
65 char dump_str[(dump_len * 4) + 1]; /* 1234:5678\n\0 */
66
67 ucs_warn("got active message id %d, but no handler installed", id);
68 ucs_warn("payload %zu of %zu bytes:\n%s", ucs_min(length, dump_len), length,
69 ucs_str_dump_hex(data, ucs_min(length, dump_len),
70 dump_str, sizeof(dump_str), 16));
71 ucs_log_print_backtrace(UCS_LOG_LEVEL_WARN);
72 return UCS_OK;
73 }
74
uct_iface_set_stub_am_handler(uct_base_iface_t * iface,uint8_t id)75 static void uct_iface_set_stub_am_handler(uct_base_iface_t *iface, uint8_t id)
76 {
77 iface->am[id].cb = uct_iface_stub_am_handler;
78 iface->am[id].arg = (void*)(uintptr_t)id;
79 iface->am[id].flags = UCT_CB_FLAG_ASYNC;
80 }
81
uct_iface_set_am_handler(uct_iface_h tl_iface,uint8_t id,uct_am_callback_t cb,void * arg,uint32_t flags)82 ucs_status_t uct_iface_set_am_handler(uct_iface_h tl_iface, uint8_t id,
83 uct_am_callback_t cb, void *arg,
84 uint32_t flags)
85 {
86 uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t);
87 ucs_status_t status;
88 uct_iface_attr_t attr;
89
90 if (id >= UCT_AM_ID_MAX) {
91 ucs_error("active message id out-of-range (got: %d max: %d)", id,
92 (int)UCT_AM_ID_MAX);
93 return UCS_ERR_INVALID_PARAM;
94 }
95
96 if (cb == NULL) {
97 uct_iface_set_stub_am_handler(iface, id);
98 return UCS_OK;
99 }
100
101 status = uct_iface_query(tl_iface, &attr);
102 if (status != UCS_OK) {
103 return status;
104 }
105
106 UCT_CB_FLAGS_CHECK(flags);
107
108 /* If user wants a synchronous callback, it must be supported, or the
109 * callback could be called from another thread.
110 */
111 if (!(flags & UCT_CB_FLAG_ASYNC) && !(attr.cap.flags & UCT_IFACE_FLAG_CB_SYNC)) {
112 ucs_error("Synchronous callback requested, but not supported");
113 return UCS_ERR_INVALID_PARAM;
114 }
115
116 iface->am[id].cb = cb;
117 iface->am[id].arg = arg;
118 iface->am[id].flags = flags;
119 return UCS_OK;
120 }
121
uct_iface_set_am_tracer(uct_iface_h tl_iface,uct_am_tracer_t tracer,void * arg)122 ucs_status_t uct_iface_set_am_tracer(uct_iface_h tl_iface, uct_am_tracer_t tracer,
123 void *arg)
124 {
125 uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t);
126
127 iface->am_tracer = tracer;
128 iface->am_tracer_arg = arg;
129 return UCS_OK;
130 }
131
uct_iface_dump_am(uct_base_iface_t * iface,uct_am_trace_type_t type,uint8_t id,const void * data,size_t length,char * buffer,size_t max)132 void uct_iface_dump_am(uct_base_iface_t *iface, uct_am_trace_type_t type,
133 uint8_t id, const void *data, size_t length,
134 char *buffer, size_t max)
135 {
136 if (iface->am_tracer != NULL) {
137 iface->am_tracer(iface->am_tracer_arg, type, id, data, length, buffer, max);
138 }
139 }
140
uct_iface_mpool_empty_warn(uct_base_iface_t * iface,ucs_mpool_t * mp)141 void uct_iface_mpool_empty_warn(uct_base_iface_t *iface, ucs_mpool_t *mp)
142 {
143 static ucs_time_t warn_time = 0;
144 ucs_time_t now = ucs_get_time();
145
146 /* Limit the rate of warning to once in 30 seconds. This gives reasonable
147 * indication about a deadlock without flooding with warnings messages. */
148 if (warn_time == 0) {
149 warn_time = now;
150 }
151 if (now - warn_time > ucs_time_from_sec(30)) {
152 ucs_warn("Memory pool %s is empty", ucs_mpool_name(mp));
153 warn_time = now;
154 }
155 }
156
uct_iface_query(uct_iface_h iface,uct_iface_attr_t * iface_attr)157 ucs_status_t uct_iface_query(uct_iface_h iface, uct_iface_attr_t *iface_attr)
158 {
159 return iface->ops.iface_query(iface, iface_attr);
160 }
161
uct_iface_get_device_address(uct_iface_h iface,uct_device_addr_t * addr)162 ucs_status_t uct_iface_get_device_address(uct_iface_h iface, uct_device_addr_t *addr)
163 {
164 return iface->ops.iface_get_device_address(iface, addr);
165 }
166
uct_iface_get_address(uct_iface_h iface,uct_iface_addr_t * addr)167 ucs_status_t uct_iface_get_address(uct_iface_h iface, uct_iface_addr_t *addr)
168 {
169 return iface->ops.iface_get_address(iface, addr);
170 }
171
uct_iface_is_reachable(const uct_iface_h iface,const uct_device_addr_t * dev_addr,const uct_iface_addr_t * iface_addr)172 int uct_iface_is_reachable(const uct_iface_h iface, const uct_device_addr_t *dev_addr,
173 const uct_iface_addr_t *iface_addr)
174 {
175 return iface->ops.iface_is_reachable(iface, dev_addr, iface_addr);
176 }
177
uct_ep_check(const uct_ep_h ep,unsigned flags,uct_completion_t * comp)178 ucs_status_t uct_ep_check(const uct_ep_h ep, unsigned flags,
179 uct_completion_t *comp)
180 {
181 return ep->iface->ops.ep_check(ep, flags, comp);
182 }
183
uct_iface_event_fd_get(uct_iface_h iface,int * fd_p)184 ucs_status_t uct_iface_event_fd_get(uct_iface_h iface, int *fd_p)
185 {
186 return iface->ops.iface_event_fd_get(iface, fd_p);
187 }
188
uct_iface_event_arm(uct_iface_h iface,unsigned events)189 ucs_status_t uct_iface_event_arm(uct_iface_h iface, unsigned events)
190 {
191 return iface->ops.iface_event_arm(iface, events);
192 }
193
uct_iface_close(uct_iface_h iface)194 void uct_iface_close(uct_iface_h iface)
195 {
196 iface->ops.iface_close(iface);
197 }
198
uct_base_iface_progress_enable(uct_iface_h tl_iface,unsigned flags)199 void uct_base_iface_progress_enable(uct_iface_h tl_iface, unsigned flags)
200 {
201 uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t);
202 uct_base_iface_progress_enable_cb(iface,
203 (ucs_callback_t)iface->super.ops.iface_progress,
204 flags);
205 }
206
uct_base_iface_progress_enable_cb(uct_base_iface_t * iface,ucs_callback_t cb,unsigned flags)207 void uct_base_iface_progress_enable_cb(uct_base_iface_t *iface,
208 ucs_callback_t cb, unsigned flags)
209 {
210 uct_priv_worker_t *worker = iface->worker;
211 unsigned thread_safe;
212
213 UCS_ASYNC_BLOCK(worker->async);
214
215 thread_safe = flags & UCT_PROGRESS_THREAD_SAFE;
216 flags &= ~UCT_PROGRESS_THREAD_SAFE;
217
218 /* Add callback only if previous flags are 0 and new flags != 0 */
219 if ((!iface->progress_flags && flags) &&
220 (iface->prog.id == UCS_CALLBACKQ_ID_NULL)) {
221 if (thread_safe) {
222 iface->prog.id = ucs_callbackq_add_safe(&worker->super.progress_q,
223 cb, iface,
224 UCS_CALLBACKQ_FLAG_FAST);
225 } else {
226 iface->prog.id = ucs_callbackq_add(&worker->super.progress_q, cb,
227 iface, UCS_CALLBACKQ_FLAG_FAST);
228 }
229 }
230 iface->progress_flags |= flags;
231
232 UCS_ASYNC_UNBLOCK(worker->async);
233 }
234
uct_base_iface_progress_disable(uct_iface_h tl_iface,unsigned flags)235 void uct_base_iface_progress_disable(uct_iface_h tl_iface, unsigned flags)
236 {
237 uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t);
238 uct_priv_worker_t *worker = iface->worker;
239 unsigned thread_safe;
240
241 UCS_ASYNC_BLOCK(worker->async);
242
243 thread_safe = flags & UCT_PROGRESS_THREAD_SAFE;
244 flags &= ~UCT_PROGRESS_THREAD_SAFE;
245
246 /* Remove callback only if previous flags != 0, and removing the given
247 * flags makes it become 0.
248 */
249 if ((iface->progress_flags && !(iface->progress_flags & ~flags)) &&
250 (iface->prog.id != UCS_CALLBACKQ_ID_NULL)) {
251 if (thread_safe) {
252 ucs_callbackq_remove_safe(&worker->super.progress_q, iface->prog.id);
253 } else {
254 ucs_callbackq_remove(&worker->super.progress_q, iface->prog.id);
255 }
256 iface->prog.id = UCS_CALLBACKQ_ID_NULL;
257 }
258 iface->progress_flags &= ~flags;
259
260 UCS_ASYNC_UNBLOCK(worker->async);
261 }
262
uct_base_iface_flush(uct_iface_h tl_iface,unsigned flags,uct_completion_t * comp)263 ucs_status_t uct_base_iface_flush(uct_iface_h tl_iface, unsigned flags,
264 uct_completion_t *comp)
265 {
266 UCT_TL_IFACE_STAT_FLUSH(ucs_derived_of(tl_iface, uct_base_iface_t));
267 return UCS_OK;
268 }
269
uct_base_iface_fence(uct_iface_h tl_iface,unsigned flags)270 ucs_status_t uct_base_iface_fence(uct_iface_h tl_iface, unsigned flags)
271 {
272 UCT_TL_IFACE_STAT_FENCE(ucs_derived_of(tl_iface, uct_base_iface_t));
273 return UCS_OK;
274 }
275
uct_base_ep_flush(uct_ep_h tl_ep,unsigned flags,uct_completion_t * comp)276 ucs_status_t uct_base_ep_flush(uct_ep_h tl_ep, unsigned flags,
277 uct_completion_t *comp)
278 {
279 UCT_TL_EP_STAT_FLUSH(ucs_derived_of(tl_ep, uct_base_ep_t));
280 return UCS_OK;
281 }
282
uct_base_ep_fence(uct_ep_h tl_ep,unsigned flags)283 ucs_status_t uct_base_ep_fence(uct_ep_h tl_ep, unsigned flags)
284 {
285 UCT_TL_EP_STAT_FENCE(ucs_derived_of(tl_ep, uct_base_ep_t));
286 return UCS_OK;
287 }
288
uct_ep_failed_purge_cb(uct_pending_req_t * self,void * arg)289 static void uct_ep_failed_purge_cb(uct_pending_req_t *self, void *arg)
290 {
291 uct_pending_req_queue_push((ucs_queue_head_t*)arg, self);
292 }
293
uct_ep_failed_purge(uct_ep_h tl_ep,uct_pending_purge_callback_t cb,void * arg)294 static void uct_ep_failed_purge(uct_ep_h tl_ep, uct_pending_purge_callback_t cb,
295 void *arg)
296 {
297 uct_failed_iface_t *iface = ucs_derived_of(tl_ep->iface,
298 uct_failed_iface_t);
299 uct_pending_req_t *req;
300
301 ucs_queue_for_each_extract(req, &iface->pend_q, priv, 1) {
302 if (cb != NULL) {
303 cb(req, arg);
304 } else {
305 ucs_warn("ep=%p cancelling user pending request %p", tl_ep, req);
306 }
307 }
308 }
309
uct_ep_failed_destroy(uct_ep_h tl_ep)310 static void uct_ep_failed_destroy(uct_ep_h tl_ep)
311 {
312 /* Warn user if some pending reqs left*/
313 uct_ep_failed_purge (tl_ep, NULL, NULL);
314
315 ucs_free(tl_ep->iface);
316 ucs_free(tl_ep);
317 }
318
uct_set_ep_failed(ucs_class_t * cls,uct_ep_h tl_ep,uct_iface_h tl_iface,ucs_status_t status)319 ucs_status_t uct_set_ep_failed(ucs_class_t *cls, uct_ep_h tl_ep,
320 uct_iface_h tl_iface, ucs_status_t status)
321 {
322 uct_failed_iface_t *f_iface;
323 uct_iface_ops_t *ops;
324 uct_base_iface_t *iface = ucs_derived_of(tl_iface, uct_base_iface_t);
325
326 ucs_debug("set ep %p to failed state", tl_ep);
327
328 /* TBD: consider allocating one instance per interface
329 * rather than for each endpoint */
330 f_iface = ucs_malloc(sizeof(*f_iface), "failed iface");
331 if (f_iface == NULL) {
332 ucs_error("Could not create failed iface (nomem)");
333 return status;
334 }
335
336 ucs_queue_head_init(&f_iface->pend_q);
337 ops = &f_iface->super.ops;
338
339 /* Move all pending requests to the queue.
340 * Failed ep will use that queue for purge. */
341 uct_ep_pending_purge(tl_ep, uct_ep_failed_purge_cb, &f_iface->pend_q);
342
343 ops->ep_put_short = (uct_ep_put_short_func_t)ucs_empty_function_return_ep_timeout;
344 ops->ep_put_bcopy = (uct_ep_put_bcopy_func_t)ucs_empty_function_return_bc_ep_timeout;
345 ops->ep_put_zcopy = (uct_ep_put_zcopy_func_t)ucs_empty_function_return_ep_timeout;
346 ops->ep_get_short = (uct_ep_get_short_func_t)ucs_empty_function_return_ep_timeout;
347 ops->ep_get_bcopy = (uct_ep_get_bcopy_func_t)ucs_empty_function_return_ep_timeout;
348 ops->ep_get_zcopy = (uct_ep_get_zcopy_func_t)ucs_empty_function_return_ep_timeout;
349 ops->ep_am_short = (uct_ep_am_short_func_t)ucs_empty_function_return_ep_timeout;
350 ops->ep_am_bcopy = (uct_ep_am_bcopy_func_t)ucs_empty_function_return_bc_ep_timeout;
351 ops->ep_am_zcopy = (uct_ep_am_zcopy_func_t)ucs_empty_function_return_ep_timeout;
352 ops->ep_atomic_cswap64 = (uct_ep_atomic_cswap64_func_t)ucs_empty_function_return_ep_timeout;
353 ops->ep_atomic_cswap32 = (uct_ep_atomic_cswap32_func_t)ucs_empty_function_return_ep_timeout;
354 ops->ep_atomic64_post = (uct_ep_atomic64_post_func_t)ucs_empty_function_return_ep_timeout;
355 ops->ep_atomic32_post = (uct_ep_atomic32_post_func_t)ucs_empty_function_return_ep_timeout;
356 ops->ep_atomic64_fetch = (uct_ep_atomic64_fetch_func_t)ucs_empty_function_return_ep_timeout;
357 ops->ep_atomic32_fetch = (uct_ep_atomic32_fetch_func_t)ucs_empty_function_return_ep_timeout;
358 ops->ep_tag_eager_short = (uct_ep_tag_eager_short_func_t)ucs_empty_function_return_ep_timeout;
359 ops->ep_tag_eager_bcopy = (uct_ep_tag_eager_bcopy_func_t)ucs_empty_function_return_ep_timeout;
360 ops->ep_tag_eager_zcopy = (uct_ep_tag_eager_zcopy_func_t)ucs_empty_function_return_ep_timeout;
361 ops->ep_tag_rndv_zcopy = (uct_ep_tag_rndv_zcopy_func_t)ucs_empty_function_return_ep_timeout;
362 ops->ep_tag_rndv_cancel = (uct_ep_tag_rndv_cancel_func_t)ucs_empty_function_return_ep_timeout;
363 ops->ep_tag_rndv_request = (uct_ep_tag_rndv_request_func_t)ucs_empty_function_return_ep_timeout;
364 ops->ep_pending_add = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy;
365 ops->ep_pending_purge = uct_ep_failed_purge;
366 ops->ep_flush = (uct_ep_flush_func_t)ucs_empty_function_return_ep_timeout;
367 ops->ep_fence = (uct_ep_fence_func_t)ucs_empty_function_return_ep_timeout;
368 ops->ep_check = (uct_ep_check_func_t)ucs_empty_function_return_ep_timeout;
369 ops->ep_connect_to_ep = (uct_ep_connect_to_ep_func_t)ucs_empty_function_return_ep_timeout;
370 ops->ep_destroy = uct_ep_failed_destroy;
371 ops->ep_get_address = (uct_ep_get_address_func_t)ucs_empty_function_return_ep_timeout;
372
373 ucs_class_call_cleanup_chain(cls, tl_ep, -1);
374
375 tl_ep->iface = &f_iface->super;
376
377 if (iface->err_handler) {
378 return iface->err_handler(iface->err_handler_arg, tl_ep, status);
379 } else if (status == UCS_ERR_CANCELED) {
380 ucs_debug("error %s was suppressed for ep %p",
381 ucs_status_string(UCS_ERR_CANCELED), tl_ep);
382 /* Suppress this since the cancellation is initiated by user. */
383 status = UCS_OK;
384 } else {
385 ucs_debug("error %s was not handled for ep %p",
386 ucs_status_string(status), tl_ep);
387 }
388
389 return status;
390 }
391
uct_base_iface_query(uct_base_iface_t * iface,uct_iface_attr_t * iface_attr)392 void uct_base_iface_query(uct_base_iface_t *iface, uct_iface_attr_t *iface_attr)
393 {
394 memset(iface_attr, 0, sizeof(*iface_attr));
395
396 iface_attr->max_num_eps = iface->config.max_num_eps;
397 iface_attr->dev_num_paths = 1;
398 }
399
uct_single_device_resource(uct_md_h md,const char * dev_name,uct_device_type_t dev_type,uct_tl_device_resource_t ** tl_devices_p,unsigned * num_tl_devices_p)400 ucs_status_t uct_single_device_resource(uct_md_h md, const char *dev_name,
401 uct_device_type_t dev_type,
402 uct_tl_device_resource_t **tl_devices_p,
403 unsigned *num_tl_devices_p)
404 {
405 uct_tl_device_resource_t *device;
406
407 device = ucs_calloc(1, sizeof(*device), "device resource");
408 if (NULL == device) {
409 ucs_error("failed to allocate device resource");
410 return UCS_ERR_NO_MEMORY;
411 }
412
413 ucs_snprintf_zero(device->name, sizeof(device->name), "%s", dev_name);
414 device->type = dev_type;
415
416 *num_tl_devices_p = 1;
417 *tl_devices_p = device;
418 return UCS_OK;
419 }
420
UCS_CLASS_INIT_FUNC(uct_iface_t,uct_iface_ops_t * ops)421 UCS_CLASS_INIT_FUNC(uct_iface_t, uct_iface_ops_t *ops)
422 {
423 ucs_assert_always(ops->ep_flush != NULL);
424 ucs_assert_always(ops->ep_fence != NULL);
425 ucs_assert_always(ops->ep_destroy != NULL);
426 ucs_assert_always(ops->iface_flush != NULL);
427 ucs_assert_always(ops->iface_fence != NULL);
428 ucs_assert_always(ops->iface_progress_enable != NULL);
429 ucs_assert_always(ops->iface_progress_disable != NULL);
430 ucs_assert_always(ops->iface_progress != NULL);
431 ucs_assert_always(ops->iface_close != NULL);
432 ucs_assert_always(ops->iface_query != NULL);
433 ucs_assert_always(ops->iface_get_device_address != NULL);
434 ucs_assert_always(ops->iface_is_reachable != NULL);
435
436 self->ops = *ops;
437 return UCS_OK;
438 }
439
UCS_CLASS_CLEANUP_FUNC(uct_iface_t)440 UCS_CLASS_CLEANUP_FUNC(uct_iface_t)
441 {
442 }
443
444 UCS_CLASS_DEFINE(uct_iface_t, void);
445
446
UCS_CLASS_INIT_FUNC(uct_base_iface_t,uct_iface_ops_t * ops,uct_md_h md,uct_worker_h worker,const uct_iface_params_t * params,const uct_iface_config_t * config UCS_STATS_ARG (ucs_stats_node_t * stats_parent)UCS_STATS_ARG (const char * iface_name))447 UCS_CLASS_INIT_FUNC(uct_base_iface_t, uct_iface_ops_t *ops, uct_md_h md,
448 uct_worker_h worker, const uct_iface_params_t *params,
449 const uct_iface_config_t *config
450 UCS_STATS_ARG(ucs_stats_node_t *stats_parent)
451 UCS_STATS_ARG(const char *iface_name))
452 {
453 uint64_t alloc_methods_bitmap;
454 uct_alloc_method_t method;
455 unsigned i;
456 uint8_t id;
457
458 UCS_CLASS_CALL_SUPER_INIT(uct_iface_t, ops);
459
460 UCT_CB_FLAGS_CHECK((params->field_mask &
461 UCT_IFACE_PARAM_FIELD_ERR_HANDLER_FLAGS) ?
462 params->err_handler_flags : 0);
463
464 self->md = md;
465 self->worker = ucs_derived_of(worker, uct_priv_worker_t);
466 self->am_tracer = NULL;
467 self->am_tracer_arg = NULL;
468 self->err_handler = (params->field_mask &
469 UCT_IFACE_PARAM_FIELD_ERR_HANDLER) ?
470 params->err_handler : NULL;
471 self->err_handler_flags = (params->field_mask &
472 UCT_IFACE_PARAM_FIELD_ERR_HANDLER_FLAGS) ?
473 params->err_handler_flags : 0;
474 self->err_handler_arg = (params->field_mask &
475 UCT_IFACE_PARAM_FIELD_ERR_HANDLER_ARG) ?
476 params->err_handler_arg : NULL;
477 self->progress_flags = 0;
478 uct_worker_progress_init(&self->prog);
479
480 for (id = 0; id < UCT_AM_ID_MAX; ++id) {
481 uct_iface_set_stub_am_handler(self, id);
482 }
483
484 /* Copy allocation methods configuration. In the process, remove duplicates. */
485 UCS_STATIC_ASSERT(sizeof(alloc_methods_bitmap) * 8 >= UCT_ALLOC_METHOD_LAST);
486 self->config.num_alloc_methods = 0;
487 alloc_methods_bitmap = 0;
488 for (i = 0; i < config->alloc_methods.count; ++i) {
489 method = config->alloc_methods.methods[i];
490 if (alloc_methods_bitmap & UCS_BIT(method)) {
491 continue;
492 }
493
494 ucs_assert(self->config.num_alloc_methods < UCT_ALLOC_METHOD_LAST);
495 self->config.alloc_methods[self->config.num_alloc_methods++] = method;
496 alloc_methods_bitmap |= UCS_BIT(method);
497 }
498
499 self->config.failure_level = (ucs_log_level_t)config->failure;
500 self->config.max_num_eps = config->max_num_eps;
501
502 return UCS_STATS_NODE_ALLOC(&self->stats, &uct_iface_stats_class,
503 stats_parent, "-%s-%p", iface_name, self);
504 }
505
UCS_CLASS_CLEANUP_FUNC(uct_base_iface_t)506 static UCS_CLASS_CLEANUP_FUNC(uct_base_iface_t)
507 {
508 UCS_STATS_NODE_FREE(self->stats);
509 }
510
511 UCS_CLASS_DEFINE(uct_base_iface_t, uct_iface_t);
512
513
uct_iface_accept(uct_iface_h iface,uct_conn_request_h conn_request)514 ucs_status_t uct_iface_accept(uct_iface_h iface,
515 uct_conn_request_h conn_request)
516 {
517 return iface->ops.iface_accept(iface, conn_request);
518 }
519
520
uct_iface_reject(uct_iface_h iface,uct_conn_request_h conn_request)521 ucs_status_t uct_iface_reject(uct_iface_h iface,
522 uct_conn_request_h conn_request)
523 {
524 return iface->ops.iface_reject(iface, conn_request);
525 }
526
527
uct_ep_create(const uct_ep_params_t * params,uct_ep_h * ep_p)528 ucs_status_t uct_ep_create(const uct_ep_params_t *params, uct_ep_h *ep_p)
529 {
530 if (params->field_mask & UCT_EP_PARAM_FIELD_IFACE) {
531 return params->iface->ops.ep_create(params, ep_p);
532 } else if (params->field_mask & UCT_EP_PARAM_FIELD_CM) {
533 return params->cm->ops->ep_create(params, ep_p);
534 }
535
536 return UCS_ERR_INVALID_PARAM;
537 }
538
uct_ep_disconnect(uct_ep_h ep,unsigned flags)539 ucs_status_t uct_ep_disconnect(uct_ep_h ep, unsigned flags)
540 {
541 return ep->iface->ops.ep_disconnect(ep, flags);
542 }
543
uct_ep_destroy(uct_ep_h ep)544 void uct_ep_destroy(uct_ep_h ep)
545 {
546 ep->iface->ops.ep_destroy(ep);
547 }
548
uct_ep_get_address(uct_ep_h ep,uct_ep_addr_t * addr)549 ucs_status_t uct_ep_get_address(uct_ep_h ep, uct_ep_addr_t *addr)
550 {
551 return ep->iface->ops.ep_get_address(ep, addr);
552 }
553
uct_ep_connect_to_ep(uct_ep_h ep,const uct_device_addr_t * dev_addr,const uct_ep_addr_t * ep_addr)554 ucs_status_t uct_ep_connect_to_ep(uct_ep_h ep, const uct_device_addr_t *dev_addr,
555 const uct_ep_addr_t *ep_addr)
556 {
557 return ep->iface->ops.ep_connect_to_ep(ep, dev_addr, ep_addr);
558 }
559
uct_cm_client_ep_conn_notify(uct_ep_h ep)560 ucs_status_t uct_cm_client_ep_conn_notify(uct_ep_h ep)
561 {
562 return ep->iface->ops.cm_ep_conn_notify(ep);
563 }
564
UCS_CLASS_INIT_FUNC(uct_ep_t,uct_iface_t * iface)565 UCS_CLASS_INIT_FUNC(uct_ep_t, uct_iface_t *iface)
566 {
567 self->iface = iface;
568 return UCS_OK;
569 }
570
UCS_CLASS_CLEANUP_FUNC(uct_ep_t)571 UCS_CLASS_CLEANUP_FUNC(uct_ep_t)
572 {
573 }
574
575 UCS_CLASS_DEFINE(uct_ep_t, void);
576
UCS_CLASS_INIT_FUNC(uct_base_ep_t,uct_base_iface_t * iface)577 UCS_CLASS_INIT_FUNC(uct_base_ep_t, uct_base_iface_t *iface)
578 {
579 UCS_CLASS_CALL_SUPER_INIT(uct_ep_t, &iface->super);
580
581 return UCS_STATS_NODE_ALLOC(&self->stats, &uct_ep_stats_class, iface->stats,
582 "-%p", self);
583 }
584
UCS_CLASS_CLEANUP_FUNC(uct_base_ep_t)585 static UCS_CLASS_CLEANUP_FUNC(uct_base_ep_t)
586 {
587 UCS_STATS_NODE_FREE(self->stats);
588 }
589
590 UCS_CLASS_DEFINE(uct_base_ep_t, uct_ep_t);
591
592
593 UCS_CONFIG_DEFINE_ARRAY(alloc_methods, sizeof(uct_alloc_method_t),
594 UCS_CONFIG_TYPE_ENUM(uct_alloc_method_names));
595
596 ucs_config_field_t uct_iface_config_table[] = {
597 {"MAX_SHORT", "",
598 "The configuration parameter replaced by: "
599 "UCX_<IB transport>_TX_MIN_INLINE for IB, UCX_MM_FIFO_SIZE for MM",
600 UCS_CONFIG_DEPRECATED_FIELD_OFFSET, UCS_CONFIG_TYPE_DEPRECATED},
601
602 {"MAX_BCOPY", "",
603 "The configuration parameter replaced by: "
604 "UCX_<transport>_SEG_SIZE where <transport> is one of: IB, MM, SELF, TCP",
605 UCS_CONFIG_DEPRECATED_FIELD_OFFSET, UCS_CONFIG_TYPE_DEPRECATED},
606
607 {"ALLOC", "huge,thp,md,mmap,heap",
608 "Priority of methods to allocate intermediate buffers for communication",
609 ucs_offsetof(uct_iface_config_t, alloc_methods), UCS_CONFIG_TYPE_ARRAY(alloc_methods)},
610
611 {"FAILURE", "error",
612 "Level of network failure reporting",
613 ucs_offsetof(uct_iface_config_t, failure), UCS_CONFIG_TYPE_ENUM(ucs_log_level_names)},
614
615 {"MAX_NUM_EPS", "inf",
616 "Maximum number of endpoints that the transport interface is able to create",
617 ucs_offsetof(uct_iface_config_t, max_num_eps), UCS_CONFIG_TYPE_ULUNITS},
618
619 {NULL}
620 };
621