1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED.
3 *
4 * See file LICENSE for terms.
5 */
6
7 #ifdef HAVE_CONFIG_H
8 # include "config.h"
9 #endif
10
11 #include "ucp_proxy_ep.h"
12 #include "ucp_ep.inl"
13
14 #include <ucs/debug/log.h>
15
16
17 #define UCP_PROXY_EP_PASTE_ARG_NAME(_, _index) \
18 , UCS_PP_TOKENPASTE(arg, _index)
19
20 #define UCP_PROXY_EP_PASTE_ARG_TYPE(_, _bundle) \
21 , UCS_PP_TUPLE_1 _bundle UCS_PP_TOKENPASTE(arg, UCS_PP_TUPLE_0 _bundle)
22
23 /* Generate list of typed arguments for a proxy function prototype */
24 #define UCP_PROXY_EP_FUNC_ARGS(...) \
25 uct_ep_h ep \
26 UCS_PP_FOREACH(UCP_PROXY_EP_PASTE_ARG_TYPE, _, \
27 UCS_PP_ZIP((UCS_PP_SEQ(UCS_PP_NUM_ARGS(__VA_ARGS__))), \
28 (__VA_ARGS__)))
29
30 /* Generate a list of arguments passed to function call */
31 #define UCP_PROXY_EP_FUNC_CALL(...) \
32 proxy_ep->uct_ep \
33 UCS_PP_FOREACH(UCP_PROXY_EP_PASTE_ARG_NAME, _, \
34 UCS_PP_SEQ(UCS_PP_NUM_ARGS(__VA_ARGS__)))
35
36 /* Generate a return statement based on return type */
37 #define UCP_PROXY_EP_RETURN(_retval) \
38 UCS_PP_TOKENPASTE(UCP_PROXY_EP_RETURN_, _retval)
39
40 #define UCP_PROXY_EP_RETURN_ucs_status_t return
41 #define UCP_PROXY_EP_RETURN_ucs_status_ptr_t return
42 #define UCP_PROXY_EP_RETURN_ssize_t return
43 #define UCP_PROXY_EP_RETURN_void
44
45
46 /*
47 * Define a proxy endpoint operation, which redirects the call to the underlying
48 * transport endpoint.
49 */
50 #define UCP_PROXY_EP_DEFINE_OP(_retval, _name, ...) \
51 static _retval ucp_proxy_ep_##_name(UCP_PROXY_EP_FUNC_ARGS(__VA_ARGS__)) \
52 { \
53 ucp_proxy_ep_t *proxy_ep = ucs_derived_of(ep, ucp_proxy_ep_t); \
54 UCP_PROXY_EP_RETURN(_retval) \
55 uct_ep_##_name(UCP_PROXY_EP_FUNC_CALL(__VA_ARGS__)); \
56 }
57
58
59 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, put_short, const void*, unsigned,
60 uint64_t, uct_rkey_t)
61 UCP_PROXY_EP_DEFINE_OP(ssize_t, put_bcopy, uct_pack_callback_t, void*,
62 uint64_t, uct_rkey_t)
63 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, put_zcopy, const uct_iov_t*, size_t,
64 uint64_t, uct_rkey_t, uct_completion_t*)
65 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, get_bcopy, uct_unpack_callback_t, void*,
66 size_t, uint64_t, uct_rkey_t, uct_completion_t*)
67 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, get_zcopy, const uct_iov_t*, size_t,
68 uint64_t, uct_rkey_t, uct_completion_t*)
69 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, am_short, uint8_t, uint64_t, const void*,
70 unsigned)
71 UCP_PROXY_EP_DEFINE_OP(ssize_t, am_bcopy, uint8_t, uct_pack_callback_t, void*,
72 unsigned)
73 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, am_zcopy, uint8_t, const void*, unsigned,
74 const uct_iov_t*, size_t, unsigned, uct_completion_t*)
75 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic_cswap64, uint64_t, uint64_t,
76 uint64_t, uct_rkey_t, uint64_t*, uct_completion_t*)
77 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic_cswap32, uint32_t, uint32_t,
78 uint64_t, uct_rkey_t, uint32_t*, uct_completion_t*)
79 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic64_post, uct_atomic_op_t,
80 uint64_t, uint64_t, uct_rkey_t)
81 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic32_post, uct_atomic_op_t,
82 uint32_t, uint64_t, uct_rkey_t)
83 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic64_fetch, uct_atomic_op_t, uint64_t,
84 uint64_t*, uint64_t, uct_rkey_t, uct_completion_t*)
85 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, atomic32_fetch, uct_atomic_op_t, uint32_t,
86 uint32_t*, uint64_t, uct_rkey_t, uct_completion_t*)
87 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, tag_eager_short, uct_tag_t, const void*,
88 size_t)
89 UCP_PROXY_EP_DEFINE_OP(ssize_t, tag_eager_bcopy, uct_tag_t, uint64_t,
90 uct_pack_callback_t, void*, unsigned)
91 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, tag_eager_zcopy, uct_tag_t, uint64_t,
92 const uct_iov_t*, size_t, unsigned, uct_completion_t*)
93 UCP_PROXY_EP_DEFINE_OP(ucs_status_ptr_t, tag_rndv_zcopy, uct_tag_t, const void*,
94 unsigned, const uct_iov_t*, size_t, unsigned,
95 uct_completion_t*)
96 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, tag_rndv_cancel, void*)
97 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, tag_rndv_request, uct_tag_t, const void*,
98 unsigned, unsigned)
99 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, pending_add, uct_pending_req_t*, unsigned)
100 UCP_PROXY_EP_DEFINE_OP(void, pending_purge, uct_pending_purge_callback_t, void*)
101 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, flush, unsigned, uct_completion_t*)
102 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, fence, unsigned)
103 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, check, unsigned, uct_completion_t*)
104 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, get_address, uct_ep_addr_t*)
105 UCP_PROXY_EP_DEFINE_OP(ucs_status_t, connect_to_ep, const uct_device_addr_t*,
106 const uct_ep_addr_t*)
107 static UCS_CLASS_DEFINE_NAMED_DELETE_FUNC(ucp_proxy_ep_destroy, ucp_proxy_ep_t,
108 uct_ep_t);
109
ucp_proxy_ep_fatal(uct_iface_h iface,...)110 static ucs_status_t ucp_proxy_ep_fatal(uct_iface_h iface, ...)
111 {
112 ucs_bug("unsupported function on proxy endpoint");
113 return UCS_ERR_UNSUPPORTED;
114 }
115
UCS_CLASS_INIT_FUNC(ucp_proxy_ep_t,const uct_iface_ops_t * ops,ucp_ep_h ucp_ep,uct_ep_h uct_ep,int is_owner)116 UCS_CLASS_INIT_FUNC(ucp_proxy_ep_t, const uct_iface_ops_t *ops, ucp_ep_h ucp_ep,
117 uct_ep_h uct_ep, int is_owner)
118 {
119 #define UCP_PROXY_EP_SET_OP(_name) \
120 self->iface.ops._name = (ops->_name != NULL) ? ops->_name : ucp_proxy_##_name
121
122 self->super.iface = &self->iface;
123 self->ucp_ep = ucp_ep;
124 self->uct_ep = uct_ep;
125 self->is_owner = is_owner;
126
127 UCP_PROXY_EP_SET_OP(ep_put_short);
128 UCP_PROXY_EP_SET_OP(ep_put_short);
129 UCP_PROXY_EP_SET_OP(ep_put_bcopy);
130 UCP_PROXY_EP_SET_OP(ep_put_zcopy);
131 UCP_PROXY_EP_SET_OP(ep_get_bcopy);
132 UCP_PROXY_EP_SET_OP(ep_get_zcopy);
133 UCP_PROXY_EP_SET_OP(ep_am_short);
134 UCP_PROXY_EP_SET_OP(ep_am_bcopy);
135 UCP_PROXY_EP_SET_OP(ep_am_zcopy);
136 UCP_PROXY_EP_SET_OP(ep_atomic_cswap64);
137 UCP_PROXY_EP_SET_OP(ep_atomic_cswap32);
138 UCP_PROXY_EP_SET_OP(ep_atomic64_post);
139 UCP_PROXY_EP_SET_OP(ep_atomic32_post);
140 UCP_PROXY_EP_SET_OP(ep_atomic64_fetch);
141 UCP_PROXY_EP_SET_OP(ep_atomic32_fetch);
142 UCP_PROXY_EP_SET_OP(ep_tag_eager_short);
143 UCP_PROXY_EP_SET_OP(ep_tag_eager_bcopy);
144 UCP_PROXY_EP_SET_OP(ep_tag_eager_zcopy);
145 UCP_PROXY_EP_SET_OP(ep_tag_rndv_zcopy);
146 UCP_PROXY_EP_SET_OP(ep_tag_rndv_cancel);
147 UCP_PROXY_EP_SET_OP(ep_tag_rndv_request);
148 UCP_PROXY_EP_SET_OP(ep_pending_add);
149 UCP_PROXY_EP_SET_OP(ep_pending_purge);
150 UCP_PROXY_EP_SET_OP(ep_flush);
151 UCP_PROXY_EP_SET_OP(ep_fence);
152 UCP_PROXY_EP_SET_OP(ep_check);
153 UCP_PROXY_EP_SET_OP(ep_destroy);
154 UCP_PROXY_EP_SET_OP(ep_get_address);
155 UCP_PROXY_EP_SET_OP(ep_connect_to_ep);
156
157 self->iface.ops.iface_tag_recv_zcopy = (uct_iface_tag_recv_zcopy_func_t)ucp_proxy_ep_fatal;
158 self->iface.ops.iface_tag_recv_cancel = (uct_iface_tag_recv_cancel_func_t)ucp_proxy_ep_fatal;
159 self->iface.ops.ep_create = (uct_ep_create_func_t)ucp_proxy_ep_fatal;
160 self->iface.ops.iface_flush = (uct_iface_flush_func_t)ucp_proxy_ep_fatal;
161 self->iface.ops.iface_fence = (uct_iface_fence_func_t)ucp_proxy_ep_fatal;
162 self->iface.ops.iface_progress_enable = (uct_iface_progress_enable_func_t)ucp_proxy_ep_fatal;
163 self->iface.ops.iface_progress_disable = (uct_iface_progress_disable_func_t)ucp_proxy_ep_fatal;
164 self->iface.ops.iface_progress = (uct_iface_progress_func_t)ucp_proxy_ep_fatal;
165 self->iface.ops.iface_event_fd_get = (uct_iface_event_fd_get_func_t)ucp_proxy_ep_fatal;
166 self->iface.ops.iface_event_arm = (uct_iface_event_arm_func_t)ucp_proxy_ep_fatal;
167 self->iface.ops.iface_close = (uct_iface_close_func_t)ucp_proxy_ep_fatal;
168 self->iface.ops.iface_query = (uct_iface_query_func_t)ucp_proxy_ep_fatal;
169 self->iface.ops.iface_get_device_address = (uct_iface_get_device_address_func_t)ucp_proxy_ep_fatal;
170 self->iface.ops.iface_get_address = (uct_iface_get_address_func_t)ucp_proxy_ep_fatal;
171 self->iface.ops.iface_is_reachable = (uct_iface_is_reachable_func_t)ucp_proxy_ep_fatal;
172
173 return UCS_OK;
174 }
175
UCS_CLASS_CLEANUP_FUNC(ucp_proxy_ep_t)176 static UCS_CLASS_CLEANUP_FUNC(ucp_proxy_ep_t)
177 {
178 if ((self->uct_ep != NULL) && self->is_owner) {
179 uct_ep_destroy(self->uct_ep);
180 }
181 }
182
ucp_proxy_ep_test(uct_ep_h uct_ep)183 int ucp_proxy_ep_test(uct_ep_h uct_ep)
184 {
185 return uct_ep->iface->ops.ep_destroy == ucp_proxy_ep_destroy;
186 }
187
ucp_proxy_ep_extract(uct_ep_h ep)188 uct_ep_h ucp_proxy_ep_extract(uct_ep_h ep)
189 {
190 ucp_proxy_ep_t *proxy_ep = ucs_derived_of(ep, ucp_proxy_ep_t);
191 uct_ep_h uct_ep;
192
193 uct_ep = proxy_ep->uct_ep;
194 proxy_ep->uct_ep = NULL;
195 return uct_ep;
196 }
197
ucp_proxy_ep_replace_if_owned(uct_ep_h uct_ep,uct_ep_h owned_ep,uct_ep_h replacement_ep)198 static void ucp_proxy_ep_replace_if_owned(uct_ep_h uct_ep, uct_ep_h owned_ep,
199 uct_ep_h replacement_ep)
200 {
201 ucp_proxy_ep_t *proxy_ep;
202
203 if (ucp_proxy_ep_test(uct_ep)) {
204 proxy_ep = ucs_derived_of(uct_ep, ucp_proxy_ep_t);
205 if (proxy_ep->uct_ep == owned_ep) {
206 proxy_ep->uct_ep = replacement_ep;
207 }
208 ucs_assert(replacement_ep != NULL);
209 }
210 }
211
ucp_proxy_ep_replace(ucp_proxy_ep_t * proxy_ep)212 void ucp_proxy_ep_replace(ucp_proxy_ep_t *proxy_ep)
213 {
214 ucp_ep_h ucp_ep = proxy_ep->ucp_ep;
215 ucp_lane_index_t lane;
216 uct_ep_h tl_ep = NULL;
217
218 ucs_assert(proxy_ep->uct_ep != NULL);
219 for (lane = 0; lane < ucp_ep_num_lanes(ucp_ep); ++lane) {
220 if (ucp_ep->uct_eps[lane] == &proxy_ep->super) {
221 ucs_assert(proxy_ep->uct_ep != NULL); /* make sure there is only one match */
222 ucp_ep->uct_eps[lane] = proxy_ep->uct_ep;
223 tl_ep = ucp_ep->uct_eps[lane];
224 proxy_ep->uct_ep = NULL;
225 }
226 }
227
228 /* go through the lanes and check if the proxy ep that is being destroyed,
229 * is pointed to by another proxy ep. if so, redirect that other proxy ep
230 * to point to the underlying uct ep. */
231 for (lane = 0; lane < ucp_ep_num_lanes(ucp_ep); ++lane) {
232 ucp_proxy_ep_replace_if_owned(ucp_ep->uct_eps[lane], &proxy_ep->super,
233 tl_ep);
234 }
235
236 uct_ep_destroy(&proxy_ep->super);
237 }
238
ucp_proxy_ep_set_uct_ep(ucp_proxy_ep_t * proxy_ep,uct_ep_h uct_ep,int is_owner)239 void ucp_proxy_ep_set_uct_ep(ucp_proxy_ep_t *proxy_ep, uct_ep_h uct_ep,
240 int is_owner)
241 {
242 proxy_ep->uct_ep = uct_ep;
243 proxy_ep->is_owner = is_owner;
244 }
245
246 UCS_CLASS_DEFINE(ucp_proxy_ep_t, void);
247