1 /*
2 * Copyright (c) 2018-2019 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL); Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #ifndef _OFI_HOOK_H_
34 #define _OFI_HOOK_H_
35
36 #include <assert.h>
37
38 #include <rdma/fabric.h>
39 #include <rdma/fi_atomic.h>
40 #include <rdma/fi_collective.h>
41 #include <rdma/fi_cm.h>
42 #include <rdma/fi_domain.h>
43 #include <rdma/fi_endpoint.h>
44 #include <rdma/fi_eq.h>
45 #include <rdma/fi_rma.h>
46 #include <rdma/fi_tagged.h>
47
48 #include <ofi.h>
49 #include <rdma/providers/fi_prov.h>
50
51 /* This field needs to be updated whenever new FI class is added in fabric.h */
52 #define HOOK_FI_CLASS_MAX (FI_CLASS_NIC + 1)
53
54 /*
55 * Hooks are installed from top down.
56 * Values must start at 0 and increment by one.
57 */
58 // TODO figure out how to remove this now that we have ini/fini calls
59 enum ofi_hook_class {
60 HOOK_NOOP,
61 HOOK_PERF,
62 HOOK_DEBUG,
63 MAX_HOOKS
64 };
65
66
67 /*
68 * Define hook structs so we can cast from fid to parent using simple cast.
69 * This lets us have a single close() call.
70 */
71
72 extern struct fi_ops hook_fid_ops;
73 struct fid *hook_to_hfid(const struct fid *fid);
74 struct fid_wait *hook_to_hwait(const struct fid_wait *wait);
75
76 /*
77 * TODO
78 * comment from GitHub PR #5052:
79 * "another option would be to store the ini/fini calls in a separate structure
80 * that we reference from struct fi_prov_context. We could even extend the
81 * definition of fi_prov_context with a union that is accessed based on the
82 * prov_type. That might work better if we want to support external hooks,
83 * without the external hook provider needing to implement everything"
84 */
85 struct hook_prov_ctx {
86 struct fi_provider prov;
87 /*
88 * Hooking providers can override ini/fini calls of a specific fid class
89 * to override any initializations that the common code may have done.
90 * For example, this allows overriding any specific op and not having to
91 * hook into every resource creation call until the point where the op
92 * can be overridden. Refer to hook_perf for an example.
93 *
94 * Note: if a hooking provider overrides any of the resource creation calls
95 * (e.g. fi_endpoint) directly, then these ini/fini calls won't be
96 * invoked. */
97 int (*ini_fid[HOOK_FI_CLASS_MAX])(struct fid *fid);
98 int (*fini_fid[HOOK_FI_CLASS_MAX])(struct fid *fid);
99 };
100
101 /*
102 * TODO
103 * comment from GitHub PR #5052:
104 * "We could set all ini/fini calls to a no-op as part of hook initialization
105 * to avoid this check"
106 */
hook_ini_fid(struct hook_prov_ctx * prov_ctx,struct fid * fid)107 static inline int hook_ini_fid(struct hook_prov_ctx *prov_ctx, struct fid *fid)
108 {
109 return (prov_ctx->ini_fid[fid->fclass] ?
110 prov_ctx->ini_fid[fid->fclass](fid) : 0);
111 }
112
hook_fini_fid(struct hook_prov_ctx * prov_ctx,struct fid * fid)113 static inline int hook_fini_fid(struct hook_prov_ctx *prov_ctx, struct fid *fid)
114 {
115 return (prov_ctx->fini_fid[fid->fclass] ?
116 prov_ctx->fini_fid[fid->fclass](fid) : 0);
117 }
118
119 struct hook_fabric {
120 struct fid_fabric fabric;
121 struct fid_fabric *hfabric;
122 enum ofi_hook_class hclass;
123 struct fi_provider *hprov;
124 struct hook_prov_ctx *prov_ctx;
125 };
126
127 void hook_fabric_init(struct hook_fabric *fabric, enum ofi_hook_class hclass,
128 struct fid_fabric *hfabric, struct fi_provider *hprov,
129 struct fi_ops *f_ops, struct hook_prov_ctx *prov_ctx);
130
131 struct hook_fabric *hook_to_fabric(const struct fid *fid);
132
hook_to_prov_ctx(const struct fid * fid)133 static inline struct hook_prov_ctx *hook_to_prov_ctx(const struct fid *fid)
134 {
135 return (hook_to_fabric(fid))->prov_ctx;
136 }
137
138 static inline struct fi_provider *
hook_fabric_to_hprov(const struct hook_fabric * fabric)139 hook_fabric_to_hprov(const struct hook_fabric *fabric)
140 {
141 return fabric->hprov;
142 }
143
hook_to_hprov(const struct fid * fid)144 static inline struct fi_provider *hook_to_hprov(const struct fid *fid)
145 {
146 return hook_fabric_to_hprov(hook_to_fabric(fid));
147 }
148
149 struct hook_domain {
150 struct fid_domain domain;
151 struct fid_domain *hdomain;
152 struct hook_fabric *fabric;
153 };
154
155 int hook_domain(struct fid_fabric *fabric, struct fi_info *info,
156 struct fid_domain **domain, void *context);
157
158
159 struct hook_av {
160 struct fid_av av;
161 struct fid_av *hav;
162 struct hook_domain *domain;
163 };
164
165 int hook_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
166 struct fid_av **av, void *context);
167
168
169 struct hook_wait {
170 struct fid_wait wait;
171 struct fid_wait *hwait;
172 struct hook_fabric *fabric;
173 };
174
175 int hook_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
176 struct fid_wait **waitset);
177 int hook_trywait(struct fid_fabric *fabric, struct fid **fids, int count);
178
179
180 struct hook_poll {
181 struct fid_poll poll;
182 struct fid_poll *hpoll;
183 struct hook_domain *domain;
184 };
185
186 int hook_poll_open(struct fid_domain *domain, struct fi_poll_attr *attr,
187 struct fid_poll **pollset);
188
189 /*
190 * EQ
191 */
192
193 extern struct fi_ops_eq hook_eq_ops;
194
195 struct hook_eq {
196 struct fid_eq eq;
197 struct fid_eq *heq;
198 struct hook_fabric *fabric;
199 };
200
201 ssize_t hook_eq_read(struct fid_eq *eq, uint32_t *event,
202 void *buf, size_t len, uint64_t flags);
203 ssize_t hook_eq_sread(struct fid_eq *eq, uint32_t *event,
204 void *buf, size_t len, int timeout, uint64_t flags);
205 int hook_eq_init(struct fid_fabric *fabric, struct fi_eq_attr *attr,
206 struct fid_eq **eq, void *context, struct hook_eq *myeq);
207 int hook_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr,
208 struct fid_eq **eq, void *context);
209
210 /*
211 * CQ
212 */
213
214 struct hook_cq {
215 struct fid_cq cq;
216 struct fid_cq *hcq;
217 struct hook_domain *domain;
218 void *context;
219 };
220
221 int hook_cq_init(struct fid_domain *domain, struct fi_cq_attr *attr,
222 struct fid_cq **cq, void *context, struct hook_cq *mycq);
223 int hook_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
224 struct fid_cq **cq, void *context);
225 const char *
226 hook_cq_strerror(struct fid_cq *cq, int prov_errno,
227 const void *err_data, char *buf, size_t len);
228
229 struct hook_cntr {
230 struct fid_cntr cntr;
231 struct fid_cntr *hcntr;
232 struct hook_domain *domain;
233 };
234
235 int hook_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
236 struct fid_cntr **cntr, void *context);
237
238
239 struct hook_ep {
240 struct fid_ep ep;
241 struct fid_ep *hep;
242 struct hook_domain *domain;
243 void *context;
244 };
245
246 int hook_endpoint_init(struct fid_domain *domain, struct fi_info *info,
247 struct fid_ep **ep, void *context, struct hook_ep *myep);
248 int hook_endpoint(struct fid_domain *domain, struct fi_info *info,
249 struct fid_ep **ep, void *context);
250 int hook_scalable_ep(struct fid_domain *domain, struct fi_info *info,
251 struct fid_ep **sep, void *context);
252 int hook_srx_ctx(struct fid_domain *domain,
253 struct fi_rx_attr *attr, struct fid_ep **rx_ep,
254 void *context);
255
256 int hook_query_atomic(struct fid_domain *domain, enum fi_datatype datatype,
257 enum fi_op op, struct fi_atomic_attr *attr, uint64_t flags);
258
259 extern struct fi_ops hook_fabric_fid_ops;
260 extern struct fi_ops_fabric hook_fabric_ops;
261 extern struct fi_ops_domain hook_domain_ops;
262 extern struct fi_ops_cq hook_cq_ops;
263 extern struct fi_ops_cntr hook_cntr_ops;
264
265 extern struct fi_ops_cm hook_cm_ops;
266 extern struct fi_ops_msg hook_msg_ops;
267 extern struct fi_ops_rma hook_rma_ops;
268 extern struct fi_ops_tagged hook_tagged_ops;
269 extern struct fi_ops_atomic hook_atomic_ops;
270
271
272 struct hook_pep {
273 struct fid_pep pep;
274 struct fid_pep *hpep;
275 struct hook_fabric *fabric;
276 };
277
278 int hook_passive_ep(struct fid_fabric *fabric, struct fi_info *info,
279 struct fid_pep **pep, void *context);
280
281
282 struct hook_stx {
283 struct fid_stx stx;
284 struct fid_stx *hstx;
285 struct hook_domain *domain;
286 };
287
288 int hook_stx_ctx(struct fid_domain *domain,
289 struct fi_tx_attr *attr, struct fid_stx **stx,
290 void *context);
291
292
293 struct hook_mr {
294 struct fid_mr mr;
295 struct fid_mr *hmr;
296 struct hook_domain *domain;
297 };
298
299
300 #endif /* _OFI_HOOK_H_ */
301