1 /*
2 * Copyright (c) 2013-2018 Intel Corporation. All rights reserved.
3 * Copyright (c) 2016-2018 Cisco Systems, Inc. All rights reserved.
4 * Copyright (c) 2019 Amazon.com, Inc. or its affiliates. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #ifndef _OFI_H_
36 #define _OFI_H_
37
38 #include "config.h"
39
40 #include <assert.h>
41 #include <pthread.h>
42 #include <stdlib.h>
43 #include <string.h>
44 #include <stdarg.h>
45 #include <sys/param.h>
46 #include <netinet/in.h>
47 #include <ifaddrs.h>
48
49 #include <ofi_abi.h>
50 #include <ofi_file.h>
51 #include <ofi_lock.h>
52 #include <ofi_atom.h>
53 #include <ofi_mem.h>
54 #include <ofi_net.h>
55 #include <rdma/providers/fi_prov.h>
56 #include <rdma/providers/fi_log.h>
57
58 #include <rdma/fabric.h>
59 #include <rdma/fi_atomic.h>
60
61 #include <ofi_osd.h>
62
63
64 #ifdef __cplusplus
65 extern "C" {
66 #endif
67
68 /* For in-tree providers */
69 #define OFI_VERSION_LATEST FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION)
70 /* The lower minor digit is reserved for custom libfabric builds */
71 #define OFI_VERSION_DEF_PROV \
72 FI_VERSION(FI_MAJOR_VERSION * 100 + FI_MINOR_VERSION, \
73 FI_REVISION_VERSION * 10)
74
75 #define OFI_GETINFO_INTERNAL (1ULL << 58)
76 #define OFI_CORE_PROV_ONLY (1ULL << 59)
77 #define OFI_GETINFO_HIDDEN (1ULL << 60)
78
79 #define OFI_ORDER_RAR_SET (FI_ORDER_RAR | FI_ORDER_RMA_RAR | \
80 FI_ORDER_ATOMIC_RAR)
81 #define OFI_ORDER_RAW_SET (FI_ORDER_RAW | FI_ORDER_RMA_RAW | \
82 FI_ORDER_ATOMIC_RAW)
83 #define OFI_ORDER_WAR_SET (FI_ORDER_WAR | FI_ORDER_RMA_WAR | \
84 FI_ORDER_ATOMIC_WAR)
85 #define OFI_ORDER_WAW_SET (FI_ORDER_WAW | FI_ORDER_RMA_WAW | \
86 FI_ORDER_ATOMIC_WAW)
87
88 #define OFI_IGNORED_TX_CAPS /* older Rx caps not applicable to Tx */ \
89 (FI_REMOTE_READ | FI_REMOTE_WRITE | FI_RECV | FI_DIRECTED_RECV | \
90 FI_VARIABLE_MSG | FI_MULTI_RECV | FI_SOURCE | FI_RMA_EVENT | \
91 FI_SOURCE_ERR)
92 #define OFI_IGNORED_RX_CAPS /* Older Tx caps not applicable to Rx */ \
93 (FI_READ | FI_WRITE | FI_SEND | FI_FENCE | FI_MULTICAST | \
94 FI_NAMED_RX_CTX)
95
96
97 #define sizeof_field(type, field) sizeof(((type *)0)->field)
98
99 #ifndef MIN
100 #define MIN(a, b) \
101 ({ typeof (a) _a = (a); \
102 typeof (b) _b = (b); \
103 _a < _b ? _a : _b; })
104 #endif
105
106 #ifndef MAX
107 #define MAX(a, b) \
108 ({ typeof (a) _a = (a); \
109 typeof (b) _b = (b); \
110 _a > _b ? _a : _b; })
111 #endif
112
113 #define ofi_div_ceil(a, b) ((a + b - 1) / b)
114
ofi_val64_gt(uint64_t x,uint64_t y)115 static inline int ofi_val64_gt(uint64_t x, uint64_t y) {
116 return ((int64_t) (x - y)) > 0;
117 }
ofi_val64_ge(uint64_t x,uint64_t y)118 static inline int ofi_val64_ge(uint64_t x, uint64_t y) {
119 return ((int64_t) (x - y)) >= 0;
120 }
121 #define ofi_val64_lt(x, y) ofi_val64_gt(y, x)
122
ofi_val32_gt(uint32_t x,uint32_t y)123 static inline int ofi_val32_gt(uint32_t x, uint32_t y) {
124 return ((int32_t) (x - y)) > 0;
125 }
ofi_val32_ge(uint32_t x,uint32_t y)126 static inline int ofi_val32_ge(uint32_t x, uint32_t y) {
127 return ((int32_t) (x - y)) >= 0;
128 }
129 #define ofi_val32_lt(x, y) ofi_val32_gt(y, x)
130
131 #define ofi_val32_inrange(start, length, value) \
132 ofi_val32_ge(value, start) && ofi_val32_lt(value, start + length)
133 #define ofi_val64_inrange(start, length, value) \
134 ofi_val64_ge(value, start) && ofi_val64_lt(value, start + length)
135
136 #define OFI_MAGIC_64 (0x0F1C0DE0F1C0DE64)
137
138 #ifndef BIT
139 #define BIT(nr) (1UL << (nr))
140 #endif
141
142 #ifndef BIT_ULL
143 #define BIT_ULL(nr) (1ULL << (nr))
144 #endif
145
146 #ifndef GENMASK
147 #define GENMASK(h, l) (((1U << ((h) - (l) + 1)) - 1) << (l))
148 #endif
149
150 #ifndef GENMASK_ULL
151 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
152 #endif
153
154 #ifndef ARRAY_SIZE
155 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
156 #endif
157
158 #define TAB " "
159
160 #define CASEENUMSTR(SYM) \
161 case SYM: { ofi_strcatf(buf, #SYM); break; }
162 #define IFFLAGSTR(flags, SYM) \
163 do { if (flags & SYM) ofi_strcatf(buf, #SYM ", "); } while(0)
164 #define CASEENUMSTRN(SYM, N) \
165 case SYM: { ofi_strncatf(buf, N, #SYM); break; }
166 #define IFFLAGSTRN(flags, SYM, N) \
167 do { if (flags & SYM) ofi_strncatf(buf, N, #SYM ", "); } while(0)
168
169 #define ofi_strcatf(dest, ...) \
170 ofi_strncatf(dest, OFI_BUFSIZ, __VA_ARGS__)
171
172 /*
173 * CPU specific features
174 */
175 enum {
176 OFI_CLWB_REG = 2,
177 OFI_CLWB_BIT = (1 << 24),
178 OFI_CLFLUSHOPT_REG = 1,
179 OFI_CLFLUSHOPT_BIT = (1 << 24),
180 OFI_CLFLUSH_REG = 3,
181 OFI_CLFLUSH_BIT = (1 << 23),
182 };
183
184 int ofi_cpu_supports(unsigned func, unsigned reg, unsigned bit);
185
186
187 enum ofi_prov_type {
188 OFI_PROV_CORE,
189 OFI_PROV_UTIL,
190 OFI_PROV_HOOK,
191 };
192
193 /* Restrict to size of struct fi_provider::context (struct fi_context) */
194 struct fi_prov_context {
195 enum ofi_prov_type type;
196 int disable_logging;
197 };
198
199 struct fi_filter {
200 char **names;
201 int negated;
202 };
203
204 extern struct fi_filter prov_log_filter;
205 extern struct fi_provider core_prov;
206
207 void ofi_create_filter(struct fi_filter *filter, const char *env_name);
208 void ofi_free_filter(struct fi_filter *filter);
209 int ofi_apply_filter(struct fi_filter *filter, const char *name);
210
211 int ofi_nic_close(struct fid *fid);
212 struct fid_nic *ofi_nic_dup(const struct fid_nic *nic);
213 int ofi_nic_tostr(const struct fid *fid_nic, char *buf, size_t len);
214
215 struct fi_provider *ofi_get_hook(const char *name);
216
217 void fi_log_init(void);
218 void fi_log_fini(void);
219 void fi_param_init(void);
220 void fi_param_fini(void);
221 void fi_param_undefine(const struct fi_provider *provider);
222 void ofi_hook_init(void);
223 void ofi_hook_fini(void);
224 void ofi_hook_install(struct fid_fabric *hfabric, struct fid_fabric **fabric,
225 struct fi_provider *prov);
226 void ofi_remove_comma(char *buffer);
227 void ofi_strncatf(char *dest, size_t n, const char *fmt, ...);
228
229 const char *ofi_hex_str(const uint8_t *data, size_t len);
230
roundup_power_of_two(uint64_t n)231 static inline uint64_t roundup_power_of_two(uint64_t n)
232 {
233 if (!n || !(n & (n - 1)))
234 return n;
235 n--;
236 n |= n >> 1;
237 n |= n >> 2;
238 n |= n >> 4;
239 n |= n >> 8;
240 n |= n >> 16;
241 n |= n >> 32;
242 n++;
243 return n;
244 }
245
rounddown_power_of_two(uint64_t n)246 static inline uint64_t rounddown_power_of_two(uint64_t n)
247 {
248 uint64_t pof2 = roundup_power_of_two(n);
249 return (pof2 > n) ? (pof2 >> 1) : pof2;
250 }
251
ofi_get_aligned_size(size_t size,size_t alignment)252 static inline size_t ofi_get_aligned_size(size_t size, size_t alignment)
253 {
254 return ((size % alignment) == 0) ?
255 size : ((size / alignment) + 1) * alignment;
256 }
257
ofi_get_page_start(const void * addr,size_t page_size)258 static inline void *ofi_get_page_start(const void *addr, size_t page_size)
259 {
260 return (void *)((uintptr_t) addr & ~(page_size - 1));
261 }
262
ofi_get_page_end(const void * addr,size_t page_size)263 static inline void *ofi_get_page_end(const void *addr, size_t page_size)
264 {
265 return ofi_get_page_start((const char *) addr + page_size -1, page_size);
266 }
267
268 static inline size_t
ofi_get_page_bytes(const void * addr,size_t len,size_t page_size)269 ofi_get_page_bytes(const void *addr, size_t len, size_t page_size)
270 {
271 return (char *)ofi_get_page_end((const char *) addr + len, page_size) -
272 (char *)ofi_get_page_start(addr, page_size);
273 }
274
275 #define FI_TAG_GENERIC 0xAAAAAAAAAAAAAAAAULL
276
277
278 uint64_t ofi_max_tag(uint64_t mem_tag_format);
279 uint64_t ofi_tag_format(uint64_t max_tag);
280 uint8_t ofi_msb(uint64_t num);
281 uint8_t ofi_lsb(uint64_t num);
282
283 int ofi_send_allowed(uint64_t caps);
284 int ofi_recv_allowed(uint64_t caps);
285 int ofi_rma_initiate_allowed(uint64_t caps);
286 int ofi_rma_target_allowed(uint64_t caps);
287 int ofi_ep_bind_valid(const struct fi_provider *prov, struct fid *bfid,
288 uint64_t flags);
289 int ofi_check_rx_mode(const struct fi_info *info, uint64_t flags);
290
291 uint64_t ofi_gettime_ns(void);
292 uint64_t ofi_gettime_us(void);
293 uint64_t ofi_gettime_ms(void);
294
ofi_timeout_time(int timeout)295 static inline uint64_t ofi_timeout_time(int timeout)
296 {
297 return (timeout >= 0) ? ofi_gettime_ms() + timeout : 0;
298 }
299
ofi_adjust_timeout(uint64_t timeout_time,int * timeout)300 static inline int ofi_adjust_timeout(uint64_t timeout_time, int *timeout)
301 {
302 if (*timeout >= 0) {
303 *timeout = (int) (timeout_time - ofi_gettime_ms());
304 return (*timeout <= 0) ? -FI_ETIMEDOUT : 0;
305 }
306 return 0;
307 }
308
309 #define OFI_ENUM_VAL(X) X
310 #define OFI_STR(X) #X
311 #define OFI_STR_INT(X) OFI_STR(X)
312
313
314 /*
315 * Key Index
316 */
317
318 /*
319 * The key_idx object and related functions can be used to generate unique keys
320 * from an index. The key and index would refer to an object defined by the user.
321 * A local endpoint can exchange this key with a remote endpoint in the first message.
322 * The remote endpoint would then use this key in subsequent messages to reference
323 * the correct object at the local endpoint.
324 */
325 struct ofi_key_idx {
326 uint64_t seq_no;
327 /* The uniqueness of the generated key would depend on how many bits are
328 * used for the index */
329 uint8_t idx_bits;
330 };
331
ofi_key_idx_init(struct ofi_key_idx * key_idx,uint8_t idx_bits)332 static inline void ofi_key_idx_init(struct ofi_key_idx *key_idx, uint8_t idx_bits)
333 {
334 key_idx->seq_no = 0;
335 key_idx->idx_bits = idx_bits;
336 }
337
ofi_idx2key(struct ofi_key_idx * key_idx,uint64_t idx)338 static inline uint64_t ofi_idx2key(struct ofi_key_idx *key_idx, uint64_t idx)
339 {
340 return ((++(key_idx->seq_no)) << key_idx->idx_bits) | idx;
341 }
342
ofi_key2idx(struct ofi_key_idx * key_idx,uint64_t key)343 static inline uint64_t ofi_key2idx(struct ofi_key_idx *key_idx, uint64_t key)
344 {
345 return key & ((1ULL << key_idx->idx_bits) - 1);
346 }
347
ofi_xorshift_random(uint32_t val)348 static inline uint32_t ofi_xorshift_random(uint32_t val)
349 {
350 /*
351 * Xorshift Random Number Generators are from 224.
352 * R. P. Brent, Some long-period random number
353 * generators using shifts and xors, ANZIAM
354 * Journal 48 (CTAC2006), C188-C202, 2007.
355 * Presented at the 13th Biennial Computational
356 * Techniques and Applications
357 * Conference (CTAC06), Townsville, 2-5 July 2006.
358 * arXiv:1004.3115v1
359 */
360 val ^= val << 13;
361 val ^= val >> 17;
362 val ^= val << 5;
363
364 return val;
365 }
366
367
368 #ifdef __cplusplus
369 }
370 #endif
371
372 #endif /* _OFI_H_ */
373