1 /*
2 * Copyright (c) 2015-2017 Los Alamos National Security, LLC.
3 * All rights reserved.
4 * Copyright (c) 2015-2017 Cray Inc. All rights reserved.
5 * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36 #if HAVE_CONFIG_H
37 #include <config.h>
38 #endif /* HAVE_CONFIG_H */
39
40 #ifndef _GNIX_UTIL_H_
41 #define _GNIX_UTIL_H_
42
43 #include <stdio.h>
44 #include <ofi.h>
45
46 extern struct fi_provider gnix_prov;
47 #if HAVE_CRITERION
48 extern int gnix_first_pe_on_node; /* globally visible for criterion */
49 #endif
50
51 /*
52 * For debug logging (ENABLE_DEBUG)
53 * Q: should this just always be available?
54 */
55 #ifndef ENABLE_DEBUG
56
57 #define GNIX_LOG_INTERNAL(FI_LOG_FN, LEVEL, subsystem, fmt, ...) \
58 FI_LOG_FN(&gnix_prov, subsystem, fmt, ##__VA_ARGS__)
59
60 #define GNIX_FI_PRINT(prov, subsystem, ...)
61
62 #else
63
64 /* defined in gnix_init.c */
65 extern __thread pid_t gnix_debug_pid;
66 extern __thread uint32_t gnix_debug_tid;
67 extern ofi_atomic32_t gnix_debug_next_tid;
68
69 #define GNIX_FI_PRINT(prov, subsystem, ...) \
70 do { \
71 fi_log(prov, FI_LOG_WARN, subsystem, \
72 __func__, __LINE__, __VA_ARGS__); \
73 } while (0)
74
75
76 /* These macros are used to prepend the log message with the pid and
77 * unique thread id. Do not use them directly. Rather use the normal
78 * GNIX_* macros.
79 */
80 #define GNIX_LOG_INTERNAL(FI_LOG_FN, LEVEL, subsystem, fmt, ...) \
81 do { \
82 if (fi_log_enabled(&gnix_prov, LEVEL, subsystem)) { \
83 const int fmt_len = 256; \
84 char new_fmt[fmt_len]; \
85 if (gnix_debug_tid == ~(uint32_t) 0) { \
86 gnix_debug_tid = ofi_atomic_inc32(&gnix_debug_next_tid); \
87 } \
88 if (gnix_debug_pid == ~(uint32_t) 0) { \
89 gnix_debug_pid = getpid(); \
90 } \
91 snprintf(new_fmt, fmt_len, "[%%d:%%d] %s", fmt); \
92 FI_LOG_FN(&gnix_prov, subsystem, new_fmt, \
93 gnix_debug_pid, gnix_debug_tid, ##__VA_ARGS__); \
94 } \
95 } while (0)
96
97 #endif
98
99 #define GNIX_WARN(subsystem, ...) \
100 GNIX_LOG_INTERNAL(FI_WARN, FI_LOG_WARN, subsystem, __VA_ARGS__)
101 #define GNIX_TRACE(subsystem, ...) \
102 GNIX_LOG_INTERNAL(FI_TRACE, FI_LOG_TRACE, subsystem, __VA_ARGS__)
103 #define GNIX_INFO(subsystem, ...) \
104 GNIX_LOG_INTERNAL(FI_INFO, FI_LOG_INFO, subsystem, __VA_ARGS__)
105 #if ENABLE_DEBUG
106 #define GNIX_DEBUG(subsystem, ...) \
107 GNIX_LOG_INTERNAL(FI_DBG, FI_LOG_DEBUG, subsystem, __VA_ARGS__)
108 #define GNIX_DBG_TRACE(subsystem, ...) \
109 GNIX_LOG_INTERNAL(FI_TRACE, FI_LOG_TRACE, subsystem, __VA_ARGS__)
110 #else
111 #define GNIX_DEBUG(subsystem, ...) \
112 do {} while (0)
113 #define GNIX_DBG_TRACE(subsystem, ...) \
114 do {} while (0)
115 #endif
116
117 #define GNIX_ERR(subsystem, ...) \
118 GNIX_LOG_INTERNAL(GNIX_FI_PRINT, FI_LOG_WARN, subsystem, __VA_ARGS__)
119 #define GNIX_FATAL(subsystem, ...) \
120 do { \
121 GNIX_LOG_INTERNAL(GNIX_FI_PRINT, FI_LOG_WARN, subsystem, __VA_ARGS__); \
122 abort(); \
123 } while (0)
124
125 #if 1
126 #define GNIX_LOG_DUMP_TXD(txd)
127 #else
128 #define GNIX_LOG_DUMP_TXD(txd) \
129 do { \
130 gni_mem_handle_t *tl_mdh = &(txd)->gni_desc.local_mem_hndl; \
131 gni_mem_handle_t *tr_mdh = &(txd)->gni_desc.remote_mem_hndl; \
132 GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", \
133 (txd)->gni_desc.local_addr, \
134 (txd)->gni_desc.remote_addr, \
135 (txd)->gni_desc.length); \
136 GNIX_INFO(FI_LOG_EP_DATA, \
137 "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", \
138 *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), \
139 *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), \
140 fab_req->amo.rem_mr_key); \
141 } while (0)
142 #endif
143
144 /* slist and dlist utilities */
145 #include "ofi_list.h"
146
dlist_node_init(struct dlist_entry * e)147 static inline void dlist_node_init(struct dlist_entry *e)
148 {
149 e->prev = e->next = NULL;
150 }
151
152 #define DLIST_IN_LIST(e) e.prev != e.next
153
154 #define DLIST_HEAD(dlist) struct dlist_entry dlist = { &(dlist), &(dlist) }
155
156 #define dlist_entry(e, type, member) container_of(e, type, member)
157
158 #define dlist_first_entry(h, type, member) \
159 (dlist_empty(h) ? NULL : dlist_entry((h)->next, type, member))
160
161 /* Iterate over the entries in the list */
162 #define dlist_for_each(h, e, member) \
163 for (e = dlist_first_entry(h, typeof(*e), member); \
164 e && (&e->member != h); \
165 e = dlist_entry((&e->member)->next, typeof(*e), member))
166
167 /* Iterate over the entries in the list, possibly deleting elements */
168 #define dlist_for_each_safe(h, e, n, member) \
169 for (e = dlist_first_entry(h, typeof(*e), member), \
170 n = e ? dlist_entry((&e->member)->next, \
171 typeof(*e), member) : NULL; \
172 e && (&e->member != h); \
173 e = n, n = dlist_entry((&e->member)->next, typeof(*e), member))
174
175 #define rwlock_t pthread_rwlock_t
176 #define rwlock_init(lock) pthread_rwlock_init(lock, NULL)
177 #define rwlock_destroy(lock) pthread_rwlock_destroy(lock)
178 #define rwlock_wrlock(lock) pthread_rwlock_wrlock(lock)
179 #define rwlock_rdlock(lock) pthread_rwlock_rdlock(lock)
180 #define rwlock_unlock(lock) pthread_rwlock_unlock(lock)
181
182 /*
183 * prototypes
184 */
185 int _gnix_get_cq_limit(void);
186 int gnixu_get_rdma_credentials(void *addr, uint8_t *ptag, uint32_t *cookie);
187 int gnixu_to_fi_errno(int err);
188
189 int _gnix_task_is_not_app(void);
190 int _gnix_job_enable_unassigned_cpus(void);
191 int _gnix_job_disable_unassigned_cpus(void);
192 int _gnix_job_enable_affinity_apply(void);
193 int _gnix_job_disable_affinity_apply(void);
194
195 void _gnix_app_cleanup(void);
196 int _gnix_job_fma_limit(uint32_t dev_id, uint8_t ptag, uint32_t *limit);
197 int _gnix_job_cq_limit(uint32_t dev_id, uint8_t ptag, uint32_t *limit);
198 int _gnix_pes_on_node(uint32_t *num_pes);
199 int _gnix_pe_node_rank(int *pe_node_rank);
200 int _gnix_nics_per_rank(uint32_t *nics_per_rank);
201 void _gnix_dump_gni_res(uint8_t ptag);
202 int _gnix_get_num_corespec_cpus(uint32_t *num_core_spec_cpus);
203
204 struct gnix_reference {
205 ofi_atomic32_t references;
206 void (*destruct)(void *obj);
207 };
208
209 /* Should not be used unless the reference counting variable has a
210 * non-standard name
211 */
212 #define __ref_get(ptr, var) \
213 ({ \
214 struct gnix_reference *ref = &(ptr)->var; \
215 int references_held = ofi_atomic_inc32(&ref->references); \
216 GNIX_DEBUG(FI_LOG_CORE, "%p refs %d\n", \
217 ref, references_held); \
218 assert(references_held > 0); \
219 references_held; })
220
221 #define __ref_put(ptr, var) \
222 ({ \
223 struct gnix_reference *ref = &(ptr)->var; \
224 int references_held = ofi_atomic_dec32(&ref->references); \
225 GNIX_DEBUG(FI_LOG_CORE, "%p refs %d\n", \
226 ref, references_held); \
227 assert(references_held >= 0); \
228 if (references_held == 0) \
229 ref->destruct((void *) (ptr)); \
230 references_held; })
231
232 /* by default, all of the gnix reference counting variables are
233 * named 'ref_cnt'. The macros provided below will autofill the var arg.
234 */
235 #define _gnix_ref_get(ptr) __ref_get(ptr, ref_cnt)
236 #define _gnix_ref_put(ptr) __ref_put(ptr, ref_cnt)
237
238 /**
239 * Only allow FI_REMOTE_CQ_DATA when the EP cap, FI_RMA_EVENT, is also set.
240 *
241 * @return zero if FI_REMOTE_CQ_DATA is not permitted; otherwise one.
242 */
243 #define GNIX_ALLOW_FI_REMOTE_CQ_DATA(_flags, _ep_caps) \
244 (((_flags) & FI_REMOTE_CQ_DATA) && \
245 ((_ep_caps) & FI_RMA_EVENT))
246
_gnix_ref_init(struct gnix_reference * ref,int initial_value,void (* destruct)(void *))247 static inline void _gnix_ref_init(
248 struct gnix_reference *ref,
249 int initial_value,
250 void (*destruct)(void *))
251 {
252 ofi_atomic_initialize32(&ref->references, initial_value);
253 GNIX_DEBUG(FI_LOG_CORE, "%p refs %d\n",
254 ref, initial_value);
255 ref->destruct = destruct;
256 }
257
258 #define __STRINGIFY(expr) #expr
259 #define STRINGIFY(expr) __STRINGIFY(expr)
260
261 #define __COND_FUNC(cond, lock, func) \
262 do { \
263 if ((cond)) { \
264 func(lock); \
265 } \
266 } while (0)
267
268 #define COND_ACQUIRE(cond, lock) \
269 __COND_FUNC((cond), (lock), fastlock_acquire)
270 #define COND_READ_ACQUIRE(cond, lock) \
271 __COND_FUNC((cond), (lock), rwlock_rdlock)
272 #define COND_WRITE_ACQUIRE(cond, lock) \
273 __COND_FUNC((cond), (lock), rwlock_wrlock)
274
275 #define COND_RELEASE(cond, lock) \
276 __COND_FUNC((cond), (lock), fastlock_release)
277 #define COND_RW_RELEASE(cond, lock) \
278 __COND_FUNC((cond), (lock), rwlock_unlock)
279 #ifdef __GNUC__
280 #define __PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
281 #else
282 #define __PREFETCH(addr, rw, locality) ((void *) 0)
283 #endif
284
285 #define READ_PREFETCH(addr) __PREFETCH(addr, 0, 3)
286 #define WRITE_PREFETCH(addr) __PREFETCH(addr, 1, 3)
287
288 #endif
289