1 /*
2 * Copyright (c) 2013 Mellanox Technologies, Inc.
3 * All rights reserved.
4 * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
5 * $COPYRIGHT$
6 *
7 * Additional copyrights may follow
8 *
9 * $HEADER$
10 */
11 /**
12 * @file
13 */
14 #ifndef MCA_MEMHEAP_BASE_H
15 #define MCA_MEMHEAP_BASE_H
16
17 #include "oshmem_config.h"
18 #include "opal/class/opal_list.h"
19 #include "opal/class/opal_value_array.h"
20 #include "oshmem/mca/mca.h"
21
22 #include "oshmem/mca/memheap/memheap.h"
23
24 BEGIN_C_DECLS
25
26 /*
27 * Global functions for MCA: overall MEMHEAP open and close
28 */
29 OSHMEM_DECLSPEC int mca_memheap_base_select(void);
30
31 /*
32 * Globals
33 */
34
35 /* only used within base -- no need to DECLSPEC */
36 #define MEMHEAP_BASE_MIN_ORDER 3 /* forces 64 bit alignment */
37 #define MEMHEAP_BASE_PAGE_ORDER 21
38 #define MEMHEAP_BASE_PRIVATE_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* should be at least the same as a huge page size */
39 #define MEMHEAP_BASE_MIN_SIZE (1ULL << MEMHEAP_BASE_PAGE_ORDER) /* must fit into at least one huge page */
40
41 extern int mca_memheap_base_already_opened;
42 extern int mca_memheap_base_key_exchange;
43
44 #define MCA_MEMHEAP_MAX_SEGMENTS 32
45 #define HEAP_SEG_INDEX 0
46 #define MCA_MEMHEAP_SEG_COUNT 2
47
48 #define MEMHEAP_SEG_INVALID 0xFFFF
49
50
51 typedef struct mca_memheap_base_config {
52 long device_nic_mem_seg_size; /* Used for SHMEM_HINT_DEVICE_NIC_MEM */
53 } mca_memheap_base_config_t;
54
55
56 typedef struct mca_memheap_map {
57 map_segment_t mem_segs[MCA_MEMHEAP_MAX_SEGMENTS]; /* TODO: change into pointer array */
58 int n_segments;
59 int num_transports;
60 } mca_memheap_map_t;
61
62 extern mca_memheap_map_t mca_memheap_base_map;
63 extern mca_memheap_base_config_t mca_memheap_base_config;
64
65 int mca_memheap_base_alloc_init(mca_memheap_map_t *, size_t, long, char *);
66 void mca_memheap_base_alloc_exit(mca_memheap_map_t *);
67 int mca_memheap_base_static_init(mca_memheap_map_t *);
68 void mca_memheap_base_static_exit(mca_memheap_map_t *);
69 int mca_memheap_base_reg(mca_memheap_map_t *);
70 int mca_memheap_base_dereg(mca_memheap_map_t *);
71 int memheap_oob_init(mca_memheap_map_t *);
72 void memheap_oob_destruct(void);
73
74 OSHMEM_DECLSPEC int mca_memheap_base_is_symmetric_addr(const void* va);
75 OSHMEM_DECLSPEC sshmem_mkey_t *mca_memheap_base_get_mkey(void* va,
76 int tr_id);
77 OSHMEM_DECLSPEC sshmem_mkey_t * mca_memheap_base_get_cached_mkey_slow(shmem_ctx_t ctx,
78 map_segment_t *s,
79 int pe,
80 void* va,
81 int btl_id,
82 void** rva);
83 OSHMEM_DECLSPEC void mca_memheap_modex_recv_all(void);
84
85 /* This function is for internal usage only
86 * return value:
87 * 0 - addr is not symmetric address
88 * 1 - addr is part of user memheap
89 * 2 - addr is part of private memheap
90 * 3 - addr is static variable
91 */
92 typedef enum {
93 ADDR_INVALID = 0, ADDR_USER, ADDR_PRIVATE, ADDR_STATIC,
94 } addr_type_t;
95
96 OSHMEM_DECLSPEC int mca_memheap_base_detect_addr_type(void* va);
97
memheap_log2(unsigned long long val)98 static inline unsigned memheap_log2(unsigned long long val)
99 {
100 /* add 1 if val is NOT a power of 2 (to do the ceil) */
101 unsigned int count = (val & (val - 1) ? 1 : 0);
102
103 while (val > 0) {
104 val = val >> 1;
105 count++;
106 }
107
108 return count > 0 ? count - 1 : 0;
109 }
110
memheap_down_align_addr(void * addr,unsigned int shift)111 static inline void *memheap_down_align_addr(void* addr, unsigned int shift)
112 {
113 return (void*) (((intptr_t) addr) & (~(intptr_t) 0) << shift);
114 }
115
memheap_up_align_addr(void * addr,unsigned int shift)116 static inline void *memheap_up_align_addr(void*addr, unsigned int shift)
117 {
118 return (void*) ((((intptr_t) addr) | ~((~(intptr_t) 0) << shift)));
119 }
120
memheap_align(unsigned long top)121 static inline unsigned long long memheap_align(unsigned long top)
122 {
123 return ((top + MEMHEAP_BASE_MIN_SIZE - 1) & ~(MEMHEAP_BASE_MIN_SIZE - 1));
124 }
125
126 /*
127 * MCA framework
128 */
129 OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_memheap_base_framework;
130
131 /* ******************************************************************** */
132 #ifdef __BASE_FILE__
133 #define __SPML_FILE__ __BASE_FILE__
134 #else
135 #define __SPML_FILE__ __FILE__
136 #endif
137
138 #ifdef OPAL_ENABLE_DEBUG
139 #define MEMHEAP_VERBOSE(level, ...) \
140 oshmem_output_verbose(level, oshmem_memheap_base_framework.framework_output, \
141 "%s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
142 #else
143 #define MEMHEAP_VERBOSE(level, ...)
144 #endif
145
146 #define MEMHEAP_ERROR(...) \
147 oshmem_output(oshmem_memheap_base_framework.framework_output, \
148 "Error %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
149
150 #define MEMHEAP_WARN(...) \
151 oshmem_output_verbose(0, oshmem_memheap_base_framework.framework_output, \
152 "Warning %s:%d - %s()", __SPML_FILE__, __LINE__, __func__, __VA_ARGS__)
153
154 extern int mca_memheap_seg_cmp(const void *k, const void *v);
155
156 /* Turn ON/OFF debug output from build (default 0) */
157 #ifndef MEMHEAP_BASE_DEBUG
158 #define MEMHEAP_BASE_DEBUG 0
159 #endif
160 #define MEMHEAP_VERBOSE_FASTPATH(...)
161
162 extern mca_memheap_map_t* memheap_map;
163
map_segment_is_va_in(map_base_segment_t * s,void * va)164 static inline int map_segment_is_va_in(map_base_segment_t *s, void *va)
165 {
166 return (va >= s->va_base && va < s->va_end);
167 }
168
memheap_find_seg(int segno)169 static inline map_segment_t *memheap_find_seg(int segno)
170 {
171 return &mca_memheap_base_map.mem_segs[segno];
172 }
173
memheap_is_va_in_segment(void * va,int segno)174 static inline int memheap_is_va_in_segment(void *va, int segno)
175 {
176 return map_segment_is_va_in(&memheap_find_seg(segno)->super, va);
177 }
178
memheap_find_segnum(void * va)179 static inline int memheap_find_segnum(void *va)
180 {
181 int i;
182
183 for (i = 0; i < mca_memheap_base_map.n_segments; i++) {
184 if (memheap_is_va_in_segment(va, i)) {
185 return i;
186 }
187 }
188 return MEMHEAP_SEG_INVALID;
189 }
190
memheap_va2rva(void * va,void * local_base,void * remote_base)191 static inline void* memheap_va2rva(void* va, void* local_base, void* remote_base)
192 {
193 return (void*) (remote_base > local_base ?
194 (uintptr_t)va + ((uintptr_t)remote_base - (uintptr_t)local_base) :
195 (uintptr_t)va - ((uintptr_t)local_base - (uintptr_t)remote_base));
196 }
197
map_segment_va2rva(mkey_segment_t * seg,void * va)198 static inline void *map_segment_va2rva(mkey_segment_t *seg, void *va)
199 {
200 return memheap_va2rva(va, seg->super.va_base, seg->rva_base);
201 }
202
map_segment_find_va(map_base_segment_t * segs,size_t elem_size,void * va)203 static inline map_base_segment_t *map_segment_find_va(map_base_segment_t *segs,
204 size_t elem_size, void *va)
205 {
206 map_base_segment_t *rseg;
207 int i;
208
209 for (i = 0; i < MCA_MEMHEAP_MAX_SEGMENTS; i++) {
210 rseg = (map_base_segment_t *)((char *)segs + elem_size * i);
211 if (OPAL_LIKELY(map_segment_is_va_in(rseg, va))) {
212 return rseg;
213 }
214 }
215
216 return NULL;
217 }
218
219 void mkey_segment_init(mkey_segment_t *seg, sshmem_mkey_t *mkey, uint32_t segno);
220
memheap_find_va(void * va)221 static inline map_segment_t *memheap_find_va(void* va)
222 {
223 map_segment_t *s = NULL;
224 int i;
225
226 for (i = 0; i < memheap_map->n_segments; i++) {
227 if (memheap_is_va_in_segment(va, i)) {
228 s = &memheap_map->mem_segs[i];
229 break;
230 }
231 }
232
233 #if MEMHEAP_BASE_DEBUG == 1
234 if (s) {
235 MEMHEAP_VERBOSE(5, "match seg#%02ld: 0x%llX - 0x%llX %llu bytes va=%p",
236 s - memheap_map->mem_segs,
237 (long long)s->super.va_base,
238 (long long)s->super.va_end,
239 (long long)(s->super.va_end - s->super.va_base),
240 (void *)va);
241 }
242 #endif
243 return s;
244 }
245
mca_memheap_base_get_cached_mkey(shmem_ctx_t ctx,int pe,void * va,int btl_id,void ** rva)246 static inline sshmem_mkey_t *mca_memheap_base_get_cached_mkey(shmem_ctx_t ctx,
247 int pe,
248 void* va,
249 int btl_id,
250 void** rva)
251 {
252 map_segment_t *s;
253 sshmem_mkey_t *mkey;
254
255 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p", pe, va);
256 s = memheap_find_va(va);
257 if (OPAL_UNLIKELY(NULL == s))
258 return NULL ;
259
260 if (OPAL_UNLIKELY(!MAP_SEGMENT_IS_VALID(s)))
261 return NULL ;
262
263 if (OPAL_UNLIKELY(pe == oshmem_my_proc_id())) {
264 *rva = va;
265 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (local) %lx %p", pe, va,
266 s->mkeys[btl_id].u.key, *rva);
267 return &s->mkeys[btl_id];
268 }
269
270 if (OPAL_LIKELY(s->mkeys_cache[pe])) {
271 mkey = &s->mkeys_cache[pe][btl_id];
272 *rva = memheap_va2rva(va, s->super.va_base, mkey->va_base);
273 MEMHEAP_VERBOSE_FASTPATH(10, "rkey: pe=%d va=%p -> (cached) %lx %p", pe, (void *)va, mkey->u.key, (void *)*rva);
274 return mkey;
275 }
276
277 return mca_memheap_base_get_cached_mkey_slow(ctx, s, pe, va, btl_id, rva);
278 }
279
mca_memheap_base_num_transports(void)280 static inline int mca_memheap_base_num_transports(void)
281 {
282 return memheap_map->num_transports;
283 }
284
mca_memheap_seg2base_va(int seg)285 static inline void* mca_memheap_seg2base_va(int seg)
286 {
287 return memheap_map->mem_segs[seg].super.va_base;
288 }
289
290 END_C_DECLS
291
292 #endif /* MCA_MEMHEAP_BASE_H */
293